1	//===- llvm/CodeGen/GlobalISel/IRTranslator.cpp - IRTranslator ---*- C++ -*-==//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	/// \file
9	/// This file implements the IRTranslator class.
10	//===----------------------------------------------------------------------===//
11
12	#include "llvm/CodeGen/GlobalISel/IRTranslator.h"
13	#include "llvm/ADT/PostOrderIterator.h"
14	#include "llvm/ADT/STLExtras.h"
15	#include "llvm/ADT/ScopeExit.h"
16	#include "llvm/ADT/SmallSet.h"
17	#include "llvm/ADT/SmallVector.h"
18	#include "llvm/Analysis/AliasAnalysis.h"
19	#include "llvm/Analysis/AssumptionCache.h"
20	#include "llvm/Analysis/BranchProbabilityInfo.h"
21	#include "llvm/Analysis/Loads.h"
22	#include "llvm/Analysis/OptimizationRemarkEmitter.h"
23	#include "llvm/Analysis/ValueTracking.h"
24	#include "llvm/Analysis/VectorUtils.h"
25	#include "llvm/CodeGen/Analysis.h"
26	#include "llvm/CodeGen/GlobalISel/CSEInfo.h"
27	#include "llvm/CodeGen/GlobalISel/CSEMIRBuilder.h"
28	#include "llvm/CodeGen/GlobalISel/CallLowering.h"
29	#include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"
30	#include "llvm/CodeGen/GlobalISel/InlineAsmLowering.h"
31	#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
32	#include "llvm/CodeGen/LowLevelTypeUtils.h"
33	#include "llvm/CodeGen/MachineBasicBlock.h"
34	#include "llvm/CodeGen/MachineFrameInfo.h"
35	#include "llvm/CodeGen/MachineFunction.h"
36	#include "llvm/CodeGen/MachineInstrBuilder.h"
37	#include "llvm/CodeGen/MachineMemOperand.h"
38	#include "llvm/CodeGen/MachineModuleInfo.h"
39	#include "llvm/CodeGen/MachineOperand.h"
40	#include "llvm/CodeGen/MachineRegisterInfo.h"
41	#include "llvm/CodeGen/RuntimeLibcalls.h"
42	#include "llvm/CodeGen/StackProtector.h"
43	#include "llvm/CodeGen/SwitchLoweringUtils.h"
44	#include "llvm/CodeGen/TargetFrameLowering.h"
45	#include "llvm/CodeGen/TargetInstrInfo.h"
46	#include "llvm/CodeGen/TargetLowering.h"
47	#include "llvm/CodeGen/TargetOpcodes.h"
48	#include "llvm/CodeGen/TargetPassConfig.h"
49	#include "llvm/CodeGen/TargetRegisterInfo.h"
50	#include "llvm/CodeGen/TargetSubtargetInfo.h"
51	#include "llvm/CodeGenTypes/LowLevelType.h"
52	#include "llvm/IR/BasicBlock.h"
53	#include "llvm/IR/CFG.h"
54	#include "llvm/IR/Constant.h"
55	#include "llvm/IR/Constants.h"
56	#include "llvm/IR/DataLayout.h"
57	#include "llvm/IR/DerivedTypes.h"
58	#include "llvm/IR/DiagnosticInfo.h"
59	#include "llvm/IR/Function.h"
60	#include "llvm/IR/GetElementPtrTypeIterator.h"
61	#include "llvm/IR/InlineAsm.h"
62	#include "llvm/IR/InstrTypes.h"
63	#include "llvm/IR/Instructions.h"
64	#include "llvm/IR/IntrinsicInst.h"
65	#include "llvm/IR/Intrinsics.h"
66	#include "llvm/IR/IntrinsicsAMDGPU.h"
67	#include "llvm/IR/LLVMContext.h"
68	#include "llvm/IR/Metadata.h"
69	#include "llvm/IR/PatternMatch.h"
70	#include "llvm/IR/Statepoint.h"
71	#include "llvm/IR/Type.h"
72	#include "llvm/IR/User.h"
73	#include "llvm/IR/Value.h"
74	#include "llvm/InitializePasses.h"
75	#include "llvm/MC/MCContext.h"
76	#include "llvm/Pass.h"
77	#include "llvm/Support/Casting.h"
78	#include "llvm/Support/CodeGen.h"
79	#include "llvm/Support/Debug.h"
80	#include "llvm/Support/ErrorHandling.h"
81	#include "llvm/Support/MathExtras.h"
82	#include "llvm/Support/raw_ostream.h"
83	#include "llvm/Target/TargetIntrinsicInfo.h"
84	#include "llvm/Target/TargetMachine.h"
85	#include "llvm/Transforms/Utils/Local.h"
86	#include "llvm/Transforms/Utils/MemoryOpRemark.h"
87	#include <algorithm>
88	#include <cassert>
89	#include <cstdint>
90	#include <iterator>
91	#include <optional>
92	#include <string>
93	#include <utility>
94	#include <vector>
95
96	#define DEBUG_TYPE "irtranslator"
97
98	using namespace llvm;
99
100	static cl::opt<bool>
101	EnableCSEInIRTranslator("enable-cse-in-irtranslator",
102	cl::desc("Should enable CSE in irtranslator"),
103	cl::Optional, cl::init(Val: false));
104	char IRTranslator::ID = 0;
105
106	INITIALIZE_PASS_BEGIN(IRTranslator, DEBUG_TYPE, "IRTranslator LLVM IR -> MI",
107	false, false)
108	INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
109	INITIALIZE_PASS_DEPENDENCY(GISelCSEAnalysisWrapperPass)
110	INITIALIZE_PASS_DEPENDENCY(BlockFrequencyInfoWrapperPass)
111	INITIALIZE_PASS_DEPENDENCY(StackProtector)
112	INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
113	INITIALIZE_PASS_END(IRTranslator, DEBUG_TYPE, "IRTranslator LLVM IR -> MI",
114	false, false)
115
116	static void reportTranslationError(MachineFunction &MF,
117	const TargetPassConfig &TPC,
118	OptimizationRemarkEmitter &ORE,
119	OptimizationRemarkMissed &R) {
120	MF.getProperties().set(MachineFunctionProperties::Property::FailedISel);
121
122	// Print the function name explicitly if we don't have a debug location (which
123	// makes the diagnostic less useful) or if we're going to emit a raw error.
124	if (!R.getLocation().isValid() || TPC.isGlobalISelAbortEnabled())
125	R << (" (in function: " + MF.getName() + ")").str();
126
127	if (TPC.isGlobalISelAbortEnabled())
128	report_fatal_error(reason: Twine(R.getMsg()));
129	else
130	ORE.emit(OptDiag&: R);
131	}
132
133	IRTranslator::IRTranslator(CodeGenOptLevel optlevel)
134	: MachineFunctionPass(ID), OptLevel(optlevel) {}
135
136	#ifndef NDEBUG
137	namespace {
138	/// Verify that every instruction created has the same DILocation as the
139	/// instruction being translated.
140	class DILocationVerifier : public GISelChangeObserver {
141	const Instruction *CurrInst = nullptr;
142
143	public:
144	DILocationVerifier() = default;
145	~DILocationVerifier() = default;
146
147	const Instruction *getCurrentInst() const { return CurrInst; }
148	void setCurrentInst(const Instruction *Inst) { CurrInst = Inst; }
149
150	void erasingInstr(MachineInstr &MI) override {}
151	void changingInstr(MachineInstr &MI) override {}
152	void changedInstr(MachineInstr &MI) override {}
153
154	void createdInstr(MachineInstr &MI) override {
155	assert(getCurrentInst() && "Inserted instruction without a current MI");
156
157	// Only print the check message if we're actually checking it.
158	#ifndef NDEBUG
159	LLVM_DEBUG(dbgs() << "Checking DILocation from " << *CurrInst
160	<< " was copied to " << MI);
161	#endif
162	// We allow insts in the entry block to have no debug loc because
163	// they could have originated from constants, and we don't want a jumpy
164	// debug experience.
165	assert((CurrInst->getDebugLoc() == MI.getDebugLoc() ||
166	(MI.getParent()->isEntryBlock() && !MI.getDebugLoc()) ||
167	(MI.isDebugInstr())) &&
168	"Line info was not transferred to all instructions");
169	}
170	};
171	} // namespace
172	#endif // ifndef NDEBUG
173
174
175	void IRTranslator::getAnalysisUsage(AnalysisUsage &AU) const {
176	AU.addRequired<StackProtector>();
177	AU.addRequired<TargetPassConfig>();
178	AU.addRequired<GISelCSEAnalysisWrapperPass>();
179	AU.addRequired<AssumptionCacheTracker>();
180	if (OptLevel != CodeGenOptLevel::None) {
181	AU.addRequired<BranchProbabilityInfoWrapperPass>();
182	AU.addRequired<AAResultsWrapperPass>();
183	}
184	AU.addRequired<TargetLibraryInfoWrapperPass>();
185	AU.addPreserved<TargetLibraryInfoWrapperPass>();
186	getSelectionDAGFallbackAnalysisUsage(AU);
187	MachineFunctionPass::getAnalysisUsage(AU);
188	}
189
190	IRTranslator::ValueToVRegInfo::VRegListT &
191	IRTranslator::allocateVRegs(const Value &Val) {
192	auto VRegsIt = VMap.findVRegs(V: Val);
193	if (VRegsIt != VMap.vregs_end())
194	return *VRegsIt->second;
195	auto *Regs = VMap.getVRegs(V: Val);
196	auto *Offsets = VMap.getOffsets(V: Val);
197	SmallVector<LLT, 4> SplitTys;
198	computeValueLLTs(DL: *DL, Ty&: *Val.getType(), ValueTys&: SplitTys,
199	Offsets: Offsets->empty() ? Offsets : nullptr);
200	for (unsigned i = 0; i < SplitTys.size(); ++i)
201	Regs->push_back(Elt: 0);
202	return *Regs;
203	}
204
205	ArrayRef<Register> IRTranslator::getOrCreateVRegs(const Value &Val) {
206	auto VRegsIt = VMap.findVRegs(V: Val);
207	if (VRegsIt != VMap.vregs_end())
208	return *VRegsIt->second;
209
210	if (Val.getType()->isVoidTy())
211	return *VMap.getVRegs(V: Val);
212
213	// Create entry for this type.
214	auto *VRegs = VMap.getVRegs(V: Val);
215	auto *Offsets = VMap.getOffsets(V: Val);
216
217	if (!Val.getType()->isTokenTy())
218	assert(Val.getType()->isSized() &&
219	"Don't know how to create an empty vreg");
220
221	SmallVector<LLT, 4> SplitTys;
222	computeValueLLTs(DL: *DL, Ty&: *Val.getType(), ValueTys&: SplitTys,
223	Offsets: Offsets->empty() ? Offsets : nullptr);
224
225	if (!isa<Constant>(Val)) {
226	for (auto Ty : SplitTys)
227	VRegs->push_back(Elt: MRI->createGenericVirtualRegister(Ty));
228	return *VRegs;
229	}
230
231	if (Val.getType()->isAggregateType()) {
232	// UndefValue, ConstantAggregateZero
233	auto &C = cast<Constant>(Val);
234	unsigned Idx = 0;
235	while (auto Elt = C.getAggregateElement(Elt: Idx++)) {
236	auto EltRegs = getOrCreateVRegs(Val: *Elt);
237	llvm::copy(Range&: EltRegs, Out: std::back_inserter(x&: *VRegs));
238	}
239	} else {
240	assert(SplitTys.size() == 1 && "unexpectedly split LLT");
241	VRegs->push_back(Elt: MRI->createGenericVirtualRegister(Ty: SplitTys[0]));
242	bool Success = translate(C: cast<Constant>(Val), Reg: VRegs->front());
243	if (!Success) {
244	OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
245	MF->getFunction().getSubprogram(),
246	&MF->getFunction().getEntryBlock());
247	R << "unable to translate constant: " << ore::NV("Type", Val.getType());
248	reportTranslationError(MF&: *MF, TPC: *TPC, ORE&: *ORE, R);
249	return *VRegs;
250	}
251	}
252
253	return *VRegs;
254	}
255
256	int IRTranslator::getOrCreateFrameIndex(const AllocaInst &AI) {
257	auto MapEntry = FrameIndices.find(Val: &AI);
258	if (MapEntry != FrameIndices.end())
259	return MapEntry->second;
260
261	uint64_t ElementSize = DL->getTypeAllocSize(Ty: AI.getAllocatedType());
262	uint64_t Size =
263	ElementSize * cast<ConstantInt>(Val: AI.getArraySize())->getZExtValue();
264
265	// Always allocate at least one byte.
266	Size = std::max<uint64_t>(a: Size, b: 1u);
267
268	int &FI = FrameIndices[&AI];
269	FI = MF->getFrameInfo().CreateStackObject(Size, Alignment: AI.getAlign(), isSpillSlot: false, Alloca: &AI);
270	return FI;
271	}
272
273	Align IRTranslator::getMemOpAlign(const Instruction &I) {
274	if (const StoreInst *SI = dyn_cast<StoreInst>(Val: &I))
275	return SI->getAlign();
276	if (const LoadInst *LI = dyn_cast<LoadInst>(Val: &I))
277	return LI->getAlign();
278	if (const AtomicCmpXchgInst *AI = dyn_cast<AtomicCmpXchgInst>(Val: &I))
279	return AI->getAlign();
280	if (const AtomicRMWInst *AI = dyn_cast<AtomicRMWInst>(Val: &I))
281	return AI->getAlign();
282
283	OptimizationRemarkMissed R("gisel-irtranslator", "", &I);
284	R << "unable to translate memop: " << ore::NV("Opcode", &I);
285	reportTranslationError(MF&: *MF, TPC: *TPC, ORE&: *ORE, R);
286	return Align(1);
287	}
288
289	MachineBasicBlock &IRTranslator::getMBB(const BasicBlock &BB) {
290	MachineBasicBlock *&MBB = BBToMBB[&BB];
291	assert(MBB && "BasicBlock was not encountered before");
292	return *MBB;
293	}
294
295	void IRTranslator::addMachineCFGPred(CFGEdge Edge, MachineBasicBlock *NewPred) {
296	assert(NewPred && "new predecessor must be a real MachineBasicBlock");
297	MachinePreds[Edge].push_back(Elt: NewPred);
298	}
299
300	bool IRTranslator::translateBinaryOp(unsigned Opcode, const User &U,
301	MachineIRBuilder &MIRBuilder) {
302	// Get or create a virtual register for each value.
303	// Unless the value is a Constant => loadimm cst?
304	// or inline constant each time?
305	// Creation of a virtual register needs to have a size.
306	Register Op0 = getOrCreateVReg(Val: *U.getOperand(i: 0));
307	Register Op1 = getOrCreateVReg(Val: *U.getOperand(i: 1));
308	Register Res = getOrCreateVReg(Val: U);
309	uint32_t Flags = 0;
310	if (isa<Instruction>(Val: U)) {
311	const Instruction &I = cast<Instruction>(Val: U);
312	Flags = MachineInstr::copyFlagsFromInstruction(I);
313	}
314
315	MIRBuilder.buildInstr(Opc: Opcode, DstOps: {Res}, SrcOps: {Op0, Op1}, Flags);
316	return true;
317	}
318
319	bool IRTranslator::translateUnaryOp(unsigned Opcode, const User &U,
320	MachineIRBuilder &MIRBuilder) {
321	Register Op0 = getOrCreateVReg(Val: *U.getOperand(i: 0));
322	Register Res = getOrCreateVReg(Val: U);
323	uint32_t Flags = 0;
324	if (isa<Instruction>(Val: U)) {
325	const Instruction &I = cast<Instruction>(Val: U);
326	Flags = MachineInstr::copyFlagsFromInstruction(I);
327	}
328	MIRBuilder.buildInstr(Opc: Opcode, DstOps: {Res}, SrcOps: {Op0}, Flags);
329	return true;
330	}
331
332	bool IRTranslator::translateFNeg(const User &U, MachineIRBuilder &MIRBuilder) {
333	return translateUnaryOp(Opcode: TargetOpcode::G_FNEG, U, MIRBuilder);
334	}
335
336	bool IRTranslator::translateCompare(const User &U,
337	MachineIRBuilder &MIRBuilder) {
338	auto *CI = dyn_cast<CmpInst>(Val: &U);
339	Register Op0 = getOrCreateVReg(Val: *U.getOperand(i: 0));
340	Register Op1 = getOrCreateVReg(Val: *U.getOperand(i: 1));
341	Register Res = getOrCreateVReg(Val: U);
342	CmpInst::Predicate Pred =
343	CI ? CI->getPredicate() : static_cast<CmpInst::Predicate>(
344	cast<ConstantExpr>(Val: U).getPredicate());
345	if (CmpInst::isIntPredicate(P: Pred))
346	MIRBuilder.buildICmp(Pred, Res, Op0, Op1);
347	else if (Pred == CmpInst::FCMP_FALSE)
348	MIRBuilder.buildCopy(
349	Res, Op: getOrCreateVReg(Val: *Constant::getNullValue(Ty: U.getType())));
350	else if (Pred == CmpInst::FCMP_TRUE)
351	MIRBuilder.buildCopy(
352	Res, Op: getOrCreateVReg(Val: *Constant::getAllOnesValue(Ty: U.getType())));
353	else {
354	uint32_t Flags = 0;
355	if (CI)
356	Flags = MachineInstr::copyFlagsFromInstruction(I: *CI);
357	MIRBuilder.buildFCmp(Pred, Res, Op0, Op1, Flags);
358	}
359
360	return true;
361	}
362
363	bool IRTranslator::translateRet(const User &U, MachineIRBuilder &MIRBuilder) {
364	const ReturnInst &RI = cast<ReturnInst>(Val: U);
365	const Value *Ret = RI.getReturnValue();
366	if (Ret && DL->getTypeStoreSize(Ty: Ret->getType()).isZero())
367	Ret = nullptr;
368
369	ArrayRef<Register> VRegs;
370	if (Ret)
371	VRegs = getOrCreateVRegs(Val: *Ret);
372
373	Register SwiftErrorVReg = 0;
374	if (CLI->supportSwiftError() && SwiftError.getFunctionArg()) {
375	SwiftErrorVReg = SwiftError.getOrCreateVRegUseAt(
376	&RI, &MIRBuilder.getMBB(), SwiftError.getFunctionArg());
377	}
378
379	// The target may mess up with the insertion point, but
380	// this is not important as a return is the last instruction
381	// of the block anyway.
382	return CLI->lowerReturn(MIRBuilder, Val: Ret, VRegs, FLI&: FuncInfo, SwiftErrorVReg);
383	}
384
385	void IRTranslator::emitBranchForMergedCondition(
386	const Value *Cond, MachineBasicBlock *TBB, MachineBasicBlock *FBB,
387	MachineBasicBlock *CurBB, MachineBasicBlock *SwitchBB,
388	BranchProbability TProb, BranchProbability FProb, bool InvertCond) {
389	// If the leaf of the tree is a comparison, merge the condition into
390	// the caseblock.
391	if (const CmpInst *BOp = dyn_cast<CmpInst>(Val: Cond)) {
392	CmpInst::Predicate Condition;
393	if (const ICmpInst *IC = dyn_cast<ICmpInst>(Val: Cond)) {
394	Condition = InvertCond ? IC->getInversePredicate() : IC->getPredicate();
395	} else {
396	const FCmpInst *FC = cast<FCmpInst>(Val: Cond);
397	Condition = InvertCond ? FC->getInversePredicate() : FC->getPredicate();
398	}
399
400	SwitchCG::CaseBlock CB(Condition, false, BOp->getOperand(i_nocapture: 0),
401	BOp->getOperand(i_nocapture: 1), nullptr, TBB, FBB, CurBB,
402	CurBuilder->getDebugLoc(), TProb, FProb);
403	SL->SwitchCases.push_back(x: CB);
404	return;
405	}
406
407	// Create a CaseBlock record representing this branch.
408	CmpInst::Predicate Pred = InvertCond ? CmpInst::ICMP_NE : CmpInst::ICMP_EQ;
409	SwitchCG::CaseBlock CB(
410	Pred, false, Cond, ConstantInt::getTrue(Context&: MF->getFunction().getContext()),
411	nullptr, TBB, FBB, CurBB, CurBuilder->getDebugLoc(), TProb, FProb);
412	SL->SwitchCases.push_back(x: CB);
413	}
414
415	static bool isValInBlock(const Value *V, const BasicBlock *BB) {
416	if (const Instruction *I = dyn_cast<Instruction>(Val: V))
417	return I->getParent() == BB;
418	return true;
419	}
420
421	void IRTranslator::findMergedConditions(
422	const Value *Cond, MachineBasicBlock *TBB, MachineBasicBlock *FBB,
423	MachineBasicBlock *CurBB, MachineBasicBlock *SwitchBB,
424	Instruction::BinaryOps Opc, BranchProbability TProb,
425	BranchProbability FProb, bool InvertCond) {
426	using namespace PatternMatch;
427	assert((Opc == Instruction::And || Opc == Instruction::Or) &&
428	"Expected Opc to be AND/OR");
429	// Skip over not part of the tree and remember to invert op and operands at
430	// next level.
431	Value *NotCond;
432	if (match(V: Cond, P: m_OneUse(SubPattern: m_Not(V: m_Value(V&: NotCond)))) &&
433	isValInBlock(V: NotCond, BB: CurBB->getBasicBlock())) {
434	findMergedConditions(Cond: NotCond, TBB, FBB, CurBB, SwitchBB, Opc, TProb, FProb,
435	InvertCond: !InvertCond);
436	return;
437	}
438
439	const Instruction *BOp = dyn_cast<Instruction>(Val: Cond);
440	const Value *BOpOp0, *BOpOp1;
441	// Compute the effective opcode for Cond, taking into account whether it needs
442	// to be inverted, e.g.
443	// and (not (or A, B)), C
444	// gets lowered as
445	// and (and (not A, not B), C)
446	Instruction::BinaryOps BOpc = (Instruction::BinaryOps)0;
447	if (BOp) {
448	BOpc = match(V: BOp, P: m_LogicalAnd(L: m_Value(V&: BOpOp0), R: m_Value(V&: BOpOp1)))
449	? Instruction::And
450	: (match(V: BOp, P: m_LogicalOr(L: m_Value(V&: BOpOp0), R: m_Value(V&: BOpOp1)))
451	? Instruction::Or
452	: (Instruction::BinaryOps)0);
453	if (InvertCond) {
454	if (BOpc == Instruction::And)
455	BOpc = Instruction::Or;
456	else if (BOpc == Instruction::Or)
457	BOpc = Instruction::And;
458	}
459	}
460
461	// If this node is not part of the or/and tree, emit it as a branch.
462	// Note that all nodes in the tree should have same opcode.
463	bool BOpIsInOrAndTree = BOpc && BOpc == Opc && BOp->hasOneUse();
464	if (!BOpIsInOrAndTree || BOp->getParent() != CurBB->getBasicBlock() ||
465	!isValInBlock(V: BOpOp0, BB: CurBB->getBasicBlock()) ||
466	!isValInBlock(V: BOpOp1, BB: CurBB->getBasicBlock())) {
467	emitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB, TProb, FProb,
468	InvertCond);
469	return;
470	}
471
472	// Create TmpBB after CurBB.
473	MachineFunction::iterator BBI(CurBB);
474	MachineBasicBlock *TmpBB =
475	MF->CreateMachineBasicBlock(BB: CurBB->getBasicBlock());
476	CurBB->getParent()->insert(MBBI: ++BBI, MBB: TmpBB);
477
478	if (Opc == Instruction::Or) {
479	// Codegen X | Y as:
480	// BB1:
481	// jmp_if_X TBB
482	// jmp TmpBB
483	// TmpBB:
484	// jmp_if_Y TBB
485	// jmp FBB
486	//
487
488	// We have flexibility in setting Prob for BB1 and Prob for TmpBB.
489	// The requirement is that
490	// TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB)
491	// = TrueProb for original BB.
492	// Assuming the original probabilities are A and B, one choice is to set
493	// BB1's probabilities to A/2 and A/2+B, and set TmpBB's probabilities to
494	// A/(1+B) and 2B/(1+B). This choice assumes that
495	// TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB.
496	// Another choice is to assume TrueProb for BB1 equals to TrueProb for
497	// TmpBB, but the math is more complicated.
498
499	auto NewTrueProb = TProb / 2;
500	auto NewFalseProb = TProb / 2 + FProb;
501	// Emit the LHS condition.
502	findMergedConditions(Cond: BOpOp0, TBB, FBB: TmpBB, CurBB, SwitchBB, Opc, TProb: NewTrueProb,
503	FProb: NewFalseProb, InvertCond);
504
505	// Normalize A/2 and B to get A/(1+B) and 2B/(1+B).
506	SmallVector<BranchProbability, 2> Probs{TProb / 2, FProb};
507	BranchProbability::normalizeProbabilities(Begin: Probs.begin(), End: Probs.end());
508	// Emit the RHS condition into TmpBB.
509	findMergedConditions(Cond: BOpOp1, TBB, FBB, CurBB: TmpBB, SwitchBB, Opc, TProb: Probs[0],
510	FProb: Probs[1], InvertCond);
511	} else {
512	assert(Opc == Instruction::And && "Unknown merge op!");
513	// Codegen X & Y as:
514	// BB1:
515	// jmp_if_X TmpBB
516	// jmp FBB
517	// TmpBB:
518	// jmp_if_Y TBB
519	// jmp FBB
520	//
521	// This requires creation of TmpBB after CurBB.
522
523	// We have flexibility in setting Prob for BB1 and Prob for TmpBB.
524	// The requirement is that
525	// FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB)
526	// = FalseProb for original BB.
527	// Assuming the original probabilities are A and B, one choice is to set
528	// BB1's probabilities to A+B/2 and B/2, and set TmpBB's probabilities to
529	// 2A/(1+A) and B/(1+A). This choice assumes that FalseProb for BB1 ==
530	// TrueProb for BB1 * FalseProb for TmpBB.
531
532	auto NewTrueProb = TProb + FProb / 2;
533	auto NewFalseProb = FProb / 2;
534	// Emit the LHS condition.
535	findMergedConditions(Cond: BOpOp0, TBB: TmpBB, FBB, CurBB, SwitchBB, Opc, TProb: NewTrueProb,
536	FProb: NewFalseProb, InvertCond);
537
538	// Normalize A and B/2 to get 2A/(1+A) and B/(1+A).
539	SmallVector<BranchProbability, 2> Probs{TProb, FProb / 2};
540	BranchProbability::normalizeProbabilities(Begin: Probs.begin(), End: Probs.end());
541	// Emit the RHS condition into TmpBB.
542	findMergedConditions(Cond: BOpOp1, TBB, FBB, CurBB: TmpBB, SwitchBB, Opc, TProb: Probs[0],
543	FProb: Probs[1], InvertCond);
544	}
545	}
546
547	bool IRTranslator::shouldEmitAsBranches(
548	const std::vector<SwitchCG::CaseBlock> &Cases) {
549	// For multiple cases, it's better to emit as branches.
550	if (Cases.size() != 2)
551	return true;
552
553	// If this is two comparisons of the same values or'd or and'd together, they
554	// will get folded into a single comparison, so don't emit two blocks.
555	if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&
556	Cases[0].CmpRHS == Cases[1].CmpRHS) ||
557	(Cases[0].CmpRHS == Cases[1].CmpLHS &&
558	Cases[0].CmpLHS == Cases[1].CmpRHS)) {
559	return false;
560	}
561
562	// Handle: (X != null) | (Y != null) --> (X|Y) != 0
563	// Handle: (X == null) & (Y == null) --> (X|Y) == 0
564	if (Cases[0].CmpRHS == Cases[1].CmpRHS &&
565	Cases[0].PredInfo.Pred == Cases[1].PredInfo.Pred &&
566	isa<Constant>(Val: Cases[0].CmpRHS) &&
567	cast<Constant>(Val: Cases[0].CmpRHS)->isNullValue()) {
568	if (Cases[0].PredInfo.Pred == CmpInst::ICMP_EQ &&
569	Cases[0].TrueBB == Cases[1].ThisBB)
570	return false;
571	if (Cases[0].PredInfo.Pred == CmpInst::ICMP_NE &&
572	Cases[0].FalseBB == Cases[1].ThisBB)
573	return false;
574	}
575
576	return true;
577	}
578
579	bool IRTranslator::translateBr(const User &U, MachineIRBuilder &MIRBuilder) {
580	const BranchInst &BrInst = cast<BranchInst>(Val: U);
581	auto &CurMBB = MIRBuilder.getMBB();
582	auto *Succ0MBB = &getMBB(BB: *BrInst.getSuccessor(i: 0));
583
584	if (BrInst.isUnconditional()) {
585	// If the unconditional target is the layout successor, fallthrough.
586	if (OptLevel == CodeGenOptLevel::None ||
587	!CurMBB.isLayoutSuccessor(MBB: Succ0MBB))
588	MIRBuilder.buildBr(Dest&: *Succ0MBB);
589
590	// Link successors.
591	for (const BasicBlock *Succ : successors(I: &BrInst))
592	CurMBB.addSuccessor(Succ: &getMBB(BB: *Succ));
593	return true;
594	}
595
596	// If this condition is one of the special cases we handle, do special stuff
597	// now.
598	const Value *CondVal = BrInst.getCondition();
599	MachineBasicBlock *Succ1MBB = &getMBB(BB: *BrInst.getSuccessor(i: 1));
600
601	// If this is a series of conditions that are or'd or and'd together, emit
602	// this as a sequence of branches instead of setcc's with and/or operations.
603	// As long as jumps are not expensive (exceptions for multi-use logic ops,
604	// unpredictable branches, and vector extracts because those jumps are likely
605	// expensive for any target), this should improve performance.
606	// For example, instead of something like:
607	// cmp A, B
608	// C = seteq
609	// cmp D, E
610	// F = setle
611	// or C, F
612	// jnz foo
613	// Emit:
614	// cmp A, B
615	// je foo
616	// cmp D, E
617	// jle foo
618	using namespace PatternMatch;
619	const Instruction *CondI = dyn_cast<Instruction>(Val: CondVal);
620	if (!TLI->isJumpExpensive() && CondI && CondI->hasOneUse() &&
621	!BrInst.hasMetadata(KindID: LLVMContext::MD_unpredictable)) {
622	Instruction::BinaryOps Opcode = (Instruction::BinaryOps)0;
623	Value *Vec;
624	const Value *BOp0, *BOp1;
625	if (match(V: CondI, P: m_LogicalAnd(L: m_Value(V&: BOp0), R: m_Value(V&: BOp1))))
626	Opcode = Instruction::And;
627	else if (match(V: CondI, P: m_LogicalOr(L: m_Value(V&: BOp0), R: m_Value(V&: BOp1))))
628	Opcode = Instruction::Or;
629
630	if (Opcode && !(match(V: BOp0, P: m_ExtractElt(Val: m_Value(V&: Vec), Idx: m_Value())) &&
631	match(V: BOp1, P: m_ExtractElt(Val: m_Specific(V: Vec), Idx: m_Value())))) {
632	findMergedConditions(Cond: CondI, TBB: Succ0MBB, FBB: Succ1MBB, CurBB: &CurMBB, SwitchBB: &CurMBB, Opc: Opcode,
633	TProb: getEdgeProbability(Src: &CurMBB, Dst: Succ0MBB),
634	FProb: getEdgeProbability(Src: &CurMBB, Dst: Succ1MBB),
635	/*InvertCond=*/false);
636	assert(SL->SwitchCases[0].ThisBB == &CurMBB && "Unexpected lowering!");
637
638	// Allow some cases to be rejected.
639	if (shouldEmitAsBranches(Cases: SL->SwitchCases)) {
640	// Emit the branch for this block.
641	emitSwitchCase(CB&: SL->SwitchCases[0], SwitchBB: &CurMBB, MIB&: *CurBuilder);
642	SL->SwitchCases.erase(position: SL->SwitchCases.begin());
643	return true;
644	}
645
646	// Okay, we decided not to do this, remove any inserted MBB's and clear
647	// SwitchCases.
648	for (unsigned I = 1, E = SL->SwitchCases.size(); I != E; ++I)
649	MF->erase(MBBI: SL->SwitchCases[I].ThisBB);
650
651	SL->SwitchCases.clear();
652	}
653	}
654
655	// Create a CaseBlock record representing this branch.
656	SwitchCG::CaseBlock CB(CmpInst::ICMP_EQ, false, CondVal,
657	ConstantInt::getTrue(Context&: MF->getFunction().getContext()),
658	nullptr, Succ0MBB, Succ1MBB, &CurMBB,
659	CurBuilder->getDebugLoc());
660
661	// Use emitSwitchCase to actually insert the fast branch sequence for this
662	// cond branch.
663	emitSwitchCase(CB, SwitchBB: &CurMBB, MIB&: *CurBuilder);
664	return true;
665	}
666
667	void IRTranslator::addSuccessorWithProb(MachineBasicBlock *Src,
668	MachineBasicBlock *Dst,
669	BranchProbability Prob) {
670	if (!FuncInfo.BPI) {
671	Src->addSuccessorWithoutProb(Succ: Dst);
672	return;
673	}
674	if (Prob.isUnknown())
675	Prob = getEdgeProbability(Src, Dst);
676	Src->addSuccessor(Succ: Dst, Prob);
677	}
678
679	BranchProbability
680	IRTranslator::getEdgeProbability(const MachineBasicBlock *Src,
681	const MachineBasicBlock *Dst) const {
682	const BasicBlock *SrcBB = Src->getBasicBlock();
683	const BasicBlock *DstBB = Dst->getBasicBlock();
684	if (!FuncInfo.BPI) {
685	// If BPI is not available, set the default probability as 1 / N, where N is
686	// the number of successors.
687	auto SuccSize = std::max<uint32_t>(a: succ_size(BB: SrcBB), b: 1);
688	return BranchProbability(1, SuccSize);
689	}
690	return FuncInfo.BPI->getEdgeProbability(Src: SrcBB, Dst: DstBB);
691	}
692
693	bool IRTranslator::translateSwitch(const User &U, MachineIRBuilder &MIB) {
694	using namespace SwitchCG;
695	// Extract cases from the switch.
696	const SwitchInst &SI = cast<SwitchInst>(Val: U);
697	BranchProbabilityInfo *BPI = FuncInfo.BPI;
698	CaseClusterVector Clusters;
699	Clusters.reserve(n: SI.getNumCases());
700	for (const auto &I : SI.cases()) {
701	MachineBasicBlock *Succ = &getMBB(BB: *I.getCaseSuccessor());
702	assert(Succ && "Could not find successor mbb in mapping");
703	const ConstantInt *CaseVal = I.getCaseValue();
704	BranchProbability Prob =
705	BPI ? BPI->getEdgeProbability(Src: SI.getParent(), IndexInSuccessors: I.getSuccessorIndex())
706	: BranchProbability(1, SI.getNumCases() + 1);
707	Clusters.push_back(x: CaseCluster::range(Low: CaseVal, High: CaseVal, MBB: Succ, Prob));
708	}
709
710	MachineBasicBlock *DefaultMBB = &getMBB(BB: *SI.getDefaultDest());
711
712	// Cluster adjacent cases with the same destination. We do this at all
713	// optimization levels because it's cheap to do and will make codegen faster
714	// if there are many clusters.
715	sortAndRangeify(Clusters);
716
717	MachineBasicBlock *SwitchMBB = &getMBB(BB: *SI.getParent());
718
719	// If there is only the default destination, jump there directly.
720	if (Clusters.empty()) {
721	SwitchMBB->addSuccessor(Succ: DefaultMBB);
722	if (DefaultMBB != SwitchMBB->getNextNode())
723	MIB.buildBr(Dest&: *DefaultMBB);
724	return true;
725	}
726
727	SL->findJumpTables(Clusters, SI: &SI, SL: std::nullopt, DefaultMBB, PSI: nullptr, BFI: nullptr);
728	SL->findBitTestClusters(Clusters, SI: &SI);
729
730	LLVM_DEBUG({
731	dbgs() << "Case clusters: ";
732	for (const CaseCluster &C : Clusters) {
733	if (C.Kind == CC_JumpTable)
734	dbgs() << "JT:";
735	if (C.Kind == CC_BitTests)
736	dbgs() << "BT:";
737
738	C.Low->getValue().print(dbgs(), true);
739	if (C.Low != C.High) {
740	dbgs() << '-';
741	C.High->getValue().print(dbgs(), true);
742	}
743	dbgs() << ' ';
744	}
745	dbgs() << '\n';
746	});
747
748	assert(!Clusters.empty());
749	SwitchWorkList WorkList;
750	CaseClusterIt First = Clusters.begin();
751	CaseClusterIt Last = Clusters.end() - 1;
752	auto DefaultProb = getEdgeProbability(Src: SwitchMBB, Dst: DefaultMBB);
753	WorkList.push_back(Elt: {.MBB: SwitchMBB, .FirstCluster: First, .LastCluster: Last, .GE: nullptr, .LT: nullptr, .DefaultProb: DefaultProb});
754
755	while (!WorkList.empty()) {
756	SwitchWorkListItem W = WorkList.pop_back_val();
757
758	unsigned NumClusters = W.LastCluster - W.FirstCluster + 1;
759	// For optimized builds, lower large range as a balanced binary tree.
760	if (NumClusters > 3 &&
761	MF->getTarget().getOptLevel() != CodeGenOptLevel::None &&
762	!DefaultMBB->getParent()->getFunction().hasMinSize()) {
763	splitWorkItem(WorkList, W, Cond: SI.getCondition(), SwitchMBB, MIB);
764	continue;
765	}
766
767	if (!lowerSwitchWorkItem(W, Cond: SI.getCondition(), SwitchMBB, DefaultMBB, MIB))
768	return false;
769	}
770	return true;
771	}
772
773	void IRTranslator::splitWorkItem(SwitchCG::SwitchWorkList &WorkList,
774	const SwitchCG::SwitchWorkListItem &W,
775	Value *Cond, MachineBasicBlock *SwitchMBB,
776	MachineIRBuilder &MIB) {
777	using namespace SwitchCG;
778	assert(W.FirstCluster->Low->getValue().slt(W.LastCluster->Low->getValue()) &&
779	"Clusters not sorted?");
780	assert(W.LastCluster - W.FirstCluster + 1 >= 2 && "Too small to split!");
781
782	auto [LastLeft, FirstRight, LeftProb, RightProb] =
783	SL->computeSplitWorkItemInfo(W);
784
785	// Use the first element on the right as pivot since we will make less-than
786	// comparisons against it.
787	CaseClusterIt PivotCluster = FirstRight;
788	assert(PivotCluster > W.FirstCluster);
789	assert(PivotCluster <= W.LastCluster);
790
791	CaseClusterIt FirstLeft = W.FirstCluster;
792	CaseClusterIt LastRight = W.LastCluster;
793
794	const ConstantInt *Pivot = PivotCluster->Low;
795
796	// New blocks will be inserted immediately after the current one.
797	MachineFunction::iterator BBI(W.MBB);
798	++BBI;
799
800	// We will branch to the LHS if Value < Pivot. If LHS is a single cluster,
801	// we can branch to its destination directly if it's squeezed exactly in
802	// between the known lower bound and Pivot - 1.
803	MachineBasicBlock *LeftMBB;
804	if (FirstLeft == LastLeft && FirstLeft->Kind == CC_Range &&
805	FirstLeft->Low == W.GE &&
806	(FirstLeft->High->getValue() + 1LL) == Pivot->getValue()) {
807	LeftMBB = FirstLeft->MBB;
808	} else {
809	LeftMBB = FuncInfo.MF->CreateMachineBasicBlock(BB: W.MBB->getBasicBlock());
810	FuncInfo.MF->insert(MBBI: BBI, MBB: LeftMBB);
811	WorkList.push_back(
812	Elt: {.MBB: LeftMBB, .FirstCluster: FirstLeft, .LastCluster: LastLeft, .GE: W.GE, .LT: Pivot, .DefaultProb: W.DefaultProb / 2});
813	}
814
815	// Similarly, we will branch to the RHS if Value >= Pivot. If RHS is a
816	// single cluster, RHS.Low == Pivot, and we can branch to its destination
817	// directly if RHS.High equals the current upper bound.
818	MachineBasicBlock *RightMBB;
819	if (FirstRight == LastRight && FirstRight->Kind == CC_Range && W.LT &&
820	(FirstRight->High->getValue() + 1ULL) == W.LT->getValue()) {
821	RightMBB = FirstRight->MBB;
822	} else {
823	RightMBB = FuncInfo.MF->CreateMachineBasicBlock(BB: W.MBB->getBasicBlock());
824	FuncInfo.MF->insert(MBBI: BBI, MBB: RightMBB);
825	WorkList.push_back(
826	Elt: {.MBB: RightMBB, .FirstCluster: FirstRight, .LastCluster: LastRight, .GE: Pivot, .LT: W.LT, .DefaultProb: W.DefaultProb / 2});
827	}
828
829	// Create the CaseBlock record that will be used to lower the branch.
830	CaseBlock CB(ICmpInst::Predicate::ICMP_SLT, false, Cond, Pivot, nullptr,
831	LeftMBB, RightMBB, W.MBB, MIB.getDebugLoc(), LeftProb,
832	RightProb);
833
834	if (W.MBB == SwitchMBB)
835	emitSwitchCase(CB, SwitchBB: SwitchMBB, MIB);
836	else
837	SL->SwitchCases.push_back(x: CB);
838	}
839
840	void IRTranslator::emitJumpTable(SwitchCG::JumpTable &JT,
841	MachineBasicBlock *MBB) {
842	// Emit the code for the jump table
843	assert(JT.Reg != -1U && "Should lower JT Header first!");
844	MachineIRBuilder MIB(*MBB->getParent());
845	MIB.setMBB(*MBB);
846	MIB.setDebugLoc(CurBuilder->getDebugLoc());
847
848	Type *PtrIRTy = PointerType::getUnqual(C&: MF->getFunction().getContext());
849	const LLT PtrTy = getLLTForType(Ty&: *PtrIRTy, DL: *DL);
850
851	auto Table = MIB.buildJumpTable(PtrTy, JTI: JT.JTI);
852	MIB.buildBrJT(TablePtr: Table.getReg(Idx: 0), JTI: JT.JTI, IndexReg: JT.Reg);
853	}
854
855	bool IRTranslator::emitJumpTableHeader(SwitchCG::JumpTable &JT,
856	SwitchCG::JumpTableHeader &JTH,
857	MachineBasicBlock *HeaderBB) {
858	MachineIRBuilder MIB(*HeaderBB->getParent());
859	MIB.setMBB(*HeaderBB);
860	MIB.setDebugLoc(CurBuilder->getDebugLoc());
861
862	const Value &SValue = *JTH.SValue;
863	// Subtract the lowest switch case value from the value being switched on.
864	const LLT SwitchTy = getLLTForType(Ty&: *SValue.getType(), DL: *DL);
865	Register SwitchOpReg = getOrCreateVReg(Val: SValue);
866	auto FirstCst = MIB.buildConstant(Res: SwitchTy, Val: JTH.First);
867	auto Sub = MIB.buildSub(Dst: {SwitchTy}, Src0: SwitchOpReg, Src1: FirstCst);
868
869	// This value may be smaller or larger than the target's pointer type, and
870	// therefore require extension or truncating.
871	auto *PtrIRTy = PointerType::getUnqual(C&: SValue.getContext());
872	const LLT PtrScalarTy = LLT::scalar(SizeInBits: DL->getTypeSizeInBits(Ty: PtrIRTy));
873	Sub = MIB.buildZExtOrTrunc(Res: PtrScalarTy, Op: Sub);
874
875	JT.Reg = Sub.getReg(Idx: 0);
876
877	if (JTH.FallthroughUnreachable) {
878	if (JT.MBB != HeaderBB->getNextNode())
879	MIB.buildBr(Dest&: *JT.MBB);
880	return true;
881	}
882
883	// Emit the range check for the jump table, and branch to the default block
884	// for the switch statement if the value being switched on exceeds the
885	// largest case in the switch.
886	auto Cst = getOrCreateVReg(
887	Val: *ConstantInt::get(Ty: SValue.getType(), V: JTH.Last - JTH.First));
888	Cst = MIB.buildZExtOrTrunc(Res: PtrScalarTy, Op: Cst).getReg(Idx: 0);
889	auto Cmp = MIB.buildICmp(Pred: CmpInst::ICMP_UGT, Res: LLT::scalar(SizeInBits: 1), Op0: Sub, Op1: Cst);
890
891	auto BrCond = MIB.buildBrCond(Tst: Cmp.getReg(Idx: 0), Dest&: *JT.Default);
892
893	// Avoid emitting unnecessary branches to the next block.
894	if (JT.MBB != HeaderBB->getNextNode())
895	BrCond = MIB.buildBr(Dest&: *JT.MBB);
896	return true;
897	}
898
899	void IRTranslator::emitSwitchCase(SwitchCG::CaseBlock &CB,
900	MachineBasicBlock *SwitchBB,
901	MachineIRBuilder &MIB) {
902	Register CondLHS = getOrCreateVReg(Val: *CB.CmpLHS);
903	Register Cond;
904	DebugLoc OldDbgLoc = MIB.getDebugLoc();
905	MIB.setDebugLoc(CB.DbgLoc);
906	MIB.setMBB(*CB.ThisBB);
907
908	if (CB.PredInfo.NoCmp) {
909	// Branch or fall through to TrueBB.
910	addSuccessorWithProb(Src: CB.ThisBB, Dst: CB.TrueBB, Prob: CB.TrueProb);
911	addMachineCFGPred(Edge: {SwitchBB->getBasicBlock(), CB.TrueBB->getBasicBlock()},
912	NewPred: CB.ThisBB);
913	CB.ThisBB->normalizeSuccProbs();
914	if (CB.TrueBB != CB.ThisBB->getNextNode())
915	MIB.buildBr(Dest&: *CB.TrueBB);
916	MIB.setDebugLoc(OldDbgLoc);
917	return;
918	}
919
920	const LLT i1Ty = LLT::scalar(SizeInBits: 1);
921	// Build the compare.
922	if (!CB.CmpMHS) {
923	const auto *CI = dyn_cast<ConstantInt>(Val: CB.CmpRHS);
924	// For conditional branch lowering, we might try to do something silly like
925	// emit an G_ICMP to compare an existing G_ICMP i1 result with true. If so,
926	// just re-use the existing condition vreg.
927	if (MRI->getType(Reg: CondLHS).getSizeInBits() == 1 && CI && CI->isOne() &&
928	CB.PredInfo.Pred == CmpInst::ICMP_EQ) {
929	Cond = CondLHS;
930	} else {
931	Register CondRHS = getOrCreateVReg(Val: *CB.CmpRHS);
932	if (CmpInst::isFPPredicate(P: CB.PredInfo.Pred))
933	Cond =
934	MIB.buildFCmp(Pred: CB.PredInfo.Pred, Res: i1Ty, Op0: CondLHS, Op1: CondRHS).getReg(Idx: 0);
935	else
936	Cond =
937	MIB.buildICmp(Pred: CB.PredInfo.Pred, Res: i1Ty, Op0: CondLHS, Op1: CondRHS).getReg(Idx: 0);
938	}
939	} else {
940	assert(CB.PredInfo.Pred == CmpInst::ICMP_SLE &&
941	"Can only handle SLE ranges");
942
943	const APInt& Low = cast<ConstantInt>(Val: CB.CmpLHS)->getValue();
944	const APInt& High = cast<ConstantInt>(Val: CB.CmpRHS)->getValue();
945
946	Register CmpOpReg = getOrCreateVReg(Val: *CB.CmpMHS);
947	if (cast<ConstantInt>(Val: CB.CmpLHS)->isMinValue(IsSigned: true)) {
948	Register CondRHS = getOrCreateVReg(Val: *CB.CmpRHS);
949	Cond =
950	MIB.buildICmp(Pred: CmpInst::ICMP_SLE, Res: i1Ty, Op0: CmpOpReg, Op1: CondRHS).getReg(Idx: 0);
951	} else {
952	const LLT CmpTy = MRI->getType(Reg: CmpOpReg);
953	auto Sub = MIB.buildSub(Dst: {CmpTy}, Src0: CmpOpReg, Src1: CondLHS);
954	auto Diff = MIB.buildConstant(Res: CmpTy, Val: High - Low);
955	Cond = MIB.buildICmp(Pred: CmpInst::ICMP_ULE, Res: i1Ty, Op0: Sub, Op1: Diff).getReg(Idx: 0);
956	}
957	}
958
959	// Update successor info
960	addSuccessorWithProb(Src: CB.ThisBB, Dst: CB.TrueBB, Prob: CB.TrueProb);
961
962	addMachineCFGPred(Edge: {SwitchBB->getBasicBlock(), CB.TrueBB->getBasicBlock()},
963	NewPred: CB.ThisBB);
964
965	// TrueBB and FalseBB are always different unless the incoming IR is
966	// degenerate. This only happens when running llc on weird IR.
967	if (CB.TrueBB != CB.FalseBB)
968	addSuccessorWithProb(Src: CB.ThisBB, Dst: CB.FalseBB, Prob: CB.FalseProb);
969	CB.ThisBB->normalizeSuccProbs();
970
971	addMachineCFGPred(Edge: {SwitchBB->getBasicBlock(), CB.FalseBB->getBasicBlock()},
972	NewPred: CB.ThisBB);
973
974	MIB.buildBrCond(Tst: Cond, Dest&: *CB.TrueBB);
975	MIB.buildBr(Dest&: *CB.FalseBB);
976	MIB.setDebugLoc(OldDbgLoc);
977	}
978
979	bool IRTranslator::lowerJumpTableWorkItem(SwitchCG::SwitchWorkListItem W,
980	MachineBasicBlock *SwitchMBB,
981	MachineBasicBlock *CurMBB,
982	MachineBasicBlock *DefaultMBB,
983	MachineIRBuilder &MIB,
984	MachineFunction::iterator BBI,
985	BranchProbability UnhandledProbs,
986	SwitchCG::CaseClusterIt I,
987	MachineBasicBlock *Fallthrough,
988	bool FallthroughUnreachable) {
989	using namespace SwitchCG;
990	MachineFunction *CurMF = SwitchMBB->getParent();
991	// FIXME: Optimize away range check based on pivot comparisons.
992	JumpTableHeader *JTH = &SL->JTCases[I->JTCasesIndex].first;
993	SwitchCG::JumpTable *JT = &SL->JTCases[I->JTCasesIndex].second;
994	BranchProbability DefaultProb = W.DefaultProb;
995
996	// The jump block hasn't been inserted yet; insert it here.
997	MachineBasicBlock *JumpMBB = JT->MBB;
998	CurMF->insert(MBBI: BBI, MBB: JumpMBB);
999
1000	// Since the jump table block is separate from the switch block, we need
1001	// to keep track of it as a machine predecessor to the default block,
1002	// otherwise we lose the phi edges.
1003	addMachineCFGPred(Edge: {SwitchMBB->getBasicBlock(), DefaultMBB->getBasicBlock()},
1004	NewPred: CurMBB);
1005	addMachineCFGPred(Edge: {SwitchMBB->getBasicBlock(), DefaultMBB->getBasicBlock()},
1006	NewPred: JumpMBB);
1007
1008	auto JumpProb = I->Prob;
1009	auto FallthroughProb = UnhandledProbs;
1010
1011	// If the default statement is a target of the jump table, we evenly
1012	// distribute the default probability to successors of CurMBB. Also
1013	// update the probability on the edge from JumpMBB to Fallthrough.
1014	for (MachineBasicBlock::succ_iterator SI = JumpMBB->succ_begin(),
1015	SE = JumpMBB->succ_end();
1016	SI != SE; ++SI) {
1017	if (*SI == DefaultMBB) {
1018	JumpProb += DefaultProb / 2;
1019	FallthroughProb -= DefaultProb / 2;
1020	JumpMBB->setSuccProbability(I: SI, Prob: DefaultProb / 2);
1021	JumpMBB->normalizeSuccProbs();
1022	} else {
1023	// Also record edges from the jump table block to it's successors.
1024	addMachineCFGPred(Edge: {SwitchMBB->getBasicBlock(), (*SI)->getBasicBlock()},
1025	NewPred: JumpMBB);
1026	}
1027	}
1028
1029	if (FallthroughUnreachable)
1030	JTH->FallthroughUnreachable = true;
1031
1032	if (!JTH->FallthroughUnreachable)
1033	addSuccessorWithProb(Src: CurMBB, Dst: Fallthrough, Prob: FallthroughProb);
1034	addSuccessorWithProb(Src: CurMBB, Dst: JumpMBB, Prob: JumpProb);
1035	CurMBB->normalizeSuccProbs();
1036
1037	// The jump table header will be inserted in our current block, do the
1038	// range check, and fall through to our fallthrough block.
1039	JTH->HeaderBB = CurMBB;
1040	JT->Default = Fallthrough; // FIXME: Move Default to JumpTableHeader.
1041
1042	// If we're in the right place, emit the jump table header right now.
1043	if (CurMBB == SwitchMBB) {
1044	if (!emitJumpTableHeader(JT&: *JT, JTH&: *JTH, HeaderBB: CurMBB))
1045	return false;
1046	JTH->Emitted = true;
1047	}
1048	return true;
1049	}
1050	bool IRTranslator::lowerSwitchRangeWorkItem(SwitchCG::CaseClusterIt I,
1051	Value *Cond,
1052	MachineBasicBlock *Fallthrough,
1053	bool FallthroughUnreachable,
1054	BranchProbability UnhandledProbs,
1055	MachineBasicBlock *CurMBB,
1056	MachineIRBuilder &MIB,
1057	MachineBasicBlock *SwitchMBB) {
1058	using namespace SwitchCG;
1059	const Value *RHS, *LHS, *MHS;
1060	CmpInst::Predicate Pred;
1061	if (I->Low == I->High) {
1062	// Check Cond == I->Low.
1063	Pred = CmpInst::ICMP_EQ;
1064	LHS = Cond;
1065	RHS = I->Low;
1066	MHS = nullptr;
1067	} else {
1068	// Check I->Low <= Cond <= I->High.
1069	Pred = CmpInst::ICMP_SLE;
1070	LHS = I->Low;
1071	MHS = Cond;
1072	RHS = I->High;
1073	}
1074
1075	// If Fallthrough is unreachable, fold away the comparison.
1076	// The false probability is the sum of all unhandled cases.
1077	CaseBlock CB(Pred, FallthroughUnreachable, LHS, RHS, MHS, I->MBB, Fallthrough,
1078	CurMBB, MIB.getDebugLoc(), I->Prob, UnhandledProbs);
1079
1080	emitSwitchCase(CB, SwitchBB: SwitchMBB, MIB);
1081	return true;
1082	}
1083
1084	void IRTranslator::emitBitTestHeader(SwitchCG::BitTestBlock &B,
1085	MachineBasicBlock *SwitchBB) {
1086	MachineIRBuilder &MIB = *CurBuilder;
1087	MIB.setMBB(*SwitchBB);
1088
1089	// Subtract the minimum value.
1090	Register SwitchOpReg = getOrCreateVReg(Val: *B.SValue);
1091
1092	LLT SwitchOpTy = MRI->getType(Reg: SwitchOpReg);
1093	Register MinValReg = MIB.buildConstant(Res: SwitchOpTy, Val: B.First).getReg(Idx: 0);
1094	auto RangeSub = MIB.buildSub(Dst: SwitchOpTy, Src0: SwitchOpReg, Src1: MinValReg);
1095
1096	Type *PtrIRTy = PointerType::getUnqual(C&: MF->getFunction().getContext());
1097	const LLT PtrTy = getLLTForType(Ty&: *PtrIRTy, DL: *DL);
1098
1099	LLT MaskTy = SwitchOpTy;
1100	if (MaskTy.getSizeInBits() > PtrTy.getSizeInBits() ||
1101	!llvm::has_single_bit<uint32_t>(Value: MaskTy.getSizeInBits()))
1102	MaskTy = LLT::scalar(SizeInBits: PtrTy.getSizeInBits());
1103	else {
1104	// Ensure that the type will fit the mask value.
1105	for (unsigned I = 0, E = B.Cases.size(); I != E; ++I) {
1106	if (!isUIntN(N: SwitchOpTy.getSizeInBits(), x: B.Cases[I].Mask)) {
1107	// Switch table case range are encoded into series of masks.
1108	// Just use pointer type, it's guaranteed to fit.
1109	MaskTy = LLT::scalar(SizeInBits: PtrTy.getSizeInBits());
1110	break;
1111	}
1112	}
1113	}
1114	Register SubReg = RangeSub.getReg(Idx: 0);
1115	if (SwitchOpTy != MaskTy)
1116	SubReg = MIB.buildZExtOrTrunc(Res: MaskTy, Op: SubReg).getReg(Idx: 0);
1117
1118	B.RegVT = getMVTForLLT(Ty: MaskTy);
1119	B.Reg = SubReg;
1120
1121	MachineBasicBlock *MBB = B.Cases[0].ThisBB;
1122
1123	if (!B.FallthroughUnreachable)
1124	addSuccessorWithProb(Src: SwitchBB, Dst: B.Default, Prob: B.DefaultProb);
1125	addSuccessorWithProb(Src: SwitchBB, Dst: MBB, Prob: B.Prob);
1126
1127	SwitchBB->normalizeSuccProbs();
1128
1129	if (!B.FallthroughUnreachable) {
1130	// Conditional branch to the default block.
1131	auto RangeCst = MIB.buildConstant(Res: SwitchOpTy, Val: B.Range);
1132	auto RangeCmp = MIB.buildICmp(Pred: CmpInst::Predicate::ICMP_UGT, Res: LLT::scalar(SizeInBits: 1),
1133	Op0: RangeSub, Op1: RangeCst);
1134	MIB.buildBrCond(Tst: RangeCmp, Dest&: *B.Default);
1135	}
1136
1137	// Avoid emitting unnecessary branches to the next block.
1138	if (MBB != SwitchBB->getNextNode())
1139	MIB.buildBr(Dest&: *MBB);
1140	}
1141
1142	void IRTranslator::emitBitTestCase(SwitchCG::BitTestBlock &BB,
1143	MachineBasicBlock *NextMBB,
1144	BranchProbability BranchProbToNext,
1145	Register Reg, SwitchCG::BitTestCase &B,
1146	MachineBasicBlock *SwitchBB) {
1147	MachineIRBuilder &MIB = *CurBuilder;
1148	MIB.setMBB(*SwitchBB);
1149
1150	LLT SwitchTy = getLLTForMVT(Ty: BB.RegVT);
1151	Register Cmp;
1152	unsigned PopCount = llvm::popcount(Value: B.Mask);
1153	if (PopCount == 1) {
1154	// Testing for a single bit; just compare the shift count with what it
1155	// would need to be to shift a 1 bit in that position.
1156	auto MaskTrailingZeros =
1157	MIB.buildConstant(Res: SwitchTy, Val: llvm::countr_zero(Val: B.Mask));
1158	Cmp =
1159	MIB.buildICmp(Pred: ICmpInst::ICMP_EQ, Res: LLT::scalar(SizeInBits: 1), Op0: Reg, Op1: MaskTrailingZeros)
1160	.getReg(Idx: 0);
1161	} else if (PopCount == BB.Range) {
1162	// There is only one zero bit in the range, test for it directly.
1163	auto MaskTrailingOnes =
1164	MIB.buildConstant(Res: SwitchTy, Val: llvm::countr_one(Value: B.Mask));
1165	Cmp = MIB.buildICmp(Pred: CmpInst::ICMP_NE, Res: LLT::scalar(SizeInBits: 1), Op0: Reg, Op1: MaskTrailingOnes)
1166	.getReg(Idx: 0);
1167	} else {
1168	// Make desired shift.
1169	auto CstOne = MIB.buildConstant(Res: SwitchTy, Val: 1);
1170	auto SwitchVal = MIB.buildShl(Dst: SwitchTy, Src0: CstOne, Src1: Reg);
1171
1172	// Emit bit tests and jumps.
1173	auto CstMask = MIB.buildConstant(Res: SwitchTy, Val: B.Mask);
1174	auto AndOp = MIB.buildAnd(Dst: SwitchTy, Src0: SwitchVal, Src1: CstMask);
1175	auto CstZero = MIB.buildConstant(Res: SwitchTy, Val: 0);
1176	Cmp = MIB.buildICmp(Pred: CmpInst::ICMP_NE, Res: LLT::scalar(SizeInBits: 1), Op0: AndOp, Op1: CstZero)
1177	.getReg(Idx: 0);
1178	}
1179
1180	// The branch probability from SwitchBB to B.TargetBB is B.ExtraProb.
1181	addSuccessorWithProb(Src: SwitchBB, Dst: B.TargetBB, Prob: B.ExtraProb);
1182	// The branch probability from SwitchBB to NextMBB is BranchProbToNext.
1183	addSuccessorWithProb(Src: SwitchBB, Dst: NextMBB, Prob: BranchProbToNext);
1184	// It is not guaranteed that the sum of B.ExtraProb and BranchProbToNext is
1185	// one as they are relative probabilities (and thus work more like weights),
1186	// and hence we need to normalize them to let the sum of them become one.
1187	SwitchBB->normalizeSuccProbs();
1188
1189	// Record the fact that the IR edge from the header to the bit test target
1190	// will go through our new block. Neeeded for PHIs to have nodes added.
1191	addMachineCFGPred(Edge: {BB.Parent->getBasicBlock(), B.TargetBB->getBasicBlock()},
1192	NewPred: SwitchBB);
1193
1194	MIB.buildBrCond(Tst: Cmp, Dest&: *B.TargetBB);
1195
1196	// Avoid emitting unnecessary branches to the next block.
1197	if (NextMBB != SwitchBB->getNextNode())
1198	MIB.buildBr(Dest&: *NextMBB);
1199	}
1200
1201	bool IRTranslator::lowerBitTestWorkItem(
1202	SwitchCG::SwitchWorkListItem W, MachineBasicBlock *SwitchMBB,
1203	MachineBasicBlock *CurMBB, MachineBasicBlock *DefaultMBB,
1204	MachineIRBuilder &MIB, MachineFunction::iterator BBI,
1205	BranchProbability DefaultProb, BranchProbability UnhandledProbs,
1206	SwitchCG::CaseClusterIt I, MachineBasicBlock *Fallthrough,
1207	bool FallthroughUnreachable) {
1208	using namespace SwitchCG;
1209	MachineFunction *CurMF = SwitchMBB->getParent();
1210	// FIXME: Optimize away range check based on pivot comparisons.
1211	BitTestBlock *BTB = &SL->BitTestCases[I->BTCasesIndex];
1212	// The bit test blocks haven't been inserted yet; insert them here.
1213	for (BitTestCase &BTC : BTB->Cases)
1214	CurMF->insert(MBBI: BBI, MBB: BTC.ThisBB);
1215
1216	// Fill in fields of the BitTestBlock.
1217	BTB->Parent = CurMBB;
1218	BTB->Default = Fallthrough;
1219
1220	BTB->DefaultProb = UnhandledProbs;
1221	// If the cases in bit test don't form a contiguous range, we evenly
1222	// distribute the probability on the edge to Fallthrough to two
1223	// successors of CurMBB.
1224	if (!BTB->ContiguousRange) {
1225	BTB->Prob += DefaultProb / 2;
1226	BTB->DefaultProb -= DefaultProb / 2;
1227	}
1228
1229	if (FallthroughUnreachable)
1230	BTB->FallthroughUnreachable = true;
1231
1232	// If we're in the right place, emit the bit test header right now.
1233	if (CurMBB == SwitchMBB) {
1234	emitBitTestHeader(B&: *BTB, SwitchBB: SwitchMBB);
1235	BTB->Emitted = true;
1236	}
1237	return true;
1238	}
1239
1240	bool IRTranslator::lowerSwitchWorkItem(SwitchCG::SwitchWorkListItem W,
1241	Value *Cond,
1242	MachineBasicBlock *SwitchMBB,
1243	MachineBasicBlock *DefaultMBB,
1244	MachineIRBuilder &MIB) {
1245	using namespace SwitchCG;
1246	MachineFunction *CurMF = FuncInfo.MF;
1247	MachineBasicBlock *NextMBB = nullptr;
1248	MachineFunction::iterator BBI(W.MBB);
1249	if (++BBI != FuncInfo.MF->end())
1250	NextMBB = &*BBI;
1251
1252	if (EnableOpts) {
1253	// Here, we order cases by probability so the most likely case will be
1254	// checked first. However, two clusters can have the same probability in
1255	// which case their relative ordering is non-deterministic. So we use Low
1256	// as a tie-breaker as clusters are guaranteed to never overlap.
1257	llvm::sort(Start: W.FirstCluster, End: W.LastCluster + 1,
1258	Comp: [](const CaseCluster &a, const CaseCluster &b) {
1259	return a.Prob != b.Prob
1260	? a.Prob > b.Prob
1261	: a.Low->getValue().slt(RHS: b.Low->getValue());
1262	});
1263
1264	// Rearrange the case blocks so that the last one falls through if possible
1265	// without changing the order of probabilities.
1266	for (CaseClusterIt I = W.LastCluster; I > W.FirstCluster;) {
1267	--I;
1268	if (I->Prob > W.LastCluster->Prob)
1269	break;
1270	if (I->Kind == CC_Range && I->MBB == NextMBB) {
1271	std::swap(a&: *I, b&: *W.LastCluster);
1272	break;
1273	}
1274	}
1275	}
1276
1277	// Compute total probability.
1278	BranchProbability DefaultProb = W.DefaultProb;
1279	BranchProbability UnhandledProbs = DefaultProb;
1280	for (CaseClusterIt I = W.FirstCluster; I <= W.LastCluster; ++I)
1281	UnhandledProbs += I->Prob;
1282
1283	MachineBasicBlock *CurMBB = W.MBB;
1284	for (CaseClusterIt I = W.FirstCluster, E = W.LastCluster; I <= E; ++I) {
1285	bool FallthroughUnreachable = false;
1286	MachineBasicBlock *Fallthrough;
1287	if (I == W.LastCluster) {
1288	// For the last cluster, fall through to the default destination.
1289	Fallthrough = DefaultMBB;
1290	FallthroughUnreachable = isa<UnreachableInst>(
1291	Val: DefaultMBB->getBasicBlock()->getFirstNonPHIOrDbg());
1292	} else {
1293	Fallthrough = CurMF->CreateMachineBasicBlock(BB: CurMBB->getBasicBlock());
1294	CurMF->insert(MBBI: BBI, MBB: Fallthrough);
1295	}
1296	UnhandledProbs -= I->Prob;
1297
1298	switch (I->Kind) {
1299	case CC_BitTests: {
1300	if (!lowerBitTestWorkItem(W, SwitchMBB, CurMBB, DefaultMBB, MIB, BBI,
1301	DefaultProb, UnhandledProbs, I, Fallthrough,
1302	FallthroughUnreachable)) {
1303	LLVM_DEBUG(dbgs() << "Failed to lower bit test for switch");
1304	return false;
1305	}
1306	break;
1307	}
1308
1309	case CC_JumpTable: {
1310	if (!lowerJumpTableWorkItem(W, SwitchMBB, CurMBB, DefaultMBB, MIB, BBI,
1311	UnhandledProbs, I, Fallthrough,
1312	FallthroughUnreachable)) {
1313	LLVM_DEBUG(dbgs() << "Failed to lower jump table");
1314	return false;
1315	}
1316	break;
1317	}
1318	case CC_Range: {
1319	if (!lowerSwitchRangeWorkItem(I, Cond, Fallthrough,
1320	FallthroughUnreachable, UnhandledProbs,
1321	CurMBB, MIB, SwitchMBB)) {
1322	LLVM_DEBUG(dbgs() << "Failed to lower switch range");
1323	return false;
1324	}
1325	break;
1326	}
1327	}
1328	CurMBB = Fallthrough;
1329	}
1330
1331	return true;
1332	}
1333
1334	bool IRTranslator::translateIndirectBr(const User &U,
1335	MachineIRBuilder &MIRBuilder) {
1336	const IndirectBrInst &BrInst = cast<IndirectBrInst>(Val: U);
1337
1338	const Register Tgt = getOrCreateVReg(Val: *BrInst.getAddress());
1339	MIRBuilder.buildBrIndirect(Tgt);
1340
1341	// Link successors.
1342	SmallPtrSet<const BasicBlock *, 32> AddedSuccessors;
1343	MachineBasicBlock &CurBB = MIRBuilder.getMBB();
1344	for (const BasicBlock *Succ : successors(I: &BrInst)) {
1345	// It's legal for indirectbr instructions to have duplicate blocks in the
1346	// destination list. We don't allow this in MIR. Skip anything that's
1347	// already a successor.
1348	if (!AddedSuccessors.insert(Ptr: Succ).second)
1349	continue;
1350	CurBB.addSuccessor(Succ: &getMBB(BB: *Succ));
1351	}
1352
1353	return true;
1354	}
1355
1356	static bool isSwiftError(const Value *V) {
1357	if (auto Arg = dyn_cast<Argument>(Val: V))
1358	return Arg->hasSwiftErrorAttr();
1359	if (auto AI = dyn_cast<AllocaInst>(Val: V))
1360	return AI->isSwiftError();
1361	return false;
1362	}
1363
1364	bool IRTranslator::translateLoad(const User &U, MachineIRBuilder &MIRBuilder) {
1365	const LoadInst &LI = cast<LoadInst>(Val: U);
1366	TypeSize StoreSize = DL->getTypeStoreSize(Ty: LI.getType());
1367	if (StoreSize.isZero())
1368	return true;
1369
1370	ArrayRef<Register> Regs = getOrCreateVRegs(Val: LI);
1371	ArrayRef<uint64_t> Offsets = *VMap.getOffsets(V: LI);
1372	Register Base = getOrCreateVReg(Val: *LI.getPointerOperand());
1373	AAMDNodes AAInfo = LI.getAAMetadata();
1374
1375	const Value *Ptr = LI.getPointerOperand();
1376	Type *OffsetIRTy = DL->getIndexType(PtrTy: Ptr->getType());
1377	LLT OffsetTy = getLLTForType(Ty&: *OffsetIRTy, DL: *DL);
1378
1379	if (CLI->supportSwiftError() && isSwiftError(V: Ptr)) {
1380	assert(Regs.size() == 1 && "swifterror should be single pointer");
1381	Register VReg =
1382	SwiftError.getOrCreateVRegUseAt(&LI, &MIRBuilder.getMBB(), Ptr);
1383	MIRBuilder.buildCopy(Res: Regs[0], Op: VReg);
1384	return true;
1385	}
1386
1387	MachineMemOperand::Flags Flags =
1388	TLI->getLoadMemOperandFlags(LI, DL: *DL, AC, LibInfo);
1389	if (AA && !(Flags & MachineMemOperand::MOInvariant)) {
1390	if (AA->pointsToConstantMemory(
1391	Loc: MemoryLocation(Ptr, LocationSize::precise(Value: StoreSize), AAInfo))) {
1392	Flags |= MachineMemOperand::MOInvariant;
1393	}
1394	}
1395
1396	const MDNode *Ranges =
1397	Regs.size() == 1 ? LI.getMetadata(KindID: LLVMContext::MD_range) : nullptr;
1398	for (unsigned i = 0; i < Regs.size(); ++i) {
1399	Register Addr;
1400	MIRBuilder.materializePtrAdd(Res&: Addr, Op0: Base, ValueTy: OffsetTy, Value: Offsets[i] / 8);
1401
1402	MachinePointerInfo Ptr(LI.getPointerOperand(), Offsets[i] / 8);
1403	Align BaseAlign = getMemOpAlign(I: LI);
1404	auto MMO = MF->getMachineMemOperand(
1405	PtrInfo: Ptr, f: Flags, MemTy: MRI->getType(Reg: Regs[i]),
1406	base_alignment: commonAlignment(A: BaseAlign, Offset: Offsets[i] / 8), AAInfo, Ranges,
1407	SSID: LI.getSyncScopeID(), Ordering: LI.getOrdering());
1408	MIRBuilder.buildLoad(Res: Regs[i], Addr, MMO&: *MMO);
1409	}
1410
1411	return true;
1412	}
1413
1414	bool IRTranslator::translateStore(const User &U, MachineIRBuilder &MIRBuilder) {
1415	const StoreInst &SI = cast<StoreInst>(Val: U);
1416	if (DL->getTypeStoreSize(Ty: SI.getValueOperand()->getType()) == 0)
1417	return true;
1418
1419	ArrayRef<Register> Vals = getOrCreateVRegs(Val: *SI.getValueOperand());
1420	ArrayRef<uint64_t> Offsets = *VMap.getOffsets(V: *SI.getValueOperand());
1421	Register Base = getOrCreateVReg(Val: *SI.getPointerOperand());
1422
1423	Type *OffsetIRTy = DL->getIndexType(PtrTy: SI.getPointerOperandType());
1424	LLT OffsetTy = getLLTForType(Ty&: *OffsetIRTy, DL: *DL);
1425
1426	if (CLI->supportSwiftError() && isSwiftError(V: SI.getPointerOperand())) {
1427	assert(Vals.size() == 1 && "swifterror should be single pointer");
1428
1429	Register VReg = SwiftError.getOrCreateVRegDefAt(&SI, &MIRBuilder.getMBB(),
1430	SI.getPointerOperand());
1431	MIRBuilder.buildCopy(Res: VReg, Op: Vals[0]);
1432	return true;
1433	}
1434
1435	MachineMemOperand::Flags Flags = TLI->getStoreMemOperandFlags(SI, DL: *DL);
1436
1437	for (unsigned i = 0; i < Vals.size(); ++i) {
1438	Register Addr;
1439	MIRBuilder.materializePtrAdd(Res&: Addr, Op0: Base, ValueTy: OffsetTy, Value: Offsets[i] / 8);
1440
1441	MachinePointerInfo Ptr(SI.getPointerOperand(), Offsets[i] / 8);
1442	Align BaseAlign = getMemOpAlign(I: SI);
1443	auto MMO = MF->getMachineMemOperand(
1444	PtrInfo: Ptr, f: Flags, MemTy: MRI->getType(Reg: Vals[i]),
1445	base_alignment: commonAlignment(A: BaseAlign, Offset: Offsets[i] / 8), AAInfo: SI.getAAMetadata(), Ranges: nullptr,
1446	SSID: SI.getSyncScopeID(), Ordering: SI.getOrdering());
1447	MIRBuilder.buildStore(Val: Vals[i], Addr, MMO&: *MMO);
1448	}
1449	return true;
1450	}
1451
1452	static uint64_t getOffsetFromIndices(const User &U, const DataLayout &DL) {
1453	const Value *Src = U.getOperand(i: 0);
1454	Type *Int32Ty = Type::getInt32Ty(C&: U.getContext());
1455
1456	// getIndexedOffsetInType is designed for GEPs, so the first index is the
1457	// usual array element rather than looking into the actual aggregate.
1458	SmallVector<Value *, 1> Indices;
1459	Indices.push_back(Elt: ConstantInt::get(Ty: Int32Ty, V: 0));
1460
1461	if (const ExtractValueInst *EVI = dyn_cast<ExtractValueInst>(Val: &U)) {
1462	for (auto Idx : EVI->indices())
1463	Indices.push_back(Elt: ConstantInt::get(Ty: Int32Ty, V: Idx));
1464	} else if (const InsertValueInst *IVI = dyn_cast<InsertValueInst>(Val: &U)) {
1465	for (auto Idx : IVI->indices())
1466	Indices.push_back(Elt: ConstantInt::get(Ty: Int32Ty, V: Idx));
1467	} else {
1468	for (unsigned i = 1; i < U.getNumOperands(); ++i)
1469	Indices.push_back(Elt: U.getOperand(i));
1470	}
1471
1472	return 8 * static_cast<uint64_t>(
1473	DL.getIndexedOffsetInType(ElemTy: Src->getType(), Indices));
1474	}
1475
1476	bool IRTranslator::translateExtractValue(const User &U,
1477	MachineIRBuilder &MIRBuilder) {
1478	const Value *Src = U.getOperand(i: 0);
1479	uint64_t Offset = getOffsetFromIndices(U, DL: *DL);
1480	ArrayRef<Register> SrcRegs = getOrCreateVRegs(Val: *Src);
1481	ArrayRef<uint64_t> Offsets = *VMap.getOffsets(V: *Src);
1482	unsigned Idx = llvm::lower_bound(Range&: Offsets, Value&: Offset) - Offsets.begin();
1483	auto &DstRegs = allocateVRegs(Val: U);
1484
1485	for (unsigned i = 0; i < DstRegs.size(); ++i)
1486	DstRegs[i] = SrcRegs[Idx++];
1487
1488	return true;
1489	}
1490
1491	bool IRTranslator::translateInsertValue(const User &U,
1492	MachineIRBuilder &MIRBuilder) {
1493	const Value *Src = U.getOperand(i: 0);
1494	uint64_t Offset = getOffsetFromIndices(U, DL: *DL);
1495	auto &DstRegs = allocateVRegs(Val: U);
1496	ArrayRef<uint64_t> DstOffsets = *VMap.getOffsets(V: U);
1497	ArrayRef<Register> SrcRegs = getOrCreateVRegs(Val: *Src);
1498	ArrayRef<Register> InsertedRegs = getOrCreateVRegs(Val: *U.getOperand(i: 1));
1499	auto *InsertedIt = InsertedRegs.begin();
1500
1501	for (unsigned i = 0; i < DstRegs.size(); ++i) {
1502	if (DstOffsets[i] >= Offset && InsertedIt != InsertedRegs.end())
1503	DstRegs[i] = *InsertedIt++;
1504	else
1505	DstRegs[i] = SrcRegs[i];
1506	}
1507
1508	return true;
1509	}
1510
1511	bool IRTranslator::translateSelect(const User &U,
1512	MachineIRBuilder &MIRBuilder) {
1513	Register Tst = getOrCreateVReg(Val: *U.getOperand(i: 0));
1514	ArrayRef<Register> ResRegs = getOrCreateVRegs(Val: U);
1515	ArrayRef<Register> Op0Regs = getOrCreateVRegs(Val: *U.getOperand(i: 1));
1516	ArrayRef<Register> Op1Regs = getOrCreateVRegs(Val: *U.getOperand(i: 2));
1517
1518	uint32_t Flags = 0;
1519	if (const SelectInst *SI = dyn_cast<SelectInst>(Val: &U))
1520	Flags = MachineInstr::copyFlagsFromInstruction(I: *SI);
1521
1522	for (unsigned i = 0; i < ResRegs.size(); ++i) {
1523	MIRBuilder.buildSelect(Res: ResRegs[i], Tst, Op0: Op0Regs[i], Op1: Op1Regs[i], Flags);
1524	}
1525
1526	return true;
1527	}
1528
1529	bool IRTranslator::translateCopy(const User &U, const Value &V,
1530	MachineIRBuilder &MIRBuilder) {
1531	Register Src = getOrCreateVReg(Val: V);
1532	auto &Regs = *VMap.getVRegs(V: U);
1533	if (Regs.empty()) {
1534	Regs.push_back(Elt: Src);
1535	VMap.getOffsets(V: U)->push_back(Elt: 0);
1536	} else {
1537	// If we already assigned a vreg for this instruction, we can't change that.
1538	// Emit a copy to satisfy the users we already emitted.
1539	MIRBuilder.buildCopy(Res: Regs[0], Op: Src);
1540	}
1541	return true;
1542	}
1543
1544	bool IRTranslator::translateBitCast(const User &U,
1545	MachineIRBuilder &MIRBuilder) {
1546	// If we're bitcasting to the source type, we can reuse the source vreg.
1547	if (getLLTForType(Ty&: *U.getOperand(i: 0)->getType(), DL: *DL) ==
1548	getLLTForType(Ty&: *U.getType(), DL: *DL)) {
1549	// If the source is a ConstantInt then it was probably created by
1550	// ConstantHoisting and we should leave it alone.
1551	if (isa<ConstantInt>(Val: U.getOperand(i: 0)))
1552	return translateCast(Opcode: TargetOpcode::G_CONSTANT_FOLD_BARRIER, U,
1553	MIRBuilder);
1554	return translateCopy(U, V: *U.getOperand(i: 0), MIRBuilder);
1555	}
1556
1557	return translateCast(Opcode: TargetOpcode::G_BITCAST, U, MIRBuilder);
1558	}
1559
1560	bool IRTranslator::translateCast(unsigned Opcode, const User &U,
1561	MachineIRBuilder &MIRBuilder) {
1562	if (U.getType()->getScalarType()->isBFloatTy() ||
1563	U.getOperand(i: 0)->getType()->getScalarType()->isBFloatTy())
1564	return false;
1565
1566	uint32_t Flags = 0;
1567	if (const Instruction *I = dyn_cast<Instruction>(Val: &U))
1568	Flags = MachineInstr::copyFlagsFromInstruction(I: *I);
1569
1570	Register Op = getOrCreateVReg(Val: *U.getOperand(i: 0));
1571	Register Res = getOrCreateVReg(Val: U);
1572	MIRBuilder.buildInstr(Opc: Opcode, DstOps: {Res}, SrcOps: {Op}, Flags);
1573	return true;
1574	}
1575
1576	bool IRTranslator::translateGetElementPtr(const User &U,
1577	MachineIRBuilder &MIRBuilder) {
1578	Value &Op0 = *U.getOperand(i: 0);
1579	Register BaseReg = getOrCreateVReg(Val: Op0);
1580	Type *PtrIRTy = Op0.getType();
1581	LLT PtrTy = getLLTForType(Ty&: *PtrIRTy, DL: *DL);
1582	Type *OffsetIRTy = DL->getIndexType(PtrTy: PtrIRTy);
1583	LLT OffsetTy = getLLTForType(Ty&: *OffsetIRTy, DL: *DL);
1584
1585	uint32_t Flags = 0;
1586	if (isa<Instruction>(Val: U)) {
1587	const Instruction &I = cast<Instruction>(Val: U);
1588	Flags = MachineInstr::copyFlagsFromInstruction(I);
1589	}
1590
1591	// Normalize Vector GEP - all scalar operands should be converted to the
1592	// splat vector.
1593	unsigned VectorWidth = 0;
1594
1595	// True if we should use a splat vector; using VectorWidth alone is not
1596	// sufficient.
1597	bool WantSplatVector = false;
1598	if (auto *VT = dyn_cast<VectorType>(Val: U.getType())) {
1599	VectorWidth = cast<FixedVectorType>(Val: VT)->getNumElements();
1600	// We don't produce 1 x N vectors; those are treated as scalars.
1601	WantSplatVector = VectorWidth > 1;
1602	}
1603
1604	// We might need to splat the base pointer into a vector if the offsets
1605	// are vectors.
1606	if (WantSplatVector && !PtrTy.isVector()) {
1607	BaseReg = MIRBuilder
1608	.buildSplatBuildVector(Res: LLT::fixed_vector(NumElements: VectorWidth, ScalarTy: PtrTy),
1609	Src: BaseReg)
1610	.getReg(Idx: 0);
1611	PtrIRTy = FixedVectorType::get(ElementType: PtrIRTy, NumElts: VectorWidth);
1612	PtrTy = getLLTForType(Ty&: *PtrIRTy, DL: *DL);
1613	OffsetIRTy = DL->getIndexType(PtrTy: PtrIRTy);
1614	OffsetTy = getLLTForType(Ty&: *OffsetIRTy, DL: *DL);
1615	}
1616
1617	int64_t Offset = 0;
1618	for (gep_type_iterator GTI = gep_type_begin(GEP: &U), E = gep_type_end(GEP: &U);
1619	GTI != E; ++GTI) {
1620	const Value *Idx = GTI.getOperand();
1621	if (StructType *StTy = GTI.getStructTypeOrNull()) {
1622	unsigned Field = cast<Constant>(Val: Idx)->getUniqueInteger().getZExtValue();
1623	Offset += DL->getStructLayout(Ty: StTy)->getElementOffset(Idx: Field);
1624	continue;
1625	} else {
1626	uint64_t ElementSize = GTI.getSequentialElementStride(DL: *DL);
1627
1628	// If this is a scalar constant or a splat vector of constants,
1629	// handle it quickly.
1630	if (const auto *CI = dyn_cast<ConstantInt>(Val: Idx)) {
1631	if (std::optional<int64_t> Val = CI->getValue().trySExtValue()) {
1632	Offset += ElementSize * *Val;
1633	continue;
1634	}
1635	}
1636
1637	if (Offset != 0) {
1638	auto OffsetMIB = MIRBuilder.buildConstant(Res: {OffsetTy}, Val: Offset);
1639	BaseReg = MIRBuilder.buildPtrAdd(Res: PtrTy, Op0: BaseReg, Op1: OffsetMIB.getReg(Idx: 0))
1640	.getReg(Idx: 0);
1641	Offset = 0;
1642	}
1643
1644	Register IdxReg = getOrCreateVReg(Val: *Idx);
1645	LLT IdxTy = MRI->getType(Reg: IdxReg);
1646	if (IdxTy != OffsetTy) {
1647	if (!IdxTy.isVector() && WantSplatVector) {
1648	IdxReg = MIRBuilder
1649	.buildSplatBuildVector(Res: OffsetTy.changeElementType(NewEltTy: IdxTy),
1650	Src: IdxReg)
1651	.getReg(Idx: 0);
1652	}
1653
1654	IdxReg = MIRBuilder.buildSExtOrTrunc(Res: OffsetTy, Op: IdxReg).getReg(Idx: 0);
1655	}
1656
1657	// N = N + Idx * ElementSize;
1658	// Avoid doing it for ElementSize of 1.
1659	Register GepOffsetReg;
1660	if (ElementSize != 1) {
1661	auto ElementSizeMIB = MIRBuilder.buildConstant(
1662	Res: getLLTForType(Ty&: *OffsetIRTy, DL: *DL), Val: ElementSize);
1663	GepOffsetReg =
1664	MIRBuilder.buildMul(Dst: OffsetTy, Src0: IdxReg, Src1: ElementSizeMIB).getReg(Idx: 0);
1665	} else
1666	GepOffsetReg = IdxReg;
1667
1668	BaseReg = MIRBuilder.buildPtrAdd(Res: PtrTy, Op0: BaseReg, Op1: GepOffsetReg).getReg(Idx: 0);
1669	}
1670	}
1671
1672	if (Offset != 0) {
1673	auto OffsetMIB =
1674	MIRBuilder.buildConstant(Res: OffsetTy, Val: Offset);
1675
1676	if (int64_t(Offset) >= 0 && cast<GEPOperator>(Val: U).isInBounds())
1677	Flags |= MachineInstr::MIFlag::NoUWrap;
1678
1679	MIRBuilder.buildPtrAdd(Res: getOrCreateVReg(Val: U), Op0: BaseReg, Op1: OffsetMIB.getReg(Idx: 0),
1680	Flags);
1681	return true;
1682	}
1683
1684	MIRBuilder.buildCopy(Res: getOrCreateVReg(Val: U), Op: BaseReg);
1685	return true;
1686	}
1687
1688	bool IRTranslator::translateMemFunc(const CallInst &CI,
1689	MachineIRBuilder &MIRBuilder,
1690	unsigned Opcode) {
1691	const Value *SrcPtr = CI.getArgOperand(i: 1);
1692	// If the source is undef, then just emit a nop.
1693	if (isa<UndefValue>(Val: SrcPtr))
1694	return true;
1695
1696	SmallVector<Register, 3> SrcRegs;
1697
1698	unsigned MinPtrSize = UINT_MAX;
1699	for (auto AI = CI.arg_begin(), AE = CI.arg_end(); std::next(x: AI) != AE; ++AI) {
1700	Register SrcReg = getOrCreateVReg(Val: **AI);
1701	LLT SrcTy = MRI->getType(Reg: SrcReg);
1702	if (SrcTy.isPointer())
1703	MinPtrSize = std::min<unsigned>(a: SrcTy.getSizeInBits(), b: MinPtrSize);
1704	SrcRegs.push_back(Elt: SrcReg);
1705	}
1706
1707	LLT SizeTy = LLT::scalar(SizeInBits: MinPtrSize);
1708
1709	// The size operand should be the minimum of the pointer sizes.
1710	Register &SizeOpReg = SrcRegs[SrcRegs.size() - 1];
1711	if (MRI->getType(Reg: SizeOpReg) != SizeTy)
1712	SizeOpReg = MIRBuilder.buildZExtOrTrunc(Res: SizeTy, Op: SizeOpReg).getReg(Idx: 0);
1713
1714	auto ICall = MIRBuilder.buildInstr(Opcode);
1715	for (Register SrcReg : SrcRegs)
1716	ICall.addUse(RegNo: SrcReg);
1717
1718	Align DstAlign;
1719	Align SrcAlign;
1720	unsigned IsVol =
1721	cast<ConstantInt>(Val: CI.getArgOperand(i: CI.arg_size() - 1))->getZExtValue();
1722
1723	ConstantInt *CopySize = nullptr;
1724
1725	if (auto *MCI = dyn_cast<MemCpyInst>(Val: &CI)) {
1726	DstAlign = MCI->getDestAlign().valueOrOne();
1727	SrcAlign = MCI->getSourceAlign().valueOrOne();
1728	CopySize = dyn_cast<ConstantInt>(Val: MCI->getArgOperand(i: 2));
1729	} else if (auto *MCI = dyn_cast<MemCpyInlineInst>(Val: &CI)) {
1730	DstAlign = MCI->getDestAlign().valueOrOne();
1731	SrcAlign = MCI->getSourceAlign().valueOrOne();
1732	CopySize = dyn_cast<ConstantInt>(Val: MCI->getArgOperand(i: 2));
1733	} else if (auto *MMI = dyn_cast<MemMoveInst>(Val: &CI)) {
1734	DstAlign = MMI->getDestAlign().valueOrOne();
1735	SrcAlign = MMI->getSourceAlign().valueOrOne();
1736	CopySize = dyn_cast<ConstantInt>(Val: MMI->getArgOperand(i: 2));
1737	} else {
1738	auto *MSI = cast<MemSetInst>(Val: &CI);
1739	DstAlign = MSI->getDestAlign().valueOrOne();
1740	}
1741
1742	if (Opcode != TargetOpcode::G_MEMCPY_INLINE) {
1743	// We need to propagate the tail call flag from the IR inst as an argument.
1744	// Otherwise, we have to pessimize and assume later that we cannot tail call
1745	// any memory intrinsics.
1746	ICall.addImm(Val: CI.isTailCall() ? 1 : 0);
1747	}
1748
1749	// Create mem operands to store the alignment and volatile info.
1750	MachineMemOperand::Flags LoadFlags = MachineMemOperand::MOLoad;
1751	MachineMemOperand::Flags StoreFlags = MachineMemOperand::MOStore;
1752	if (IsVol) {
1753	LoadFlags |= MachineMemOperand::MOVolatile;
1754	StoreFlags |= MachineMemOperand::MOVolatile;
1755	}
1756
1757	AAMDNodes AAInfo = CI.getAAMetadata();
1758	if (AA && CopySize &&
1759	AA->pointsToConstantMemory(Loc: MemoryLocation(
1760	SrcPtr, LocationSize::precise(Value: CopySize->getZExtValue()), AAInfo))) {
1761	LoadFlags |= MachineMemOperand::MOInvariant;
1762
1763	// FIXME: pointsToConstantMemory probably does not imply dereferenceable,
1764	// but the previous usage implied it did. Probably should check
1765	// isDereferenceableAndAlignedPointer.
1766	LoadFlags |= MachineMemOperand::MODereferenceable;
1767	}
1768
1769	ICall.addMemOperand(
1770	MMO: MF->getMachineMemOperand(PtrInfo: MachinePointerInfo(CI.getArgOperand(i: 0)),
1771	F: StoreFlags, Size: 1, BaseAlignment: DstAlign, AAInfo));
1772	if (Opcode != TargetOpcode::G_MEMSET)
1773	ICall.addMemOperand(MMO: MF->getMachineMemOperand(
1774	PtrInfo: MachinePointerInfo(SrcPtr), F: LoadFlags, Size: 1, BaseAlignment: SrcAlign, AAInfo));
1775
1776	return true;
1777	}
1778
1779	bool IRTranslator::translateTrap(const CallInst &CI,
1780	MachineIRBuilder &MIRBuilder,
1781	unsigned Opcode) {
1782	StringRef TrapFuncName =
1783	CI.getAttributes().getFnAttr(Kind: "trap-func-name").getValueAsString();
1784	if (TrapFuncName.empty()) {
1785	if (Opcode == TargetOpcode::G_UBSANTRAP) {
1786	uint64_t Code = cast<ConstantInt>(Val: CI.getOperand(i_nocapture: 0))->getZExtValue();
1787	MIRBuilder.buildInstr(Opc: Opcode, DstOps: {}, SrcOps: ArrayRef<llvm::SrcOp>{Code});
1788	} else {
1789	MIRBuilder.buildInstr(Opcode);
1790	}
1791	return true;
1792	}
1793
1794	CallLowering::CallLoweringInfo Info;
1795	if (Opcode == TargetOpcode::G_UBSANTRAP)
1796	Info.OrigArgs.push_back(Elt: {getOrCreateVRegs(Val: *CI.getArgOperand(i: 0)),
1797	CI.getArgOperand(i: 0)->getType(), 0});
1798
1799	Info.Callee = MachineOperand::CreateES(SymName: TrapFuncName.data());
1800	Info.CB = &CI;
1801	Info.OrigRet = {Register(), Type::getVoidTy(C&: CI.getContext()), 0};
1802	return CLI->lowerCall(MIRBuilder, Info);
1803	}
1804
1805	bool IRTranslator::translateVectorInterleave2Intrinsic(
1806	const CallInst &CI, MachineIRBuilder &MIRBuilder) {
1807	assert(CI.getIntrinsicID() == Intrinsic::experimental_vector_interleave2 &&
1808	"This function can only be called on the interleave2 intrinsic!");
1809	// Canonicalize interleave2 to G_SHUFFLE_VECTOR (similar to SelectionDAG).
1810	Register Op0 = getOrCreateVReg(Val: *CI.getOperand(i_nocapture: 0));
1811	Register Op1 = getOrCreateVReg(Val: *CI.getOperand(i_nocapture: 1));
1812	Register Res = getOrCreateVReg(Val: CI);
1813
1814	LLT OpTy = MRI->getType(Reg: Op0);
1815	MIRBuilder.buildShuffleVector(Res, Src1: Op0, Src2: Op1,
1816	Mask: createInterleaveMask(VF: OpTy.getNumElements(), NumVecs: 2));
1817
1818	return true;
1819	}
1820
1821	bool IRTranslator::translateVectorDeinterleave2Intrinsic(
1822	const CallInst &CI, MachineIRBuilder &MIRBuilder) {
1823	assert(CI.getIntrinsicID() == Intrinsic::experimental_vector_deinterleave2 &&
1824	"This function can only be called on the deinterleave2 intrinsic!");
1825	// Canonicalize deinterleave2 to shuffles that extract sub-vectors (similar to
1826	// SelectionDAG).
1827	Register Op = getOrCreateVReg(Val: *CI.getOperand(i_nocapture: 0));
1828	auto Undef = MIRBuilder.buildUndef(Res: MRI->getType(Reg: Op));
1829	ArrayRef<Register> Res = getOrCreateVRegs(Val: CI);
1830
1831	LLT ResTy = MRI->getType(Reg: Res[0]);
1832	MIRBuilder.buildShuffleVector(Res: Res[0], Src1: Op, Src2: Undef,
1833	Mask: createStrideMask(Start: 0, Stride: 2, VF: ResTy.getNumElements()));
1834	MIRBuilder.buildShuffleVector(Res: Res[1], Src1: Op, Src2: Undef,
1835	Mask: createStrideMask(Start: 1, Stride: 2, VF: ResTy.getNumElements()));
1836
1837	return true;
1838	}
1839
1840	void IRTranslator::getStackGuard(Register DstReg,
1841	MachineIRBuilder &MIRBuilder) {
1842	const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo();
1843	MRI->setRegClass(Reg: DstReg, RC: TRI->getPointerRegClass(MF: *MF));
1844	auto MIB =
1845	MIRBuilder.buildInstr(Opc: TargetOpcode::LOAD_STACK_GUARD, DstOps: {DstReg}, SrcOps: {});
1846
1847	Value *Global = TLI->getSDagStackGuard(M: *MF->getFunction().getParent());
1848	if (!Global)
1849	return;
1850
1851	unsigned AddrSpace = Global->getType()->getPointerAddressSpace();
1852	LLT PtrTy = LLT::pointer(AddressSpace: AddrSpace, SizeInBits: DL->getPointerSizeInBits(AS: AddrSpace));
1853
1854	MachinePointerInfo MPInfo(Global);
1855	auto Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant |
1856	MachineMemOperand::MODereferenceable;
1857	MachineMemOperand *MemRef = MF->getMachineMemOperand(
1858	PtrInfo: MPInfo, f: Flags, MemTy: PtrTy, base_alignment: DL->getPointerABIAlignment(AS: AddrSpace));
1859	MIB.setMemRefs({MemRef});
1860	}
1861
1862	bool IRTranslator::translateOverflowIntrinsic(const CallInst &CI, unsigned Op,
1863	MachineIRBuilder &MIRBuilder) {
1864	ArrayRef<Register> ResRegs = getOrCreateVRegs(Val: CI);
1865	MIRBuilder.buildInstr(
1866	Opc: Op, DstOps: {ResRegs[0], ResRegs[1]},
1867	SrcOps: {getOrCreateVReg(Val: *CI.getOperand(i_nocapture: 0)), getOrCreateVReg(Val: *CI.getOperand(i_nocapture: 1))});
1868
1869	return true;
1870	}
1871
1872	bool IRTranslator::translateFixedPointIntrinsic(unsigned Op, const CallInst &CI,
1873	MachineIRBuilder &MIRBuilder) {
1874	Register Dst = getOrCreateVReg(Val: CI);
1875	Register Src0 = getOrCreateVReg(Val: *CI.getOperand(i_nocapture: 0));
1876	Register Src1 = getOrCreateVReg(Val: *CI.getOperand(i_nocapture: 1));
1877	uint64_t Scale = cast<ConstantInt>(Val: CI.getOperand(i_nocapture: 2))->getZExtValue();
1878	MIRBuilder.buildInstr(Opc: Op, DstOps: {Dst}, SrcOps: { Src0, Src1, Scale });
1879	return true;
1880	}
1881
1882	unsigned IRTranslator::getSimpleIntrinsicOpcode(Intrinsic::ID ID) {
1883	switch (ID) {
1884	default:
1885	break;
1886	case Intrinsic::bswap:
1887	return TargetOpcode::G_BSWAP;
1888	case Intrinsic::bitreverse:
1889	return TargetOpcode::G_BITREVERSE;
1890	case Intrinsic::fshl:
1891	return TargetOpcode::G_FSHL;
1892	case Intrinsic::fshr:
1893	return TargetOpcode::G_FSHR;
1894	case Intrinsic::ceil:
1895	return TargetOpcode::G_FCEIL;
1896	case Intrinsic::cos:
1897	return TargetOpcode::G_FCOS;
1898	case Intrinsic::ctpop:
1899	return TargetOpcode::G_CTPOP;
1900	case Intrinsic::exp:
1901	return TargetOpcode::G_FEXP;
1902	case Intrinsic::exp2:
1903	return TargetOpcode::G_FEXP2;
1904	case Intrinsic::exp10:
1905	return TargetOpcode::G_FEXP10;
1906	case Intrinsic::fabs:
1907	return TargetOpcode::G_FABS;
1908	case Intrinsic::copysign:
1909	return TargetOpcode::G_FCOPYSIGN;
1910	case Intrinsic::minnum:
1911	return TargetOpcode::G_FMINNUM;
1912	case Intrinsic::maxnum:
1913	return TargetOpcode::G_FMAXNUM;
1914	case Intrinsic::minimum:
1915	return TargetOpcode::G_FMINIMUM;
1916	case Intrinsic::maximum:
1917	return TargetOpcode::G_FMAXIMUM;
1918	case Intrinsic::canonicalize:
1919	return TargetOpcode::G_FCANONICALIZE;
1920	case Intrinsic::floor:
1921	return TargetOpcode::G_FFLOOR;
1922	case Intrinsic::fma:
1923	return TargetOpcode::G_FMA;
1924	case Intrinsic::log:
1925	return TargetOpcode::G_FLOG;
1926	case Intrinsic::log2:
1927	return TargetOpcode::G_FLOG2;
1928	case Intrinsic::log10:
1929	return TargetOpcode::G_FLOG10;
1930	case Intrinsic::ldexp:
1931	return TargetOpcode::G_FLDEXP;
1932	case Intrinsic::nearbyint:
1933	return TargetOpcode::G_FNEARBYINT;
1934	case Intrinsic::pow:
1935	return TargetOpcode::G_FPOW;
1936	case Intrinsic::powi:
1937	return TargetOpcode::G_FPOWI;
1938	case Intrinsic::rint:
1939	return TargetOpcode::G_FRINT;
1940	case Intrinsic::round:
1941	return TargetOpcode::G_INTRINSIC_ROUND;
1942	case Intrinsic::roundeven:
1943	return TargetOpcode::G_INTRINSIC_ROUNDEVEN;
1944	case Intrinsic::sin:
1945	return TargetOpcode::G_FSIN;
1946	case Intrinsic::sqrt:
1947	return TargetOpcode::G_FSQRT;
1948	case Intrinsic::trunc:
1949	return TargetOpcode::G_INTRINSIC_TRUNC;
1950	case Intrinsic::readcyclecounter:
1951	return TargetOpcode::G_READCYCLECOUNTER;
1952	case Intrinsic::readsteadycounter:
1953	return TargetOpcode::G_READSTEADYCOUNTER;
1954	case Intrinsic::ptrmask:
1955	return TargetOpcode::G_PTRMASK;
1956	case Intrinsic::lrint:
1957	return TargetOpcode::G_INTRINSIC_LRINT;
1958	case Intrinsic::llrint:
1959	return TargetOpcode::G_INTRINSIC_LLRINT;
1960	// FADD/FMUL require checking the FMF, so are handled elsewhere.
1961	case Intrinsic::vector_reduce_fmin:
1962	return TargetOpcode::G_VECREDUCE_FMIN;
1963	case Intrinsic::vector_reduce_fmax:
1964	return TargetOpcode::G_VECREDUCE_FMAX;
1965	case Intrinsic::vector_reduce_fminimum:
1966	return TargetOpcode::G_VECREDUCE_FMINIMUM;
1967	case Intrinsic::vector_reduce_fmaximum:
1968	return TargetOpcode::G_VECREDUCE_FMAXIMUM;
1969	case Intrinsic::vector_reduce_add:
1970	return TargetOpcode::G_VECREDUCE_ADD;
1971	case Intrinsic::vector_reduce_mul:
1972	return TargetOpcode::G_VECREDUCE_MUL;
1973	case Intrinsic::vector_reduce_and:
1974	return TargetOpcode::G_VECREDUCE_AND;
1975	case Intrinsic::vector_reduce_or:
1976	return TargetOpcode::G_VECREDUCE_OR;
1977	case Intrinsic::vector_reduce_xor:
1978	return TargetOpcode::G_VECREDUCE_XOR;
1979	case Intrinsic::vector_reduce_smax:
1980	return TargetOpcode::G_VECREDUCE_SMAX;
1981	case Intrinsic::vector_reduce_smin:
1982	return TargetOpcode::G_VECREDUCE_SMIN;
1983	case Intrinsic::vector_reduce_umax:
1984	return TargetOpcode::G_VECREDUCE_UMAX;
1985	case Intrinsic::vector_reduce_umin:
1986	return TargetOpcode::G_VECREDUCE_UMIN;
1987	case Intrinsic::lround:
1988	return TargetOpcode::G_LROUND;
1989	case Intrinsic::llround:
1990	return TargetOpcode::G_LLROUND;
1991	case Intrinsic::get_fpenv:
1992	return TargetOpcode::G_GET_FPENV;
1993	case Intrinsic::get_fpmode:
1994	return TargetOpcode::G_GET_FPMODE;
1995	}
1996	return Intrinsic::not_intrinsic;
1997	}
1998
1999	bool IRTranslator::translateSimpleIntrinsic(const CallInst &CI,
2000	Intrinsic::ID ID,
2001	MachineIRBuilder &MIRBuilder) {
2002
2003	unsigned Op = getSimpleIntrinsicOpcode(ID);
2004
2005	// Is this a simple intrinsic?
2006	if (Op == Intrinsic::not_intrinsic)
2007	return false;
2008
2009	// Yes. Let's translate it.
2010	SmallVector<llvm::SrcOp, 4> VRegs;
2011	for (const auto &Arg : CI.args())
2012	VRegs.push_back(Elt: getOrCreateVReg(Val: *Arg));
2013
2014	MIRBuilder.buildInstr(Opc: Op, DstOps: {getOrCreateVReg(Val: CI)}, SrcOps: VRegs,
2015	Flags: MachineInstr::copyFlagsFromInstruction(I: CI));
2016	return true;
2017	}
2018
2019	// TODO: Include ConstainedOps.def when all strict instructions are defined.
2020	static unsigned getConstrainedOpcode(Intrinsic::ID ID) {
2021	switch (ID) {
2022	case Intrinsic::experimental_constrained_fadd:
2023	return TargetOpcode::G_STRICT_FADD;
2024	case Intrinsic::experimental_constrained_fsub:
2025	return TargetOpcode::G_STRICT_FSUB;
2026	case Intrinsic::experimental_constrained_fmul:
2027	return TargetOpcode::G_STRICT_FMUL;
2028	case Intrinsic::experimental_constrained_fdiv:
2029	return TargetOpcode::G_STRICT_FDIV;
2030	case Intrinsic::experimental_constrained_frem:
2031	return TargetOpcode::G_STRICT_FREM;
2032	case Intrinsic::experimental_constrained_fma:
2033	return TargetOpcode::G_STRICT_FMA;
2034	case Intrinsic::experimental_constrained_sqrt:
2035	return TargetOpcode::G_STRICT_FSQRT;
2036	case Intrinsic::experimental_constrained_ldexp:
2037	return TargetOpcode::G_STRICT_FLDEXP;
2038	default:
2039	return 0;
2040	}
2041	}
2042
2043	bool IRTranslator::translateConstrainedFPIntrinsic(
2044	const ConstrainedFPIntrinsic &FPI, MachineIRBuilder &MIRBuilder) {
2045	fp::ExceptionBehavior EB = *FPI.getExceptionBehavior();
2046
2047	unsigned Opcode = getConstrainedOpcode(ID: FPI.getIntrinsicID());
2048	if (!Opcode)
2049	return false;
2050
2051	uint32_t Flags = MachineInstr::copyFlagsFromInstruction(I: FPI);
2052	if (EB == fp::ExceptionBehavior::ebIgnore)
2053	Flags |= MachineInstr::NoFPExcept;
2054
2055	SmallVector<llvm::SrcOp, 4> VRegs;
2056	VRegs.push_back(Elt: getOrCreateVReg(Val: *FPI.getArgOperand(i: 0)));
2057	if (!FPI.isUnaryOp())
2058	VRegs.push_back(Elt: getOrCreateVReg(Val: *FPI.getArgOperand(i: 1)));
2059	if (FPI.isTernaryOp())
2060	VRegs.push_back(Elt: getOrCreateVReg(Val: *FPI.getArgOperand(i: 2)));
2061
2062	MIRBuilder.buildInstr(Opc: Opcode, DstOps: {getOrCreateVReg(Val: FPI)}, SrcOps: VRegs, Flags);
2063	return true;
2064	}
2065
2066	std::optional<MCRegister> IRTranslator::getArgPhysReg(Argument &Arg) {
2067	auto VRegs = getOrCreateVRegs(Val: Arg);
2068	if (VRegs.size() != 1)
2069	return std::nullopt;
2070
2071	// Arguments are lowered as a copy of a livein physical register.
2072	auto *VRegDef = MF->getRegInfo().getVRegDef(Reg: VRegs[0]);
2073	if (!VRegDef || !VRegDef->isCopy())
2074	return std::nullopt;
2075	return VRegDef->getOperand(i: 1).getReg().asMCReg();
2076	}
2077
2078	bool IRTranslator::translateIfEntryValueArgument(bool isDeclare, Value *Val,
2079	const DILocalVariable *Var,
2080	const DIExpression *Expr,
2081	const DebugLoc &DL,
2082	MachineIRBuilder &MIRBuilder) {
2083	auto *Arg = dyn_cast<Argument>(Val);
2084	if (!Arg)
2085	return false;
2086
2087	if (!Expr->isEntryValue())
2088	return false;
2089
2090	std::optional<MCRegister> PhysReg = getArgPhysReg(Arg&: *Arg);
2091	if (!PhysReg) {
2092	LLVM_DEBUG(dbgs() << "Dropping dbg." << (isDeclare ? "declare" : "value")
2093	<< ": expression is entry_value but "
2094	<< "couldn't find a physical register\n");
2095	LLVM_DEBUG(dbgs() << *Var << "\n");
2096	return true;
2097	}
2098
2099	if (isDeclare) {
2100	// Append an op deref to account for the fact that this is a dbg_declare.
2101	Expr = DIExpression::append(Expr, Ops: dwarf::DW_OP_deref);
2102	MF->setVariableDbgInfo(Var, Expr, Reg: *PhysReg, Loc: DL);
2103	} else {
2104	MIRBuilder.buildDirectDbgValue(Reg: *PhysReg, Variable: Var, Expr);
2105	}
2106
2107	return true;
2108	}
2109
2110	static unsigned getConvOpcode(Intrinsic::ID ID) {
2111	switch (ID) {
2112	default:
2113	llvm_unreachable("Unexpected intrinsic");
2114	case Intrinsic::experimental_convergence_anchor:
2115	return TargetOpcode::CONVERGENCECTRL_ANCHOR;
2116	case Intrinsic::experimental_convergence_entry:
2117	return TargetOpcode::CONVERGENCECTRL_ENTRY;
2118	case Intrinsic::experimental_convergence_loop:
2119	return TargetOpcode::CONVERGENCECTRL_LOOP;
2120	}
2121	}
2122
2123	bool IRTranslator::translateConvergenceControlIntrinsic(
2124	const CallInst &CI, Intrinsic::ID ID, MachineIRBuilder &MIRBuilder) {
2125	MachineInstrBuilder MIB = MIRBuilder.buildInstr(Opcode: getConvOpcode(ID));
2126	Register OutputReg = getOrCreateConvergenceTokenVReg(Token: CI);
2127	MIB.addDef(RegNo: OutputReg);
2128
2129	if (ID == Intrinsic::experimental_convergence_loop) {
2130	auto Bundle = CI.getOperandBundle(ID: LLVMContext::OB_convergencectrl);
2131	assert(Bundle && "Expected a convergence control token.");
2132	Register InputReg =
2133	getOrCreateConvergenceTokenVReg(Token: *Bundle->Inputs[0].get());
2134	MIB.addUse(RegNo: InputReg);
2135	}
2136
2137	return true;
2138	}
2139
2140	bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID,
2141	MachineIRBuilder &MIRBuilder) {
2142	if (auto *MI = dyn_cast<AnyMemIntrinsic>(Val: &CI)) {
2143	if (ORE->enabled()) {
2144	if (MemoryOpRemark::canHandle(I: MI, TLI: *LibInfo)) {
2145	MemoryOpRemark R(*ORE, "gisel-irtranslator-memsize", *DL, *LibInfo);
2146	R.visit(I: MI);
2147	}
2148	}
2149	}
2150
2151	// If this is a simple intrinsic (that is, we just need to add a def of
2152	// a vreg, and uses for each arg operand, then translate it.
2153	if (translateSimpleIntrinsic(CI, ID, MIRBuilder))
2154	return true;
2155
2156	switch (ID) {
2157	default:
2158	break;
2159	case Intrinsic::lifetime_start:
2160	case Intrinsic::lifetime_end: {
2161	// No stack colouring in O0, discard region information.
2162	if (MF->getTarget().getOptLevel() == CodeGenOptLevel::None)
2163	return true;
2164
2165	unsigned Op = ID == Intrinsic::lifetime_start ? TargetOpcode::LIFETIME_START
2166	: TargetOpcode::LIFETIME_END;
2167
2168	// Get the underlying objects for the location passed on the lifetime
2169	// marker.
2170	SmallVector<const Value *, 4> Allocas;
2171	getUnderlyingObjects(V: CI.getArgOperand(i: 1), Objects&: Allocas);
2172
2173	// Iterate over each underlying object, creating lifetime markers for each
2174	// static alloca. Quit if we find a non-static alloca.
2175	for (const Value *V : Allocas) {
2176	const AllocaInst *AI = dyn_cast<AllocaInst>(Val: V);
2177	if (!AI)
2178	continue;
2179
2180	if (!AI->isStaticAlloca())
2181	return true;
2182
2183	MIRBuilder.buildInstr(Opcode: Op).addFrameIndex(Idx: getOrCreateFrameIndex(AI: *AI));
2184	}
2185	return true;
2186	}
2187	case Intrinsic::dbg_declare: {
2188	const DbgDeclareInst &DI = cast<DbgDeclareInst>(Val: CI);
2189	assert(DI.getVariable() && "Missing variable");
2190	translateDbgDeclareRecord(Address: DI.getAddress(), HasArgList: DI.hasArgList(), Variable: DI.getVariable(),
2191	Expression: DI.getExpression(), DL: DI.getDebugLoc(), MIRBuilder);
2192	return true;
2193	}
2194	case Intrinsic::dbg_label: {
2195	const DbgLabelInst &DI = cast<DbgLabelInst>(Val: CI);
2196	assert(DI.getLabel() && "Missing label");
2197
2198	assert(DI.getLabel()->isValidLocationForIntrinsic(
2199	MIRBuilder.getDebugLoc()) &&
2200	"Expected inlined-at fields to agree");
2201
2202	MIRBuilder.buildDbgLabel(Label: DI.getLabel());
2203	return true;
2204	}
2205	case Intrinsic::vaend:
2206	// No target I know of cares about va_end. Certainly no in-tree target
2207	// does. Simplest intrinsic ever!
2208	return true;
2209	case Intrinsic::vastart: {
2210	Value *Ptr = CI.getArgOperand(i: 0);
2211	unsigned ListSize = TLI->getVaListSizeInBits(DL: *DL) / 8;
2212	Align Alignment = getKnownAlignment(V: Ptr, DL: *DL);
2213
2214	MIRBuilder.buildInstr(Opc: TargetOpcode::G_VASTART, DstOps: {}, SrcOps: {getOrCreateVReg(Val: *Ptr)})
2215	.addMemOperand(MMO: MF->getMachineMemOperand(PtrInfo: MachinePointerInfo(Ptr),
2216	F: MachineMemOperand::MOStore,
2217	Size: ListSize, BaseAlignment: Alignment));
2218	return true;
2219	}
2220	case Intrinsic::dbg_assign:
2221	// A dbg.assign is a dbg.value with more information about stack locations,
2222	// typically produced during optimisation of variables with leaked
2223	// addresses. We can treat it like a normal dbg_value intrinsic here; to
2224	// benefit from the full analysis of stack/SSA locations, GlobalISel would
2225	// need to register for and use the AssignmentTrackingAnalysis pass.
2226	LLVM_FALLTHROUGH;
2227	case Intrinsic::dbg_value: {
2228	// This form of DBG_VALUE is target-independent.
2229	const DbgValueInst &DI = cast<DbgValueInst>(Val: CI);
2230	translateDbgValueRecord(V: DI.getValue(), HasArgList: DI.hasArgList(), Variable: DI.getVariable(),
2231	Expression: DI.getExpression(), DL: DI.getDebugLoc(), MIRBuilder);
2232	return true;
2233	}
2234	case Intrinsic::uadd_with_overflow:
2235	return translateOverflowIntrinsic(CI, Op: TargetOpcode::G_UADDO, MIRBuilder);
2236	case Intrinsic::sadd_with_overflow:
2237	return translateOverflowIntrinsic(CI, Op: TargetOpcode::G_SADDO, MIRBuilder);
2238	case Intrinsic::usub_with_overflow:
2239	return translateOverflowIntrinsic(CI, Op: TargetOpcode::G_USUBO, MIRBuilder);
2240	case Intrinsic::ssub_with_overflow:
2241	return translateOverflowIntrinsic(CI, Op: TargetOpcode::G_SSUBO, MIRBuilder);
2242	case Intrinsic::umul_with_overflow:
2243	return translateOverflowIntrinsic(CI, Op: TargetOpcode::G_UMULO, MIRBuilder);
2244	case Intrinsic::smul_with_overflow:
2245	return translateOverflowIntrinsic(CI, Op: TargetOpcode::G_SMULO, MIRBuilder);
2246	case Intrinsic::uadd_sat:
2247	return translateBinaryOp(Opcode: TargetOpcode::G_UADDSAT, U: CI, MIRBuilder);
2248	case Intrinsic::sadd_sat:
2249	return translateBinaryOp(Opcode: TargetOpcode::G_SADDSAT, U: CI, MIRBuilder);
2250	case Intrinsic::usub_sat:
2251	return translateBinaryOp(Opcode: TargetOpcode::G_USUBSAT, U: CI, MIRBuilder);
2252	case Intrinsic::ssub_sat:
2253	return translateBinaryOp(Opcode: TargetOpcode::G_SSUBSAT, U: CI, MIRBuilder);
2254	case Intrinsic::ushl_sat:
2255	return translateBinaryOp(Opcode: TargetOpcode::G_USHLSAT, U: CI, MIRBuilder);
2256	case Intrinsic::sshl_sat:
2257	return translateBinaryOp(Opcode: TargetOpcode::G_SSHLSAT, U: CI, MIRBuilder);
2258	case Intrinsic::umin:
2259	return translateBinaryOp(Opcode: TargetOpcode::G_UMIN, U: CI, MIRBuilder);
2260	case Intrinsic::umax:
2261	return translateBinaryOp(Opcode: TargetOpcode::G_UMAX, U: CI, MIRBuilder);
2262	case Intrinsic::smin:
2263	return translateBinaryOp(Opcode: TargetOpcode::G_SMIN, U: CI, MIRBuilder);
2264	case Intrinsic::smax:
2265	return translateBinaryOp(Opcode: TargetOpcode::G_SMAX, U: CI, MIRBuilder);
2266	case Intrinsic::abs:
2267	// TODO: Preserve "int min is poison" arg in GMIR?
2268	return translateUnaryOp(Opcode: TargetOpcode::G_ABS, U: CI, MIRBuilder);
2269	case Intrinsic::smul_fix:
2270	return translateFixedPointIntrinsic(Op: TargetOpcode::G_SMULFIX, CI, MIRBuilder);
2271	case Intrinsic::umul_fix:
2272	return translateFixedPointIntrinsic(Op: TargetOpcode::G_UMULFIX, CI, MIRBuilder);
2273	case Intrinsic::smul_fix_sat:
2274	return translateFixedPointIntrinsic(Op: TargetOpcode::G_SMULFIXSAT, CI, MIRBuilder);
2275	case Intrinsic::umul_fix_sat:
2276	return translateFixedPointIntrinsic(Op: TargetOpcode::G_UMULFIXSAT, CI, MIRBuilder);
2277	case Intrinsic::sdiv_fix:
2278	return translateFixedPointIntrinsic(Op: TargetOpcode::G_SDIVFIX, CI, MIRBuilder);
2279	case Intrinsic::udiv_fix:
2280	return translateFixedPointIntrinsic(Op: TargetOpcode::G_UDIVFIX, CI, MIRBuilder);
2281	case Intrinsic::sdiv_fix_sat:
2282	return translateFixedPointIntrinsic(Op: TargetOpcode::G_SDIVFIXSAT, CI, MIRBuilder);
2283	case Intrinsic::udiv_fix_sat:
2284	return translateFixedPointIntrinsic(Op: TargetOpcode::G_UDIVFIXSAT, CI, MIRBuilder);
2285	case Intrinsic::fmuladd: {
2286	const TargetMachine &TM = MF->getTarget();
2287	Register Dst = getOrCreateVReg(Val: CI);
2288	Register Op0 = getOrCreateVReg(Val: *CI.getArgOperand(i: 0));
2289	Register Op1 = getOrCreateVReg(Val: *CI.getArgOperand(i: 1));
2290	Register Op2 = getOrCreateVReg(Val: *CI.getArgOperand(i: 2));
2291	if (TM.Options.AllowFPOpFusion != FPOpFusion::Strict &&
2292	TLI->isFMAFasterThanFMulAndFAdd(MF: *MF,
2293	TLI->getValueType(DL: *DL, Ty: CI.getType()))) {
2294	// TODO: Revisit this to see if we should move this part of the
2295	// lowering to the combiner.
2296	MIRBuilder.buildFMA(Dst, Src0: Op0, Src1: Op1, Src2: Op2,
2297	Flags: MachineInstr::copyFlagsFromInstruction(I: CI));
2298	} else {
2299	LLT Ty = getLLTForType(Ty&: *CI.getType(), DL: *DL);
2300	auto FMul = MIRBuilder.buildFMul(
2301	Dst: Ty, Src0: Op0, Src1: Op1, Flags: MachineInstr::copyFlagsFromInstruction(I: CI));
2302	MIRBuilder.buildFAdd(Dst, Src0: FMul, Src1: Op2,
2303	Flags: MachineInstr::copyFlagsFromInstruction(I: CI));
2304	}
2305	return true;
2306	}
2307	case Intrinsic::convert_from_fp16:
2308	// FIXME: This intrinsic should probably be removed from the IR.
2309	MIRBuilder.buildFPExt(Res: getOrCreateVReg(Val: CI),
2310	Op: getOrCreateVReg(Val: *CI.getArgOperand(i: 0)),
2311	Flags: MachineInstr::copyFlagsFromInstruction(I: CI));
2312	return true;
2313	case Intrinsic::convert_to_fp16:
2314	// FIXME: This intrinsic should probably be removed from the IR.
2315	MIRBuilder.buildFPTrunc(Res: getOrCreateVReg(Val: CI),
2316	Op: getOrCreateVReg(Val: *CI.getArgOperand(i: 0)),
2317	Flags: MachineInstr::copyFlagsFromInstruction(I: CI));
2318	return true;
2319	case Intrinsic::frexp: {
2320	ArrayRef<Register> VRegs = getOrCreateVRegs(Val: CI);
2321	MIRBuilder.buildFFrexp(Fract: VRegs[0], Exp: VRegs[1],
2322	Src: getOrCreateVReg(Val: *CI.getArgOperand(i: 0)),
2323	Flags: MachineInstr::copyFlagsFromInstruction(I: CI));
2324	return true;
2325	}
2326	case Intrinsic::memcpy_inline:
2327	return translateMemFunc(CI, MIRBuilder, Opcode: TargetOpcode::G_MEMCPY_INLINE);
2328	case Intrinsic::memcpy:
2329	return translateMemFunc(CI, MIRBuilder, Opcode: TargetOpcode::G_MEMCPY);
2330	case Intrinsic::memmove:
2331	return translateMemFunc(CI, MIRBuilder, Opcode: TargetOpcode::G_MEMMOVE);
2332	case Intrinsic::memset:
2333	return translateMemFunc(CI, MIRBuilder, Opcode: TargetOpcode::G_MEMSET);
2334	case Intrinsic::eh_typeid_for: {
2335	GlobalValue *GV = ExtractTypeInfo(V: CI.getArgOperand(i: 0));
2336	Register Reg = getOrCreateVReg(Val: CI);
2337	unsigned TypeID = MF->getTypeIDFor(TI: GV);
2338	MIRBuilder.buildConstant(Res: Reg, Val: TypeID);
2339	return true;
2340	}
2341	case Intrinsic::objectsize:
2342	llvm_unreachable("llvm.objectsize.* should have been lowered already");
2343
2344	case Intrinsic::is_constant:
2345	llvm_unreachable("llvm.is.constant.* should have been lowered already");
2346
2347	case Intrinsic::stackguard:
2348	getStackGuard(DstReg: getOrCreateVReg(Val: CI), MIRBuilder);
2349	return true;
2350	case Intrinsic::stackprotector: {
2351	LLT PtrTy = getLLTForType(Ty&: *CI.getArgOperand(i: 0)->getType(), DL: *DL);
2352	Register GuardVal;
2353	if (TLI->useLoadStackGuardNode()) {
2354	GuardVal = MRI->createGenericVirtualRegister(Ty: PtrTy);
2355	getStackGuard(DstReg: GuardVal, MIRBuilder);
2356	} else
2357	GuardVal = getOrCreateVReg(Val: *CI.getArgOperand(i: 0)); // The guard's value.
2358
2359	AllocaInst *Slot = cast<AllocaInst>(Val: CI.getArgOperand(i: 1));
2360	int FI = getOrCreateFrameIndex(AI: *Slot);
2361	MF->getFrameInfo().setStackProtectorIndex(FI);
2362
2363	MIRBuilder.buildStore(
2364	Val: GuardVal, Addr: getOrCreateVReg(Val: *Slot),
2365	MMO&: *MF->getMachineMemOperand(PtrInfo: MachinePointerInfo::getFixedStack(MF&: *MF, FI),
2366	f: MachineMemOperand::MOStore |
2367	MachineMemOperand::MOVolatile,
2368	MemTy: PtrTy, base_alignment: Align(8)));
2369	return true;
2370	}
2371	case Intrinsic::stacksave: {
2372	MIRBuilder.buildInstr(Opc: TargetOpcode::G_STACKSAVE, DstOps: {getOrCreateVReg(Val: CI)}, SrcOps: {});
2373	return true;
2374	}
2375	case Intrinsic::stackrestore: {
2376	MIRBuilder.buildInstr(Opc: TargetOpcode::G_STACKRESTORE, DstOps: {},
2377	SrcOps: {getOrCreateVReg(Val: *CI.getArgOperand(i: 0))});
2378	return true;
2379	}
2380	case Intrinsic::cttz:
2381	case Intrinsic::ctlz: {
2382	ConstantInt *Cst = cast<ConstantInt>(Val: CI.getArgOperand(i: 1));
2383	bool isTrailing = ID == Intrinsic::cttz;
2384	unsigned Opcode = isTrailing
2385	? Cst->isZero() ? TargetOpcode::G_CTTZ
2386	: TargetOpcode::G_CTTZ_ZERO_UNDEF
2387	: Cst->isZero() ? TargetOpcode::G_CTLZ
2388	: TargetOpcode::G_CTLZ_ZERO_UNDEF;
2389	MIRBuilder.buildInstr(Opc: Opcode, DstOps: {getOrCreateVReg(Val: CI)},
2390	SrcOps: {getOrCreateVReg(Val: *CI.getArgOperand(i: 0))});
2391	return true;
2392	}
2393	case Intrinsic::invariant_start: {
2394	LLT PtrTy = getLLTForType(Ty&: *CI.getArgOperand(i: 0)->getType(), DL: *DL);
2395	Register Undef = MRI->createGenericVirtualRegister(Ty: PtrTy);
2396	MIRBuilder.buildUndef(Res: Undef);
2397	return true;
2398	}
2399	case Intrinsic::invariant_end:
2400	return true;
2401	case Intrinsic::expect:
2402	case Intrinsic::annotation:
2403	case Intrinsic::ptr_annotation:
2404	case Intrinsic::launder_invariant_group:
2405	case Intrinsic::strip_invariant_group: {
2406	// Drop the intrinsic, but forward the value.
2407	MIRBuilder.buildCopy(Res: getOrCreateVReg(Val: CI),
2408	Op: getOrCreateVReg(Val: *CI.getArgOperand(i: 0)));
2409	return true;
2410	}
2411	case Intrinsic::assume:
2412	case Intrinsic::experimental_noalias_scope_decl:
2413	case Intrinsic::var_annotation:
2414	case Intrinsic::sideeffect:
2415	// Discard annotate attributes, assumptions, and artificial side-effects.
2416	return true;
2417	case Intrinsic::read_volatile_register:
2418	case Intrinsic::read_register: {
2419	Value *Arg = CI.getArgOperand(i: 0);
2420	MIRBuilder
2421	.buildInstr(Opc: TargetOpcode::G_READ_REGISTER, DstOps: {getOrCreateVReg(Val: CI)}, SrcOps: {})
2422	.addMetadata(MD: cast<MDNode>(Val: cast<MetadataAsValue>(Val: Arg)->getMetadata()));
2423	return true;
2424	}
2425	case Intrinsic::write_register: {
2426	Value *Arg = CI.getArgOperand(i: 0);
2427	MIRBuilder.buildInstr(Opcode: TargetOpcode::G_WRITE_REGISTER)
2428	.addMetadata(MD: cast<MDNode>(Val: cast<MetadataAsValue>(Val: Arg)->getMetadata()))
2429	.addUse(RegNo: getOrCreateVReg(Val: *CI.getArgOperand(i: 1)));
2430	return true;
2431	}
2432	case Intrinsic::localescape: {
2433	MachineBasicBlock &EntryMBB = MF->front();
2434	StringRef EscapedName = GlobalValue::dropLLVMManglingEscape(Name: MF->getName());
2435
2436	// Directly emit some LOCAL_ESCAPE machine instrs. Label assignment emission
2437	// is the same on all targets.
2438	for (unsigned Idx = 0, E = CI.arg_size(); Idx < E; ++Idx) {
2439	Value *Arg = CI.getArgOperand(i: Idx)->stripPointerCasts();
2440	if (isa<ConstantPointerNull>(Val: Arg))
2441	continue; // Skip null pointers. They represent a hole in index space.
2442
2443	int FI = getOrCreateFrameIndex(AI: *cast<AllocaInst>(Val: Arg));
2444	MCSymbol *FrameAllocSym =
2445	MF->getMMI().getContext().getOrCreateFrameAllocSymbol(FuncName: EscapedName,
2446	Idx);
2447
2448	// This should be inserted at the start of the entry block.
2449	auto LocalEscape =
2450	MIRBuilder.buildInstrNoInsert(Opcode: TargetOpcode::LOCAL_ESCAPE)
2451	.addSym(Sym: FrameAllocSym)
2452	.addFrameIndex(Idx: FI);
2453
2454	EntryMBB.insert(I: EntryMBB.begin(), MI: LocalEscape);
2455	}
2456
2457	return true;
2458	}
2459	case Intrinsic::vector_reduce_fadd:
2460	case Intrinsic::vector_reduce_fmul: {
2461	// Need to check for the reassoc flag to decide whether we want a
2462	// sequential reduction opcode or not.
2463	Register Dst = getOrCreateVReg(Val: CI);
2464	Register ScalarSrc = getOrCreateVReg(Val: *CI.getArgOperand(i: 0));
2465	Register VecSrc = getOrCreateVReg(Val: *CI.getArgOperand(i: 1));
2466	unsigned Opc = 0;
2467	if (!CI.hasAllowReassoc()) {
2468	// The sequential ordering case.
2469	Opc = ID == Intrinsic::vector_reduce_fadd
2470	? TargetOpcode::G_VECREDUCE_SEQ_FADD
2471	: TargetOpcode::G_VECREDUCE_SEQ_FMUL;
2472	MIRBuilder.buildInstr(Opc, DstOps: {Dst}, SrcOps: {ScalarSrc, VecSrc},
2473	Flags: MachineInstr::copyFlagsFromInstruction(I: CI));
2474	return true;
2475	}
2476	// We split the operation into a separate G_FADD/G_FMUL + the reduce,
2477	// since the associativity doesn't matter.
2478	unsigned ScalarOpc;
2479	if (ID == Intrinsic::vector_reduce_fadd) {
2480	Opc = TargetOpcode::G_VECREDUCE_FADD;
2481	ScalarOpc = TargetOpcode::G_FADD;
2482	} else {
2483	Opc = TargetOpcode::G_VECREDUCE_FMUL;
2484	ScalarOpc = TargetOpcode::G_FMUL;
2485	}
2486	LLT DstTy = MRI->getType(Reg: Dst);
2487	auto Rdx = MIRBuilder.buildInstr(
2488	Opc, DstOps: {DstTy}, SrcOps: {VecSrc}, Flags: MachineInstr::copyFlagsFromInstruction(I: CI));
2489	MIRBuilder.buildInstr(Opc: ScalarOpc, DstOps: {Dst}, SrcOps: {ScalarSrc, Rdx},
2490	Flags: MachineInstr::copyFlagsFromInstruction(I: CI));
2491
2492	return true;
2493	}
2494	case Intrinsic::trap:
2495	return translateTrap(CI, MIRBuilder, Opcode: TargetOpcode::G_TRAP);
2496	case Intrinsic::debugtrap:
2497	return translateTrap(CI, MIRBuilder, Opcode: TargetOpcode::G_DEBUGTRAP);
2498	case Intrinsic::ubsantrap:
2499	return translateTrap(CI, MIRBuilder, Opcode: TargetOpcode::G_UBSANTRAP);
2500	case Intrinsic::allow_runtime_check:
2501	case Intrinsic::allow_ubsan_check:
2502	MIRBuilder.buildCopy(Res: getOrCreateVReg(Val: CI),
2503	Op: getOrCreateVReg(Val: *ConstantInt::getTrue(Ty: CI.getType())));
2504	return true;
2505	case Intrinsic::amdgcn_cs_chain:
2506	return translateCallBase(CB: CI, MIRBuilder);
2507	case Intrinsic::fptrunc_round: {
2508	uint32_t Flags = MachineInstr::copyFlagsFromInstruction(I: CI);
2509
2510	// Convert the metadata argument to a constant integer
2511	Metadata *MD = cast<MetadataAsValue>(Val: CI.getArgOperand(i: 1))->getMetadata();
2512	std::optional<RoundingMode> RoundMode =
2513	convertStrToRoundingMode(cast<MDString>(Val: MD)->getString());
2514
2515	// Add the Rounding mode as an integer
2516	MIRBuilder
2517	.buildInstr(Opc: TargetOpcode::G_INTRINSIC_FPTRUNC_ROUND,
2518	DstOps: {getOrCreateVReg(Val: CI)},
2519	SrcOps: {getOrCreateVReg(Val: *CI.getArgOperand(i: 0))}, Flags)
2520	.addImm(Val: (int)*RoundMode);
2521
2522	return true;
2523	}
2524	case Intrinsic::is_fpclass: {
2525	Value *FpValue = CI.getOperand(i_nocapture: 0);
2526	ConstantInt *TestMaskValue = cast<ConstantInt>(Val: CI.getOperand(i_nocapture: 1));
2527
2528	MIRBuilder
2529	.buildInstr(Opc: TargetOpcode::G_IS_FPCLASS, DstOps: {getOrCreateVReg(Val: CI)},
2530	SrcOps: {getOrCreateVReg(Val: *FpValue)})
2531	.addImm(Val: TestMaskValue->getZExtValue());
2532
2533	return true;
2534	}
2535	case Intrinsic::set_fpenv: {
2536	Value *FPEnv = CI.getOperand(i_nocapture: 0);
2537	MIRBuilder.buildInstr(Opc: TargetOpcode::G_SET_FPENV, DstOps: {},
2538	SrcOps: {getOrCreateVReg(Val: *FPEnv)});
2539	return true;
2540	}
2541	case Intrinsic::reset_fpenv: {
2542	MIRBuilder.buildInstr(Opc: TargetOpcode::G_RESET_FPENV, DstOps: {}, SrcOps: {});
2543	return true;
2544	}
2545	case Intrinsic::set_fpmode: {
2546	Value *FPState = CI.getOperand(i_nocapture: 0);
2547	MIRBuilder.buildInstr(Opc: TargetOpcode::G_SET_FPMODE, DstOps: {},
2548	SrcOps: { getOrCreateVReg(Val: *FPState) });
2549	return true;
2550	}
2551	case Intrinsic::reset_fpmode: {
2552	MIRBuilder.buildInstr(Opc: TargetOpcode::G_RESET_FPMODE, DstOps: {}, SrcOps: {});
2553	return true;
2554	}
2555	case Intrinsic::vscale: {
2556	MIRBuilder.buildVScale(Res: getOrCreateVReg(Val: CI), MinElts: 1);
2557	return true;
2558	}
2559	case Intrinsic::prefetch: {
2560	Value *Addr = CI.getOperand(i_nocapture: 0);
2561	unsigned RW = cast<ConstantInt>(Val: CI.getOperand(i_nocapture: 1))->getZExtValue();
2562	unsigned Locality = cast<ConstantInt>(Val: CI.getOperand(i_nocapture: 2))->getZExtValue();
2563	unsigned CacheType = cast<ConstantInt>(Val: CI.getOperand(i_nocapture: 3))->getZExtValue();
2564
2565	auto Flags = RW ? MachineMemOperand::MOStore : MachineMemOperand::MOLoad;
2566	auto &MMO = *MF->getMachineMemOperand(PtrInfo: MachinePointerInfo(Addr), f: Flags,
2567	MemTy: LLT(), base_alignment: Align());
2568
2569	MIRBuilder.buildPrefetch(Addr: getOrCreateVReg(Val: *Addr), RW, Locality, CacheType,
2570	MMO);
2571
2572	return true;
2573	}
2574
2575	case Intrinsic::experimental_vector_interleave2:
2576	case Intrinsic::experimental_vector_deinterleave2: {
2577	// Both intrinsics have at least one operand.
2578	Value *Op0 = CI.getOperand(i_nocapture: 0);
2579	LLT ResTy = getLLTForType(Ty&: *Op0->getType(), DL: MIRBuilder.getDataLayout());
2580	if (!ResTy.isFixedVector())
2581	return false;
2582
2583	if (CI.getIntrinsicID() == Intrinsic::experimental_vector_interleave2)
2584	return translateVectorInterleave2Intrinsic(CI, MIRBuilder);
2585
2586	return translateVectorDeinterleave2Intrinsic(CI, MIRBuilder);
2587	}
2588
2589	#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \
2590	case Intrinsic::INTRINSIC:
2591	#include "llvm/IR/ConstrainedOps.def"
2592	return translateConstrainedFPIntrinsic(FPI: cast<ConstrainedFPIntrinsic>(Val: CI),
2593	MIRBuilder);
2594	case Intrinsic::experimental_convergence_anchor:
2595	case Intrinsic::experimental_convergence_entry:
2596	case Intrinsic::experimental_convergence_loop:
2597	return translateConvergenceControlIntrinsic(CI, ID, MIRBuilder);
2598	}
2599	return false;
2600	}
2601
2602	bool IRTranslator::translateInlineAsm(const CallBase &CB,
2603	MachineIRBuilder &MIRBuilder) {
2604
2605	const InlineAsmLowering *ALI = MF->getSubtarget().getInlineAsmLowering();
2606
2607	if (!ALI) {
2608	LLVM_DEBUG(
2609	dbgs() << "Inline asm lowering is not supported for this target yet\n");
2610	return false;
2611	}
2612
2613	return ALI->lowerInlineAsm(
2614	MIRBuilder, CB, GetOrCreateVRegs: [&](const Value &Val) { return getOrCreateVRegs(Val); });
2615	}
2616
2617	bool IRTranslator::translateCallBase(const CallBase &CB,
2618	MachineIRBuilder &MIRBuilder) {
2619	ArrayRef<Register> Res = getOrCreateVRegs(Val: CB);
2620
2621	SmallVector<ArrayRef<Register>, 8> Args;
2622	Register SwiftInVReg = 0;
2623	Register SwiftErrorVReg = 0;
2624	for (const auto &Arg : CB.args()) {
2625	if (CLI->supportSwiftError() && isSwiftError(V: Arg)) {
2626	assert(SwiftInVReg == 0 && "Expected only one swift error argument");
2627	LLT Ty = getLLTForType(Ty&: *Arg->getType(), DL: *DL);
2628	SwiftInVReg = MRI->createGenericVirtualRegister(Ty);
2629	MIRBuilder.buildCopy(Res: SwiftInVReg, Op: SwiftError.getOrCreateVRegUseAt(
2630	&CB, &MIRBuilder.getMBB(), Arg));
2631	Args.emplace_back(Args: ArrayRef(SwiftInVReg));
2632	SwiftErrorVReg =
2633	SwiftError.getOrCreateVRegDefAt(&CB, &MIRBuilder.getMBB(), Arg);
2634	continue;
2635	}
2636	Args.push_back(Elt: getOrCreateVRegs(Val: *Arg));
2637	}
2638
2639	if (auto *CI = dyn_cast<CallInst>(Val: &CB)) {
2640	if (ORE->enabled()) {
2641	if (MemoryOpRemark::canHandle(I: CI, TLI: *LibInfo)) {
2642	MemoryOpRemark R(*ORE, "gisel-irtranslator-memsize", *DL, *LibInfo);
2643	R.visit(I: CI);
2644	}
2645	}
2646	}
2647
2648	Register ConvergenceCtrlToken = 0;
2649	if (auto Bundle = CB.getOperandBundle(ID: LLVMContext::OB_convergencectrl)) {
2650	const auto &Token = *Bundle->Inputs[0].get();
2651	ConvergenceCtrlToken = getOrCreateConvergenceTokenVReg(Token);
2652	}
2653
2654	// We don't set HasCalls on MFI here yet because call lowering may decide to
2655	// optimize into tail calls. Instead, we defer that to selection where a final
2656	// scan is done to check if any instructions are calls.
2657	bool Success = CLI->lowerCall(
2658	MIRBuilder, Call: CB, ResRegs: Res, ArgRegs: Args, SwiftErrorVReg, ConvergenceCtrlToken,
2659	GetCalleeReg: [&]() { return getOrCreateVReg(Val: *CB.getCalledOperand()); });
2660
2661	// Check if we just inserted a tail call.
2662	if (Success) {
2663	assert(!HasTailCall && "Can't tail call return twice from block?");
2664	const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo();
2665	HasTailCall = TII->isTailCall(Inst: *std::prev(x: MIRBuilder.getInsertPt()));
2666	}
2667
2668	return Success;
2669	}
2670
2671	bool IRTranslator::translateCall(const User &U, MachineIRBuilder &MIRBuilder) {
2672	const CallInst &CI = cast<CallInst>(Val: U);
2673	auto TII = MF->getTarget().getIntrinsicInfo();
2674	const Function *F = CI.getCalledFunction();
2675
2676	// FIXME: support Windows dllimport function calls and calls through
2677	// weak symbols.
2678	if (F && (F->hasDLLImportStorageClass() ||
2679	(MF->getTarget().getTargetTriple().isOSWindows() &&
2680	F->hasExternalWeakLinkage())))
2681	return false;
2682
2683	// FIXME: support control flow guard targets.
2684	if (CI.countOperandBundlesOfType(ID: LLVMContext::OB_cfguardtarget))
2685	return false;
2686
2687	// FIXME: support statepoints and related.
2688	if (isa<GCStatepointInst, GCRelocateInst, GCResultInst>(Val: U))
2689	return false;
2690
2691	if (CI.isInlineAsm())
2692	return translateInlineAsm(CB: CI, MIRBuilder);
2693
2694	diagnoseDontCall(CI);
2695
2696	Intrinsic::ID ID = Intrinsic::not_intrinsic;
2697	if (F && F->isIntrinsic()) {
2698	ID = F->getIntrinsicID();
2699	if (TII && ID == Intrinsic::not_intrinsic)
2700	ID = static_cast<Intrinsic::ID>(TII->getIntrinsicID(F));
2701	}
2702
2703	if (!F || !F->isIntrinsic() || ID == Intrinsic::not_intrinsic)
2704	return translateCallBase(CB: CI, MIRBuilder);
2705
2706	assert(ID != Intrinsic::not_intrinsic && "unknown intrinsic");
2707
2708	if (translateKnownIntrinsic(CI, ID, MIRBuilder))
2709	return true;
2710
2711	ArrayRef<Register> ResultRegs;
2712	if (!CI.getType()->isVoidTy())
2713	ResultRegs = getOrCreateVRegs(Val: CI);
2714
2715	// Ignore the callsite attributes. Backend code is most likely not expecting
2716	// an intrinsic to sometimes have side effects and sometimes not.
2717	MachineInstrBuilder MIB = MIRBuilder.buildIntrinsic(ID, Res: ResultRegs);
2718	if (isa<FPMathOperator>(Val: CI))
2719	MIB->copyIRFlags(I: CI);
2720
2721	for (const auto &Arg : enumerate(First: CI.args())) {
2722	// If this is required to be an immediate, don't materialize it in a
2723	// register.
2724	if (CI.paramHasAttr(Arg.index(), Attribute::ImmArg)) {
2725	if (ConstantInt *CI = dyn_cast<ConstantInt>(Val: Arg.value())) {
2726	// imm arguments are more convenient than cimm (and realistically
2727	// probably sufficient), so use them.
2728	assert(CI->getBitWidth() <= 64 &&
2729	"large intrinsic immediates not handled");
2730	MIB.addImm(Val: CI->getSExtValue());
2731	} else {
2732	MIB.addFPImm(Val: cast<ConstantFP>(Val: Arg.value()));
2733	}
2734	} else if (auto *MDVal = dyn_cast<MetadataAsValue>(Val: Arg.value())) {
2735	auto *MD = MDVal->getMetadata();
2736	auto *MDN = dyn_cast<MDNode>(Val: MD);
2737	if (!MDN) {
2738	if (auto *ConstMD = dyn_cast<ConstantAsMetadata>(Val: MD))
2739	MDN = MDNode::get(Context&: MF->getFunction().getContext(), MDs: ConstMD);
2740	else // This was probably an MDString.
2741	return false;
2742	}
2743	MIB.addMetadata(MD: MDN);
2744	} else {
2745	ArrayRef<Register> VRegs = getOrCreateVRegs(Val: *Arg.value());
2746	if (VRegs.size() > 1)
2747	return false;
2748	MIB.addUse(RegNo: VRegs[0]);
2749	}
2750	}
2751
2752	// Add a MachineMemOperand if it is a target mem intrinsic.
2753	TargetLowering::IntrinsicInfo Info;
2754	// TODO: Add a GlobalISel version of getTgtMemIntrinsic.
2755	if (TLI->getTgtMemIntrinsic(Info, CI, *MF, ID)) {
2756	Align Alignment = Info.align.value_or(
2757	u: DL->getABITypeAlign(Ty: Info.memVT.getTypeForEVT(Context&: F->getContext())));
2758	LLT MemTy = Info.memVT.isSimple()
2759	? getLLTForMVT(Ty: Info.memVT.getSimpleVT())
2760	: LLT::scalar(SizeInBits: Info.memVT.getStoreSizeInBits());
2761
2762	// TODO: We currently just fallback to address space 0 if getTgtMemIntrinsic
2763	// didn't yield anything useful.
2764	MachinePointerInfo MPI;
2765	if (Info.ptrVal)
2766	MPI = MachinePointerInfo(Info.ptrVal, Info.offset);
2767	else if (Info.fallbackAddressSpace)
2768	MPI = MachinePointerInfo(*Info.fallbackAddressSpace);
2769	MIB.addMemOperand(
2770	MMO: MF->getMachineMemOperand(PtrInfo: MPI, f: Info.flags, MemTy, base_alignment: Alignment, AAInfo: CI.getAAMetadata()));
2771	}
2772
2773	if (CI.isConvergent()) {
2774	if (auto Bundle = CI.getOperandBundle(ID: LLVMContext::OB_convergencectrl)) {
2775	auto *Token = Bundle->Inputs[0].get();
2776	Register TokenReg = getOrCreateVReg(Val: *Token);
2777	MIB.addUse(RegNo: TokenReg, Flags: RegState::Implicit);
2778	}
2779	}
2780
2781	return true;
2782	}
2783
2784	bool IRTranslator::findUnwindDestinations(
2785	const BasicBlock *EHPadBB,
2786	BranchProbability Prob,
2787	SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
2788	&UnwindDests) {
2789	EHPersonality Personality = classifyEHPersonality(
2790	Pers: EHPadBB->getParent()->getFunction().getPersonalityFn());
2791	bool IsMSVCCXX = Personality == EHPersonality::MSVC_CXX;
2792	bool IsCoreCLR = Personality == EHPersonality::CoreCLR;
2793	bool IsWasmCXX = Personality == EHPersonality::Wasm_CXX;
2794	bool IsSEH = isAsynchronousEHPersonality(Pers: Personality);
2795
2796	if (IsWasmCXX) {
2797	// Ignore this for now.
2798	return false;
2799	}
2800
2801	while (EHPadBB) {
2802	const Instruction *Pad = EHPadBB->getFirstNonPHI();
2803	BasicBlock *NewEHPadBB = nullptr;
2804	if (isa<LandingPadInst>(Val: Pad)) {
2805	// Stop on landingpads. They are not funclets.
2806	UnwindDests.emplace_back(Args: &getMBB(BB: *EHPadBB), Args&: Prob);
2807	break;
2808	}
2809	if (isa<CleanupPadInst>(Val: Pad)) {
2810	// Stop on cleanup pads. Cleanups are always funclet entries for all known
2811	// personalities.
2812	UnwindDests.emplace_back(Args: &getMBB(BB: *EHPadBB), Args&: Prob);
2813	UnwindDests.back().first->setIsEHScopeEntry();
2814	UnwindDests.back().first->setIsEHFuncletEntry();
2815	break;
2816	}
2817	if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Val: Pad)) {
2818	// Add the catchpad handlers to the possible destinations.
2819	for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
2820	UnwindDests.emplace_back(Args: &getMBB(BB: *CatchPadBB), Args&: Prob);
2821	// For MSVC++ and the CLR, catchblocks are funclets and need prologues.
2822	if (IsMSVCCXX || IsCoreCLR)
2823	UnwindDests.back().first->setIsEHFuncletEntry();
2824	if (!IsSEH)
2825	UnwindDests.back().first->setIsEHScopeEntry();
2826	}
2827	NewEHPadBB = CatchSwitch->getUnwindDest();
2828	} else {
2829	continue;
2830	}
2831
2832	BranchProbabilityInfo *BPI = FuncInfo.BPI;
2833	if (BPI && NewEHPadBB)
2834	Prob *= BPI->getEdgeProbability(Src: EHPadBB, Dst: NewEHPadBB);
2835	EHPadBB = NewEHPadBB;
2836	}
2837	return true;
2838	}
2839
2840	bool IRTranslator::translateInvoke(const User &U,
2841	MachineIRBuilder &MIRBuilder) {
2842	const InvokeInst &I = cast<InvokeInst>(Val: U);
2843	MCContext &Context = MF->getContext();
2844
2845	const BasicBlock *ReturnBB = I.getSuccessor(i: 0);
2846	const BasicBlock *EHPadBB = I.getSuccessor(i: 1);
2847
2848	const Function *Fn = I.getCalledFunction();
2849
2850	// FIXME: support invoking patchpoint and statepoint intrinsics.
2851	if (Fn && Fn->isIntrinsic())
2852	return false;
2853
2854	// FIXME: support whatever these are.
2855	if (I.countOperandBundlesOfType(ID: LLVMContext::OB_deopt))
2856	return false;
2857
2858	// FIXME: support control flow guard targets.
2859	if (I.countOperandBundlesOfType(ID: LLVMContext::OB_cfguardtarget))
2860	return false;
2861
2862	// FIXME: support Windows exception handling.
2863	if (!isa<LandingPadInst>(Val: EHPadBB->getFirstNonPHI()))
2864	return false;
2865
2866	// FIXME: support Windows dllimport function calls and calls through
2867	// weak symbols.
2868	if (Fn && (Fn->hasDLLImportStorageClass() ||
2869	(MF->getTarget().getTargetTriple().isOSWindows() &&
2870	Fn->hasExternalWeakLinkage())))
2871	return false;
2872
2873	bool LowerInlineAsm = I.isInlineAsm();
2874	bool NeedEHLabel = true;
2875
2876	// Emit the actual call, bracketed by EH_LABELs so that the MF knows about
2877	// the region covered by the try.
2878	MCSymbol *BeginSymbol = nullptr;
2879	if (NeedEHLabel) {
2880	MIRBuilder.buildInstr(Opcode: TargetOpcode::G_INVOKE_REGION_START);
2881	BeginSymbol = Context.createTempSymbol();
2882	MIRBuilder.buildInstr(Opcode: TargetOpcode::EH_LABEL).addSym(Sym: BeginSymbol);
2883	}
2884
2885	if (LowerInlineAsm) {
2886	if (!translateInlineAsm(CB: I, MIRBuilder))
2887	return false;
2888	} else if (!translateCallBase(CB: I, MIRBuilder))
2889	return false;
2890
2891	MCSymbol *EndSymbol = nullptr;
2892	if (NeedEHLabel) {
2893	EndSymbol = Context.createTempSymbol();
2894	MIRBuilder.buildInstr(Opcode: TargetOpcode::EH_LABEL).addSym(Sym: EndSymbol);
2895	}
2896
2897	SmallVector<std::pair<MachineBasicBlock *, BranchProbability>, 1> UnwindDests;
2898	BranchProbabilityInfo *BPI = FuncInfo.BPI;
2899	MachineBasicBlock *InvokeMBB = &MIRBuilder.getMBB();
2900	BranchProbability EHPadBBProb =
2901	BPI ? BPI->getEdgeProbability(Src: InvokeMBB->getBasicBlock(), Dst: EHPadBB)
2902	: BranchProbability::getZero();
2903
2904	if (!findUnwindDestinations(EHPadBB, Prob: EHPadBBProb, UnwindDests))
2905	return false;
2906
2907	MachineBasicBlock &EHPadMBB = getMBB(BB: *EHPadBB),
2908	&ReturnMBB = getMBB(BB: *ReturnBB);
2909	// Update successor info.
2910	addSuccessorWithProb(Src: InvokeMBB, Dst: &ReturnMBB);
2911	for (auto &UnwindDest : UnwindDests) {
2912	UnwindDest.first->setIsEHPad();
2913	addSuccessorWithProb(Src: InvokeMBB, Dst: UnwindDest.first, Prob: UnwindDest.second);
2914	}
2915	InvokeMBB->normalizeSuccProbs();
2916
2917	if (NeedEHLabel) {
2918	assert(BeginSymbol && "Expected a begin symbol!");
2919	assert(EndSymbol && "Expected an end symbol!");
2920	MF->addInvoke(LandingPad: &EHPadMBB, BeginLabel: BeginSymbol, EndLabel: EndSymbol);
2921	}
2922
2923	MIRBuilder.buildBr(Dest&: ReturnMBB);
2924	return true;
2925	}
2926
2927	bool IRTranslator::translateCallBr(const User &U,
2928	MachineIRBuilder &MIRBuilder) {
2929	// FIXME: Implement this.
2930	return false;
2931	}
2932
2933	bool IRTranslator::translateLandingPad(const User &U,
2934	MachineIRBuilder &MIRBuilder) {
2935	const LandingPadInst &LP = cast<LandingPadInst>(Val: U);
2936
2937	MachineBasicBlock &MBB = MIRBuilder.getMBB();
2938
2939	MBB.setIsEHPad();
2940
2941	// If there aren't registers to copy the values into (e.g., during SjLj
2942	// exceptions), then don't bother.
2943	const Constant *PersonalityFn = MF->getFunction().getPersonalityFn();
2944	if (TLI->getExceptionPointerRegister(PersonalityFn) == 0 &&
2945	TLI->getExceptionSelectorRegister(PersonalityFn) == 0)
2946	return true;
2947
2948	// If landingpad's return type is token type, we don't create DAG nodes
2949	// for its exception pointer and selector value. The extraction of exception
2950	// pointer or selector value from token type landingpads is not currently
2951	// supported.
2952	if (LP.getType()->isTokenTy())
2953	return true;
2954
2955	// Add a label to mark the beginning of the landing pad. Deletion of the
2956	// landing pad can thus be detected via the MachineModuleInfo.
2957	MIRBuilder.buildInstr(Opcode: TargetOpcode::EH_LABEL)
2958	.addSym(Sym: MF->addLandingPad(LandingPad: &MBB));
2959
2960	// If the unwinder does not preserve all registers, ensure that the
2961	// function marks the clobbered registers as used.
2962	const TargetRegisterInfo &TRI = *MF->getSubtarget().getRegisterInfo();
2963	if (auto *RegMask = TRI.getCustomEHPadPreservedMask(MF: *MF))
2964	MF->getRegInfo().addPhysRegsUsedFromRegMask(RegMask);
2965
2966	LLT Ty = getLLTForType(Ty&: *LP.getType(), DL: *DL);
2967	Register Undef = MRI->createGenericVirtualRegister(Ty);
2968	MIRBuilder.buildUndef(Res: Undef);
2969
2970	SmallVector<LLT, 2> Tys;
2971	for (Type *Ty : cast<StructType>(Val: LP.getType())->elements())
2972	Tys.push_back(Elt: getLLTForType(Ty&: *Ty, DL: *DL));
2973	assert(Tys.size() == 2 && "Only two-valued landingpads are supported");
2974
2975	// Mark exception register as live in.
2976	Register ExceptionReg = TLI->getExceptionPointerRegister(PersonalityFn);
2977	if (!ExceptionReg)
2978	return false;
2979
2980	MBB.addLiveIn(PhysReg: ExceptionReg);
2981	ArrayRef<Register> ResRegs = getOrCreateVRegs(Val: LP);
2982	MIRBuilder.buildCopy(Res: ResRegs[0], Op: ExceptionReg);
2983
2984	Register SelectorReg = TLI->getExceptionSelectorRegister(PersonalityFn);
2985	if (!SelectorReg)
2986	return false;
2987
2988	MBB.addLiveIn(PhysReg: SelectorReg);
2989	Register PtrVReg = MRI->createGenericVirtualRegister(Ty: Tys[0]);
2990	MIRBuilder.buildCopy(Res: PtrVReg, Op: SelectorReg);
2991	MIRBuilder.buildCast(Dst: ResRegs[1], Src: PtrVReg);
2992
2993	return true;
2994	}
2995
2996	bool IRTranslator::translateAlloca(const User &U,
2997	MachineIRBuilder &MIRBuilder) {
2998	auto &AI = cast<AllocaInst>(Val: U);
2999
3000	if (AI.isSwiftError())
3001	return true;
3002
3003	if (AI.isStaticAlloca()) {
3004	Register Res = getOrCreateVReg(Val: AI);
3005	int FI = getOrCreateFrameIndex(AI);
3006	MIRBuilder.buildFrameIndex(Res, Idx: FI);
3007	return true;
3008	}
3009
3010	// FIXME: support stack probing for Windows.
3011	if (MF->getTarget().getTargetTriple().isOSWindows())
3012	return false;
3013
3014	// Now we're in the harder dynamic case.
3015	Register NumElts = getOrCreateVReg(Val: *AI.getArraySize());
3016	Type *IntPtrIRTy = DL->getIntPtrType(AI.getType());
3017	LLT IntPtrTy = getLLTForType(Ty&: *IntPtrIRTy, DL: *DL);
3018	if (MRI->getType(Reg: NumElts) != IntPtrTy) {
3019	Register ExtElts = MRI->createGenericVirtualRegister(Ty: IntPtrTy);
3020	MIRBuilder.buildZExtOrTrunc(Res: ExtElts, Op: NumElts);
3021	NumElts = ExtElts;
3022	}
3023
3024	Type *Ty = AI.getAllocatedType();
3025
3026	Register AllocSize = MRI->createGenericVirtualRegister(Ty: IntPtrTy);
3027	Register TySize =
3028	getOrCreateVReg(Val: *ConstantInt::get(Ty: IntPtrIRTy, V: DL->getTypeAllocSize(Ty)));
3029	MIRBuilder.buildMul(Dst: AllocSize, Src0: NumElts, Src1: TySize);
3030
3031	// Round the size of the allocation up to the stack alignment size
3032	// by add SA-1 to the size. This doesn't overflow because we're computing
3033	// an address inside an alloca.
3034	Align StackAlign = MF->getSubtarget().getFrameLowering()->getStackAlign();
3035	auto SAMinusOne = MIRBuilder.buildConstant(Res: IntPtrTy, Val: StackAlign.value() - 1);
3036	auto AllocAdd = MIRBuilder.buildAdd(Dst: IntPtrTy, Src0: AllocSize, Src1: SAMinusOne,
3037	Flags: MachineInstr::NoUWrap);
3038	auto AlignCst =
3039	MIRBuilder.buildConstant(Res: IntPtrTy, Val: ~(uint64_t)(StackAlign.value() - 1));
3040	auto AlignedAlloc = MIRBuilder.buildAnd(Dst: IntPtrTy, Src0: AllocAdd, Src1: AlignCst);
3041
3042	Align Alignment = std::max(a: AI.getAlign(), b: DL->getPrefTypeAlign(Ty));
3043	if (Alignment <= StackAlign)
3044	Alignment = Align(1);
3045	MIRBuilder.buildDynStackAlloc(Res: getOrCreateVReg(Val: AI), Size: AlignedAlloc, Alignment);
3046
3047	MF->getFrameInfo().CreateVariableSizedObject(Alignment, Alloca: &AI);
3048	assert(MF->getFrameInfo().hasVarSizedObjects());
3049	return true;
3050	}
3051
3052	bool IRTranslator::translateVAArg(const User &U, MachineIRBuilder &MIRBuilder) {
3053	// FIXME: We may need more info about the type. Because of how LLT works,
3054	// we're completely discarding the i64/double distinction here (amongst
3055	// others). Fortunately the ABIs I know of where that matters don't use va_arg
3056	// anyway but that's not guaranteed.
3057	MIRBuilder.buildInstr(Opc: TargetOpcode::G_VAARG, DstOps: {getOrCreateVReg(Val: U)},
3058	SrcOps: {getOrCreateVReg(Val: *U.getOperand(i: 0)),
3059	DL->getABITypeAlign(Ty: U.getType()).value()});
3060	return true;
3061	}
3062
3063	bool IRTranslator::translateUnreachable(const User &U, MachineIRBuilder &MIRBuilder) {
3064	if (!MF->getTarget().Options.TrapUnreachable)
3065	return true;
3066
3067	auto &UI = cast<UnreachableInst>(Val: U);
3068	// We may be able to ignore unreachable behind a noreturn call.
3069	if (MF->getTarget().Options.NoTrapAfterNoreturn) {
3070	const BasicBlock &BB = *UI.getParent();
3071	if (&UI != &BB.front()) {
3072	BasicBlock::const_iterator PredI =
3073	std::prev(x: BasicBlock::const_iterator(UI));
3074	if (const CallInst *Call = dyn_cast<CallInst>(Val: &*PredI)) {
3075	if (Call->doesNotReturn())
3076	return true;
3077	}
3078	}
3079	}
3080
3081	MIRBuilder.buildTrap();
3082	return true;
3083	}
3084
3085	bool IRTranslator::translateInsertElement(const User &U,
3086	MachineIRBuilder &MIRBuilder) {
3087	// If it is a <1 x Ty> vector, use the scalar as it is
3088	// not a legal vector type in LLT.
3089	if (auto *FVT = dyn_cast<FixedVectorType>(Val: U.getType());
3090	FVT && FVT->getNumElements() == 1)
3091	return translateCopy(U, V: *U.getOperand(i: 1), MIRBuilder);
3092
3093	Register Res = getOrCreateVReg(Val: U);
3094	Register Val = getOrCreateVReg(Val: *U.getOperand(i: 0));
3095	Register Elt = getOrCreateVReg(Val: *U.getOperand(i: 1));
3096	unsigned PreferredVecIdxWidth = TLI->getVectorIdxTy(DL: *DL).getSizeInBits();
3097	Register Idx;
3098	if (auto *CI = dyn_cast<ConstantInt>(Val: U.getOperand(i: 2))) {
3099	if (CI->getBitWidth() != PreferredVecIdxWidth) {
3100	APInt NewIdx = CI->getValue().zextOrTrunc(width: PreferredVecIdxWidth);
3101	auto *NewIdxCI = ConstantInt::get(Context&: CI->getContext(), V: NewIdx);
3102	Idx = getOrCreateVReg(Val: *NewIdxCI);
3103	}
3104	}
3105	if (!Idx)
3106	Idx = getOrCreateVReg(Val: *U.getOperand(i: 2));
3107	if (MRI->getType(Reg: Idx).getSizeInBits() != PreferredVecIdxWidth) {
3108	const LLT VecIdxTy = LLT::scalar(SizeInBits: PreferredVecIdxWidth);
3109	Idx = MIRBuilder.buildZExtOrTrunc(Res: VecIdxTy, Op: Idx).getReg(Idx: 0);
3110	}
3111	MIRBuilder.buildInsertVectorElement(Res, Val, Elt, Idx);
3112	return true;
3113	}
3114
3115	bool IRTranslator::translateExtractElement(const User &U,
3116	MachineIRBuilder &MIRBuilder) {
3117	// If it is a <1 x Ty> vector, use the scalar as it is
3118	// not a legal vector type in LLT.
3119	if (cast<FixedVectorType>(Val: U.getOperand(i: 0)->getType())->getNumElements() == 1)
3120	return translateCopy(U, V: *U.getOperand(i: 0), MIRBuilder);
3121
3122	Register Res = getOrCreateVReg(Val: U);
3123	Register Val = getOrCreateVReg(Val: *U.getOperand(i: 0));
3124	unsigned PreferredVecIdxWidth = TLI->getVectorIdxTy(DL: *DL).getSizeInBits();
3125	Register Idx;
3126	if (auto *CI = dyn_cast<ConstantInt>(Val: U.getOperand(i: 1))) {
3127	if (CI->getBitWidth() != PreferredVecIdxWidth) {
3128	APInt NewIdx = CI->getValue().zextOrTrunc(width: PreferredVecIdxWidth);
3129	auto *NewIdxCI = ConstantInt::get(Context&: CI->getContext(), V: NewIdx);
3130	Idx = getOrCreateVReg(Val: *NewIdxCI);
3131	}
3132	}
3133	if (!Idx)
3134	Idx = getOrCreateVReg(Val: *U.getOperand(i: 1));
3135	if (MRI->getType(Reg: Idx).getSizeInBits() != PreferredVecIdxWidth) {
3136	const LLT VecIdxTy = LLT::scalar(SizeInBits: PreferredVecIdxWidth);
3137	Idx = MIRBuilder.buildZExtOrTrunc(Res: VecIdxTy, Op: Idx).getReg(Idx: 0);
3138	}
3139	MIRBuilder.buildExtractVectorElement(Res, Val, Idx);
3140	return true;
3141	}
3142
3143	bool IRTranslator::translateShuffleVector(const User &U,
3144	MachineIRBuilder &MIRBuilder) {
3145	// A ShuffleVector that has operates on scalable vectors is a splat vector
3146	// where the value of the splat vector is the 0th element of the first
3147	// operand, since the index mask operand is the zeroinitializer (undef and
3148	// poison are treated as zeroinitializer here).
3149	if (U.getOperand(i: 0)->getType()->isScalableTy()) {
3150	Value *Op0 = U.getOperand(i: 0);
3151	auto SplatVal = MIRBuilder.buildExtractVectorElementConstant(
3152	Res: LLT::scalar(SizeInBits: Op0->getType()->getScalarSizeInBits()),
3153	Val: getOrCreateVReg(Val: *Op0), Idx: 0);
3154	MIRBuilder.buildSplatVector(Res: getOrCreateVReg(Val: U), Val: SplatVal);
3155	return true;
3156	}
3157
3158	ArrayRef<int> Mask;
3159	if (auto *SVI = dyn_cast<ShuffleVectorInst>(Val: &U))
3160	Mask = SVI->getShuffleMask();
3161	else
3162	Mask = cast<ConstantExpr>(Val: U).getShuffleMask();
3163	ArrayRef<int> MaskAlloc = MF->allocateShuffleMask(Mask);
3164	MIRBuilder
3165	.buildInstr(Opc: TargetOpcode::G_SHUFFLE_VECTOR, DstOps: {getOrCreateVReg(Val: U)},
3166	SrcOps: {getOrCreateVReg(Val: *U.getOperand(i: 0)),
3167	getOrCreateVReg(Val: *U.getOperand(i: 1))})
3168	.addShuffleMask(Val: MaskAlloc);
3169	return true;
3170	}
3171
3172	bool IRTranslator::translatePHI(const User &U, MachineIRBuilder &MIRBuilder) {
3173	const PHINode &PI = cast<PHINode>(Val: U);
3174
3175	SmallVector<MachineInstr *, 4> Insts;
3176	for (auto Reg : getOrCreateVRegs(Val: PI)) {
3177	auto MIB = MIRBuilder.buildInstr(Opc: TargetOpcode::G_PHI, DstOps: {Reg}, SrcOps: {});
3178	Insts.push_back(Elt: MIB.getInstr());
3179	}
3180
3181	PendingPHIs.emplace_back(Args: &PI, Args: std::move(Insts));
3182	return true;
3183	}
3184
3185	bool IRTranslator::translateAtomicCmpXchg(const User &U,
3186	MachineIRBuilder &MIRBuilder) {
3187	const AtomicCmpXchgInst &I = cast<AtomicCmpXchgInst>(Val: U);
3188
3189	auto Flags = TLI->getAtomicMemOperandFlags(AI: I, DL: *DL);
3190
3191	auto Res = getOrCreateVRegs(Val: I);
3192	Register OldValRes = Res[0];
3193	Register SuccessRes = Res[1];
3194	Register Addr = getOrCreateVReg(Val: *I.getPointerOperand());
3195	Register Cmp = getOrCreateVReg(Val: *I.getCompareOperand());
3196	Register NewVal = getOrCreateVReg(Val: *I.getNewValOperand());
3197
3198	MIRBuilder.buildAtomicCmpXchgWithSuccess(
3199	OldValRes, SuccessRes, Addr, CmpVal: Cmp, NewVal,
3200	MMO&: *MF->getMachineMemOperand(
3201	PtrInfo: MachinePointerInfo(I.getPointerOperand()), f: Flags, MemTy: MRI->getType(Reg: Cmp),
3202	base_alignment: getMemOpAlign(I), AAInfo: I.getAAMetadata(), Ranges: nullptr, SSID: I.getSyncScopeID(),
3203	Ordering: I.getSuccessOrdering(), FailureOrdering: I.getFailureOrdering()));
3204	return true;
3205	}
3206
3207	bool IRTranslator::translateAtomicRMW(const User &U,
3208	MachineIRBuilder &MIRBuilder) {
3209	const AtomicRMWInst &I = cast<AtomicRMWInst>(Val: U);
3210	auto Flags = TLI->getAtomicMemOperandFlags(AI: I, DL: *DL);
3211
3212	Register Res = getOrCreateVReg(Val: I);
3213	Register Addr = getOrCreateVReg(Val: *I.getPointerOperand());
3214	Register Val = getOrCreateVReg(Val: *I.getValOperand());
3215
3216	unsigned Opcode = 0;
3217	switch (I.getOperation()) {
3218	default:
3219	return false;
3220	case AtomicRMWInst::Xchg:
3221	Opcode = TargetOpcode::G_ATOMICRMW_XCHG;
3222	break;
3223	case AtomicRMWInst::Add:
3224	Opcode = TargetOpcode::G_ATOMICRMW_ADD;
3225	break;
3226	case AtomicRMWInst::Sub:
3227	Opcode = TargetOpcode::G_ATOMICRMW_SUB;
3228	break;
3229	case AtomicRMWInst::And:
3230	Opcode = TargetOpcode::G_ATOMICRMW_AND;
3231	break;
3232	case AtomicRMWInst::Nand:
3233	Opcode = TargetOpcode::G_ATOMICRMW_NAND;
3234	break;
3235	case AtomicRMWInst::Or:
3236	Opcode = TargetOpcode::G_ATOMICRMW_OR;
3237	break;
3238	case AtomicRMWInst::Xor:
3239	Opcode = TargetOpcode::G_ATOMICRMW_XOR;
3240	break;
3241	case AtomicRMWInst::Max:
3242	Opcode = TargetOpcode::G_ATOMICRMW_MAX;
3243	break;
3244	case AtomicRMWInst::Min:
3245	Opcode = TargetOpcode::G_ATOMICRMW_MIN;
3246	break;
3247	case AtomicRMWInst::UMax:
3248	Opcode = TargetOpcode::G_ATOMICRMW_UMAX;
3249	break;
3250	case AtomicRMWInst::UMin:
3251	Opcode = TargetOpcode::G_ATOMICRMW_UMIN;
3252	break;
3253	case AtomicRMWInst::FAdd:
3254	Opcode = TargetOpcode::G_ATOMICRMW_FADD;
3255	break;
3256	case AtomicRMWInst::FSub:
3257	Opcode = TargetOpcode::G_ATOMICRMW_FSUB;
3258	break;
3259	case AtomicRMWInst::FMax:
3260	Opcode = TargetOpcode::G_ATOMICRMW_FMAX;
3261	break;
3262	case AtomicRMWInst::FMin:
3263	Opcode = TargetOpcode::G_ATOMICRMW_FMIN;
3264	break;
3265	case AtomicRMWInst::UIncWrap:
3266	Opcode = TargetOpcode::G_ATOMICRMW_UINC_WRAP;
3267	break;
3268	case AtomicRMWInst::UDecWrap:
3269	Opcode = TargetOpcode::G_ATOMICRMW_UDEC_WRAP;
3270	break;
3271	}
3272
3273	MIRBuilder.buildAtomicRMW(
3274	Opcode, OldValRes: Res, Addr, Val,
3275	MMO&: *MF->getMachineMemOperand(PtrInfo: MachinePointerInfo(I.getPointerOperand()),
3276	f: Flags, MemTy: MRI->getType(Reg: Val), base_alignment: getMemOpAlign(I),
3277	AAInfo: I.getAAMetadata(), Ranges: nullptr, SSID: I.getSyncScopeID(),
3278	Ordering: I.getOrdering()));
3279	return true;
3280	}
3281
3282	bool IRTranslator::translateFence(const User &U,
3283	MachineIRBuilder &MIRBuilder) {
3284	const FenceInst &Fence = cast<FenceInst>(Val: U);
3285	MIRBuilder.buildFence(Ordering: static_cast<unsigned>(Fence.getOrdering()),
3286	Scope: Fence.getSyncScopeID());
3287	return true;
3288	}
3289
3290	bool IRTranslator::translateFreeze(const User &U,
3291	MachineIRBuilder &MIRBuilder) {
3292	const ArrayRef<Register> DstRegs = getOrCreateVRegs(Val: U);
3293	const ArrayRef<Register> SrcRegs = getOrCreateVRegs(Val: *U.getOperand(i: 0));
3294
3295	assert(DstRegs.size() == SrcRegs.size() &&
3296	"Freeze with different source and destination type?");
3297
3298	for (unsigned I = 0; I < DstRegs.size(); ++I) {
3299	MIRBuilder.buildFreeze(Dst: DstRegs[I], Src: SrcRegs[I]);
3300	}
3301
3302	return true;
3303	}
3304
3305	void IRTranslator::finishPendingPhis() {
3306	#ifndef NDEBUG
3307	DILocationVerifier Verifier;
3308	GISelObserverWrapper WrapperObserver(&Verifier);
3309	RAIIDelegateInstaller DelInstall(*MF, &WrapperObserver);
3310	#endif // ifndef NDEBUG
3311	for (auto &Phi : PendingPHIs) {
3312	const PHINode *PI = Phi.first;
3313	if (PI->getType()->isEmptyTy())
3314	continue;
3315	ArrayRef<MachineInstr *> ComponentPHIs = Phi.second;
3316	MachineBasicBlock *PhiMBB = ComponentPHIs[0]->getParent();
3317	EntryBuilder->setDebugLoc(PI->getDebugLoc());
3318	#ifndef NDEBUG
3319	Verifier.setCurrentInst(PI);
3320	#endif // ifndef NDEBUG
3321
3322	SmallSet<const MachineBasicBlock *, 16> SeenPreds;
3323	for (unsigned i = 0; i < PI->getNumIncomingValues(); ++i) {
3324	auto IRPred = PI->getIncomingBlock(i);
3325	ArrayRef<Register> ValRegs = getOrCreateVRegs(Val: *PI->getIncomingValue(i));
3326	for (auto *Pred : getMachinePredBBs(Edge: {IRPred, PI->getParent()})) {
3327	if (SeenPreds.count(Ptr: Pred) || !PhiMBB->isPredecessor(MBB: Pred))
3328	continue;
3329	SeenPreds.insert(Ptr: Pred);
3330	for (unsigned j = 0; j < ValRegs.size(); ++j) {
3331	MachineInstrBuilder MIB(*MF, ComponentPHIs[j]);
3332	MIB.addUse(RegNo: ValRegs[j]);
3333	MIB.addMBB(MBB: Pred);
3334	}
3335	}
3336	}
3337	}
3338	}
3339
3340	void IRTranslator::translateDbgValueRecord(Value *V, bool HasArgList,
3341	const DILocalVariable *Variable,
3342	const DIExpression *Expression,
3343	const DebugLoc &DL,
3344	MachineIRBuilder &MIRBuilder) {
3345	assert(Variable->isValidLocationForIntrinsic(DL) &&
3346	"Expected inlined-at fields to agree");
3347	// Act as if we're handling a debug intrinsic.
3348	MIRBuilder.setDebugLoc(DL);
3349
3350	if (!V || HasArgList) {
3351	// DI cannot produce a valid DBG_VALUE, so produce an undef DBG_VALUE to
3352	// terminate any prior location.
3353	MIRBuilder.buildIndirectDbgValue(Reg: 0, Variable, Expr: Expression);
3354	return;
3355	}
3356
3357	if (const auto *CI = dyn_cast<Constant>(Val: V)) {
3358	MIRBuilder.buildConstDbgValue(C: *CI, Variable, Expr: Expression);
3359	return;
3360	}
3361
3362	if (auto *AI = dyn_cast<AllocaInst>(Val: V);
3363	AI && AI->isStaticAlloca() && Expression->startsWithDeref()) {
3364	// If the value is an alloca and the expression starts with a
3365	// dereference, track a stack slot instead of a register, as registers
3366	// may be clobbered.
3367	auto ExprOperands = Expression->getElements();
3368	auto *ExprDerefRemoved =
3369	DIExpression::get(Context&: AI->getContext(), Elements: ExprOperands.drop_front());
3370	MIRBuilder.buildFIDbgValue(FI: getOrCreateFrameIndex(AI: *AI), Variable,
3371	Expr: ExprDerefRemoved);
3372	return;
3373	}
3374	if (translateIfEntryValueArgument(isDeclare: false, Val: V, Var: Variable, Expr: Expression, DL,
3375	MIRBuilder))
3376	return;
3377	for (Register Reg : getOrCreateVRegs(Val: *V)) {
3378	// FIXME: This does not handle register-indirect values at offset 0. The
3379	// direct/indirect thing shouldn't really be handled by something as
3380	// implicit as reg+noreg vs reg+imm in the first place, but it seems
3381	// pretty baked in right now.
3382	MIRBuilder.buildDirectDbgValue(Reg, Variable, Expr: Expression);
3383	}
3384	return;
3385	}
3386
3387	void IRTranslator::translateDbgDeclareRecord(Value *Address, bool HasArgList,
3388	const DILocalVariable *Variable,
3389	const DIExpression *Expression,
3390	const DebugLoc &DL,
3391	MachineIRBuilder &MIRBuilder) {
3392	if (!Address || isa<UndefValue>(Val: Address)) {
3393	LLVM_DEBUG(dbgs() << "Dropping debug info for " << *Variable << "\n");
3394	return;
3395	}
3396
3397	assert(Variable->isValidLocationForIntrinsic(DL) &&
3398	"Expected inlined-at fields to agree");
3399	auto AI = dyn_cast<AllocaInst>(Val: Address);
3400	if (AI && AI->isStaticAlloca()) {
3401	// Static allocas are tracked at the MF level, no need for DBG_VALUE
3402	// instructions (in fact, they get ignored if they *do* exist).
3403	MF->setVariableDbgInfo(Var: Variable, Expr: Expression,
3404	Slot: getOrCreateFrameIndex(AI: *AI), Loc: DL);
3405	return;
3406	}
3407
3408	if (translateIfEntryValueArgument(isDeclare: true, Val: Address, Var: Variable,
3409	Expr: Expression, DL,
3410	MIRBuilder))
3411	return;
3412
3413	// A dbg.declare describes the address of a source variable, so lower it
3414	// into an indirect DBG_VALUE.
3415	MIRBuilder.setDebugLoc(DL);
3416	MIRBuilder.buildIndirectDbgValue(Reg: getOrCreateVReg(Val: *Address),
3417	Variable, Expr: Expression);
3418	return;
3419	}
3420
3421	void IRTranslator::translateDbgInfo(const Instruction &Inst,
3422	MachineIRBuilder &MIRBuilder) {
3423	for (DbgRecord &DR : Inst.getDbgRecordRange()) {
3424	if (DbgLabelRecord *DLR = dyn_cast<DbgLabelRecord>(Val: &DR)) {
3425	MIRBuilder.setDebugLoc(DLR->getDebugLoc());
3426	assert(DLR->getLabel() && "Missing label");
3427	assert(DLR->getLabel()->isValidLocationForIntrinsic(
3428	MIRBuilder.getDebugLoc()) &&
3429	"Expected inlined-at fields to agree");
3430	MIRBuilder.buildDbgLabel(Label: DLR->getLabel());
3431	continue;
3432	}
3433	DbgVariableRecord &DVR = cast<DbgVariableRecord>(Val&: DR);
3434	const DILocalVariable *Variable = DVR.getVariable();
3435	const DIExpression *Expression = DVR.getExpression();
3436	Value *V = DVR.getVariableLocationOp(OpIdx: 0);
3437	if (DVR.isDbgDeclare())
3438	translateDbgDeclareRecord(Address: V, HasArgList: DVR.hasArgList(), Variable, Expression,
3439	DL: DVR.getDebugLoc(), MIRBuilder);
3440	else
3441	translateDbgValueRecord(V, HasArgList: DVR.hasArgList(), Variable, Expression,
3442	DL: DVR.getDebugLoc(), MIRBuilder);
3443	}
3444	}
3445
3446	bool IRTranslator::translate(const Instruction &Inst) {
3447	CurBuilder->setDebugLoc(Inst.getDebugLoc());
3448	CurBuilder->setPCSections(Inst.getMetadata(KindID: LLVMContext::MD_pcsections));
3449	CurBuilder->setMMRAMetadata(Inst.getMetadata(KindID: LLVMContext::MD_mmra));
3450
3451	if (TLI->fallBackToDAGISel(Inst))
3452	return false;
3453
3454	switch (Inst.getOpcode()) {
3455	#define HANDLE_INST(NUM, OPCODE, CLASS) \
3456	case Instruction::OPCODE: \
3457	return translate##OPCODE(Inst, *CurBuilder.get());
3458	#include "llvm/IR/Instruction.def"
3459	default:
3460	return false;
3461	}
3462	}
3463
3464	bool IRTranslator::translate(const Constant &C, Register Reg) {
3465	// We only emit constants into the entry block from here. To prevent jumpy
3466	// debug behaviour remove debug line.
3467	if (auto CurrInstDL = CurBuilder->getDL())
3468	EntryBuilder->setDebugLoc(DebugLoc());
3469
3470	if (auto CI = dyn_cast<ConstantInt>(Val: &C))
3471	EntryBuilder->buildConstant(Res: Reg, Val: *CI);
3472	else if (auto CF = dyn_cast<ConstantFP>(Val: &C))
3473	EntryBuilder->buildFConstant(Res: Reg, Val: *CF);
3474	else if (isa<UndefValue>(Val: C))
3475	EntryBuilder->buildUndef(Res: Reg);
3476	else if (isa<ConstantPointerNull>(Val: C))
3477	EntryBuilder->buildConstant(Res: Reg, Val: 0);
3478	else if (auto GV = dyn_cast<GlobalValue>(Val: &C))
3479	EntryBuilder->buildGlobalValue(Res: Reg, GV);
3480	else if (auto CAZ = dyn_cast<ConstantAggregateZero>(Val: &C)) {
3481	if (!isa<FixedVectorType>(Val: CAZ->getType()))
3482	return false;
3483	// Return the scalar if it is a <1 x Ty> vector.
3484	unsigned NumElts = CAZ->getElementCount().getFixedValue();
3485	if (NumElts == 1)
3486	return translateCopy(U: C, V: *CAZ->getElementValue(Idx: 0u), MIRBuilder&: *EntryBuilder);
3487	SmallVector<Register, 4> Ops;
3488	for (unsigned I = 0; I < NumElts; ++I) {
3489	Constant &Elt = *CAZ->getElementValue(Idx: I);
3490	Ops.push_back(Elt: getOrCreateVReg(Val: Elt));
3491	}
3492	EntryBuilder->buildBuildVector(Res: Reg, Ops);
3493	} else if (auto CV = dyn_cast<ConstantDataVector>(Val: &C)) {
3494	// Return the scalar if it is a <1 x Ty> vector.
3495	if (CV->getNumElements() == 1)
3496	return translateCopy(U: C, V: *CV->getElementAsConstant(i: 0), MIRBuilder&: *EntryBuilder);
3497	SmallVector<Register, 4> Ops;
3498	for (unsigned i = 0; i < CV->getNumElements(); ++i) {
3499	Constant &Elt = *CV->getElementAsConstant(i);
3500	Ops.push_back(Elt: getOrCreateVReg(Val: Elt));
3501	}
3502	EntryBuilder->buildBuildVector(Res: Reg, Ops);
3503	} else if (auto CE = dyn_cast<ConstantExpr>(Val: &C)) {
3504	switch(CE->getOpcode()) {
3505	#define HANDLE_INST(NUM, OPCODE, CLASS) \
3506	case Instruction::OPCODE: \
3507	return translate##OPCODE(*CE, *EntryBuilder.get());
3508	#include "llvm/IR/Instruction.def"
3509	default:
3510	return false;
3511	}
3512	} else if (auto CV = dyn_cast<ConstantVector>(Val: &C)) {
3513	if (CV->getNumOperands() == 1)
3514	return translateCopy(U: C, V: *CV->getOperand(i_nocapture: 0), MIRBuilder&: *EntryBuilder);
3515	SmallVector<Register, 4> Ops;
3516	for (unsigned i = 0; i < CV->getNumOperands(); ++i) {
3517	Ops.push_back(Elt: getOrCreateVReg(Val: *CV->getOperand(i_nocapture: i)));
3518	}
3519	EntryBuilder->buildBuildVector(Res: Reg, Ops);
3520	} else if (auto *BA = dyn_cast<BlockAddress>(Val: &C)) {
3521	EntryBuilder->buildBlockAddress(Res: Reg, BA);
3522	} else
3523	return false;
3524
3525	return true;
3526	}
3527
3528	bool IRTranslator::finalizeBasicBlock(const BasicBlock &BB,
3529	MachineBasicBlock &MBB) {
3530	for (auto &BTB : SL->BitTestCases) {
3531	// Emit header first, if it wasn't already emitted.
3532	if (!BTB.Emitted)
3533	emitBitTestHeader(B&: BTB, SwitchBB: BTB.Parent);
3534
3535	BranchProbability UnhandledProb = BTB.Prob;
3536	for (unsigned j = 0, ej = BTB.Cases.size(); j != ej; ++j) {
3537	UnhandledProb -= BTB.Cases[j].ExtraProb;
3538	// Set the current basic block to the mbb we wish to insert the code into
3539	MachineBasicBlock *MBB = BTB.Cases[j].ThisBB;
3540	// If all cases cover a contiguous range, it is not necessary to jump to
3541	// the default block after the last bit test fails. This is because the
3542	// range check during bit test header creation has guaranteed that every
3543	// case here doesn't go outside the range. In this case, there is no need
3544	// to perform the last bit test, as it will always be true. Instead, make
3545	// the second-to-last bit-test fall through to the target of the last bit
3546	// test, and delete the last bit test.
3547
3548	MachineBasicBlock *NextMBB;
3549	if ((BTB.ContiguousRange || BTB.FallthroughUnreachable) && j + 2 == ej) {
3550	// Second-to-last bit-test with contiguous range: fall through to the
3551	// target of the final bit test.
3552	NextMBB = BTB.Cases[j + 1].TargetBB;
3553	} else if (j + 1 == ej) {
3554	// For the last bit test, fall through to Default.
3555	NextMBB = BTB.Default;
3556	} else {
3557	// Otherwise, fall through to the next bit test.
3558	NextMBB = BTB.Cases[j + 1].ThisBB;
3559	}
3560
3561	emitBitTestCase(BB&: BTB, NextMBB, BranchProbToNext: UnhandledProb, Reg: BTB.Reg, B&: BTB.Cases[j], SwitchBB: MBB);
3562
3563	if ((BTB.ContiguousRange || BTB.FallthroughUnreachable) && j + 2 == ej) {
3564	// We need to record the replacement phi edge here that normally
3565	// happens in emitBitTestCase before we delete the case, otherwise the
3566	// phi edge will be lost.
3567	addMachineCFGPred(Edge: {BTB.Parent->getBasicBlock(),
3568	BTB.Cases[ej - 1].TargetBB->getBasicBlock()},
3569	NewPred: MBB);
3570	// Since we're not going to use the final bit test, remove it.
3571	BTB.Cases.pop_back();
3572	break;
3573	}
3574	}
3575	// This is "default" BB. We have two jumps to it. From "header" BB and from
3576	// last "case" BB, unless the latter was skipped.
3577	CFGEdge HeaderToDefaultEdge = {BTB.Parent->getBasicBlock(),
3578	BTB.Default->getBasicBlock()};
3579	addMachineCFGPred(Edge: HeaderToDefaultEdge, NewPred: BTB.Parent);
3580	if (!BTB.ContiguousRange) {
3581	addMachineCFGPred(Edge: HeaderToDefaultEdge, NewPred: BTB.Cases.back().ThisBB);
3582	}
3583	}
3584	SL->BitTestCases.clear();
3585
3586	for (auto &JTCase : SL->JTCases) {
3587	// Emit header first, if it wasn't already emitted.
3588	if (!JTCase.first.Emitted)
3589	emitJumpTableHeader(JT&: JTCase.second, JTH&: JTCase.first, HeaderBB: JTCase.first.HeaderBB);
3590
3591	emitJumpTable(JT&: JTCase.second, MBB: JTCase.second.MBB);
3592	}
3593	SL->JTCases.clear();
3594
3595	for (auto &SwCase : SL->SwitchCases)
3596	emitSwitchCase(CB&: SwCase, SwitchBB: &CurBuilder->getMBB(), MIB&: *CurBuilder);
3597	SL->SwitchCases.clear();
3598
3599	// Check if we need to generate stack-protector guard checks.
3600	StackProtector &SP = getAnalysis<StackProtector>();
3601	if (SP.shouldEmitSDCheck(BB)) {
3602	bool FunctionBasedInstrumentation =
3603	TLI->getSSPStackGuardCheck(M: *MF->getFunction().getParent());
3604	SPDescriptor.initialize(BB: &BB, MBB: &MBB, FunctionBasedInstrumentation);
3605	}
3606	// Handle stack protector.
3607	if (SPDescriptor.shouldEmitFunctionBasedCheckStackProtector()) {
3608	LLVM_DEBUG(dbgs() << "Unimplemented stack protector case\n");
3609	return false;
3610	} else if (SPDescriptor.shouldEmitStackProtector()) {
3611	MachineBasicBlock *ParentMBB = SPDescriptor.getParentMBB();
3612	MachineBasicBlock *SuccessMBB = SPDescriptor.getSuccessMBB();
3613
3614	// Find the split point to split the parent mbb. At the same time copy all
3615	// physical registers used in the tail of parent mbb into virtual registers
3616	// before the split point and back into physical registers after the split
3617	// point. This prevents us needing to deal with Live-ins and many other
3618	// register allocation issues caused by us splitting the parent mbb. The
3619	// register allocator will clean up said virtual copies later on.
3620	MachineBasicBlock::iterator SplitPoint = findSplitPointForStackProtector(
3621	BB: ParentMBB, TII: *MF->getSubtarget().getInstrInfo());
3622
3623	// Splice the terminator of ParentMBB into SuccessMBB.
3624	SuccessMBB->splice(Where: SuccessMBB->end(), Other: ParentMBB, From: SplitPoint,
3625	To: ParentMBB->end());
3626
3627	// Add compare/jump on neq/jump to the parent BB.
3628	if (!emitSPDescriptorParent(SPD&: SPDescriptor, ParentBB: ParentMBB))
3629	return false;
3630
3631	// CodeGen Failure MBB if we have not codegened it yet.
3632	MachineBasicBlock *FailureMBB = SPDescriptor.getFailureMBB();
3633	if (FailureMBB->empty()) {
3634	if (!emitSPDescriptorFailure(SPD&: SPDescriptor, FailureBB: FailureMBB))
3635	return false;
3636	}
3637
3638	// Clear the Per-BB State.
3639	SPDescriptor.resetPerBBState();
3640	}
3641	return true;
3642	}
3643
3644	bool IRTranslator::emitSPDescriptorParent(StackProtectorDescriptor &SPD,
3645	MachineBasicBlock *ParentBB) {
3646	CurBuilder->setInsertPt(MBB&: *ParentBB, II: ParentBB->end());
3647	// First create the loads to the guard/stack slot for the comparison.
3648	Type *PtrIRTy = PointerType::getUnqual(C&: MF->getFunction().getContext());
3649	const LLT PtrTy = getLLTForType(Ty&: *PtrIRTy, DL: *DL);
3650	LLT PtrMemTy = getLLTForMVT(Ty: TLI->getPointerMemTy(DL: *DL));
3651
3652	MachineFrameInfo &MFI = ParentBB->getParent()->getFrameInfo();
3653	int FI = MFI.getStackProtectorIndex();
3654
3655	Register Guard;
3656	Register StackSlotPtr = CurBuilder->buildFrameIndex(Res: PtrTy, Idx: FI).getReg(Idx: 0);
3657	const Module &M = *ParentBB->getParent()->getFunction().getParent();
3658	Align Align = DL->getPrefTypeAlign(Ty: PointerType::getUnqual(C&: M.getContext()));
3659
3660	// Generate code to load the content of the guard slot.
3661	Register GuardVal =
3662	CurBuilder
3663	->buildLoad(Res: PtrMemTy, Addr: StackSlotPtr,
3664	PtrInfo: MachinePointerInfo::getFixedStack(MF&: *MF, FI), Alignment: Align,
3665	MMOFlags: MachineMemOperand::MOLoad | MachineMemOperand::MOVolatile)
3666	.getReg(Idx: 0);
3667
3668	if (TLI->useStackGuardXorFP()) {
3669	LLVM_DEBUG(dbgs() << "Stack protector xor'ing with FP not yet implemented");
3670	return false;
3671	}
3672
3673	// Retrieve guard check function, nullptr if instrumentation is inlined.
3674	if (const Function *GuardCheckFn = TLI->getSSPStackGuardCheck(M)) {
3675	// This path is currently untestable on GlobalISel, since the only platform
3676	// that needs this seems to be Windows, and we fall back on that currently.
3677	// The code still lives here in case that changes.
3678	// Silence warning about unused variable until the code below that uses
3679	// 'GuardCheckFn' is enabled.
3680	(void)GuardCheckFn;
3681	return false;
3682	#if 0
3683	// The target provides a guard check function to validate the guard value.
3684	// Generate a call to that function with the content of the guard slot as
3685	// argument.
3686	FunctionType *FnTy = GuardCheckFn->getFunctionType();
3687	assert(FnTy->getNumParams() == 1 && "Invalid function signature");
3688	ISD::ArgFlagsTy Flags;
3689	if (GuardCheckFn->hasAttribute(1, Attribute::AttrKind::InReg))
3690	Flags.setInReg();
3691	CallLowering::ArgInfo GuardArgInfo(
3692	{GuardVal, FnTy->getParamType(0), {Flags}});
3693
3694	CallLowering::CallLoweringInfo Info;
3695	Info.OrigArgs.push_back(GuardArgInfo);
3696	Info.CallConv = GuardCheckFn->getCallingConv();
3697	Info.Callee = MachineOperand::CreateGA(GuardCheckFn, 0);
3698	Info.OrigRet = {Register(), FnTy->getReturnType()};
3699	if (!CLI->lowerCall(MIRBuilder, Info)) {
3700	LLVM_DEBUG(dbgs() << "Failed to lower call to stack protector check\n");
3701	return false;
3702	}
3703	return true;
3704	#endif
3705	}
3706
3707	// If useLoadStackGuardNode returns true, generate LOAD_STACK_GUARD.
3708	// Otherwise, emit a volatile load to retrieve the stack guard value.
3709	if (TLI->useLoadStackGuardNode()) {
3710	Guard =
3711	MRI->createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: PtrTy.getSizeInBits()));
3712	getStackGuard(DstReg: Guard, MIRBuilder&: *CurBuilder);
3713	} else {
3714	// TODO: test using android subtarget when we support @llvm.thread.pointer.
3715	const Value *IRGuard = TLI->getSDagStackGuard(M);
3716	Register GuardPtr = getOrCreateVReg(Val: *IRGuard);
3717
3718	Guard = CurBuilder
3719	->buildLoad(Res: PtrMemTy, Addr: GuardPtr,
3720	PtrInfo: MachinePointerInfo::getFixedStack(MF&: *MF, FI), Alignment: Align,
3721	MMOFlags: MachineMemOperand::MOLoad |
3722	MachineMemOperand::MOVolatile)
3723	.getReg(Idx: 0);
3724	}
3725
3726	// Perform the comparison.
3727	auto Cmp =
3728	CurBuilder->buildICmp(Pred: CmpInst::ICMP_NE, Res: LLT::scalar(SizeInBits: 1), Op0: Guard, Op1: GuardVal);
3729	// If the guard/stackslot do not equal, branch to failure MBB.
3730	CurBuilder->buildBrCond(Tst: Cmp, Dest&: *SPD.getFailureMBB());
3731	// Otherwise branch to success MBB.
3732	CurBuilder->buildBr(Dest&: *SPD.getSuccessMBB());
3733	return true;
3734	}
3735
3736	bool IRTranslator::emitSPDescriptorFailure(StackProtectorDescriptor &SPD,
3737	MachineBasicBlock *FailureBB) {
3738	CurBuilder->setInsertPt(MBB&: *FailureBB, II: FailureBB->end());
3739
3740	const RTLIB::Libcall Libcall = RTLIB::STACKPROTECTOR_CHECK_FAIL;
3741	const char *Name = TLI->getLibcallName(Call: Libcall);
3742
3743	CallLowering::CallLoweringInfo Info;
3744	Info.CallConv = TLI->getLibcallCallingConv(Call: Libcall);
3745	Info.Callee = MachineOperand::CreateES(SymName: Name);
3746	Info.OrigRet = {Register(), Type::getVoidTy(C&: MF->getFunction().getContext()),
3747	0};
3748	if (!CLI->lowerCall(MIRBuilder&: *CurBuilder, Info)) {
3749	LLVM_DEBUG(dbgs() << "Failed to lower call to stack protector fail\n");
3750	return false;
3751	}
3752
3753	// On PS4/PS5, the "return address" must still be within the calling
3754	// function, even if it's at the very end, so emit an explicit TRAP here.
3755	// WebAssembly needs an unreachable instruction after a non-returning call,
3756	// because the function return type can be different from __stack_chk_fail's
3757	// return type (void).
3758	const TargetMachine &TM = MF->getTarget();
3759	if (TM.getTargetTriple().isPS() || TM.getTargetTriple().isWasm()) {
3760	LLVM_DEBUG(dbgs() << "Unhandled trap emission for stack protector fail\n");
3761	return false;
3762	}
3763	return true;
3764	}
3765
3766	void IRTranslator::finalizeFunction() {
3767	// Release the memory used by the different maps we
3768	// needed during the translation.
3769	PendingPHIs.clear();
3770	VMap.reset();
3771	FrameIndices.clear();
3772	MachinePreds.clear();
3773	// MachineIRBuilder::DebugLoc can outlive the DILocation it holds. Clear it
3774	// to avoid accessing free’d memory (in runOnMachineFunction) and to avoid
3775	// destroying it twice (in ~IRTranslator() and ~LLVMContext())
3776	EntryBuilder.reset();
3777	CurBuilder.reset();
3778	FuncInfo.clear();
3779	SPDescriptor.resetPerFunctionState();
3780	}
3781
3782	/// Returns true if a BasicBlock \p BB within a variadic function contains a
3783	/// variadic musttail call.
3784	static bool checkForMustTailInVarArgFn(bool IsVarArg, const BasicBlock &BB) {
3785	if (!IsVarArg)
3786	return false;
3787
3788	// Walk the block backwards, because tail calls usually only appear at the end
3789	// of a block.
3790	return llvm::any_of(Range: llvm::reverse(C: BB), P: [](const Instruction &I) {
3791	const auto *CI = dyn_cast<CallInst>(Val: &I);
3792	return CI && CI->isMustTailCall();
3793	});
3794	}
3795
3796	bool IRTranslator::runOnMachineFunction(MachineFunction &CurMF) {
3797	MF = &CurMF;
3798	const Function &F = MF->getFunction();
3799	GISelCSEAnalysisWrapper &Wrapper =
3800	getAnalysis<GISelCSEAnalysisWrapperPass>().getCSEWrapper();
3801	// Set the CSEConfig and run the analysis.
3802	GISelCSEInfo *CSEInfo = nullptr;
3803	TPC = &getAnalysis<TargetPassConfig>();
3804	bool EnableCSE = EnableCSEInIRTranslator.getNumOccurrences()
3805	? EnableCSEInIRTranslator
3806	: TPC->isGISelCSEEnabled();
3807	TLI = MF->getSubtarget().getTargetLowering();
3808
3809	if (EnableCSE) {
3810	EntryBuilder = std::make_unique<CSEMIRBuilder>(args&: CurMF);
3811	CSEInfo = &Wrapper.get(CSEOpt: TPC->getCSEConfig());
3812	EntryBuilder->setCSEInfo(CSEInfo);
3813	CurBuilder = std::make_unique<CSEMIRBuilder>(args&: CurMF);
3814	CurBuilder->setCSEInfo(CSEInfo);
3815	} else {
3816	EntryBuilder = std::make_unique<MachineIRBuilder>();
3817	CurBuilder = std::make_unique<MachineIRBuilder>();
3818	}
3819	CLI = MF->getSubtarget().getCallLowering();
3820	CurBuilder->setMF(*MF);
3821	EntryBuilder->setMF(*MF);
3822	MRI = &MF->getRegInfo();
3823	DL = &F.getParent()->getDataLayout();
3824	ORE = std::make_unique<OptimizationRemarkEmitter>(args: &F);
3825	const TargetMachine &TM = MF->getTarget();
3826	TM.resetTargetOptions(F);
3827	EnableOpts = OptLevel != CodeGenOptLevel::None && !skipFunction(F);
3828	FuncInfo.MF = MF;
3829	if (EnableOpts) {
3830	AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
3831	FuncInfo.BPI = &getAnalysis<BranchProbabilityInfoWrapperPass>().getBPI();
3832	} else {
3833	AA = nullptr;
3834	FuncInfo.BPI = nullptr;
3835	}
3836
3837	AC = &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(
3838	F&: MF->getFunction());
3839	LibInfo = &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
3840	FuncInfo.CanLowerReturn = CLI->checkReturnTypeForCallConv(MF&: *MF);
3841
3842	SL = std::make_unique<GISelSwitchLowering>(args: this, args&: FuncInfo);
3843	SL->init(tli: *TLI, tm: TM, dl: *DL);
3844
3845	assert(PendingPHIs.empty() && "stale PHIs");
3846
3847	// Targets which want to use big endian can enable it using
3848	// enableBigEndian()
3849	if (!DL->isLittleEndian() && !CLI->enableBigEndian()) {
3850	// Currently we don't properly handle big endian code.
3851	OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
3852	F.getSubprogram(), &F.getEntryBlock());
3853	R << "unable to translate in big endian mode";
3854	reportTranslationError(MF&: *MF, TPC: *TPC, ORE&: *ORE, R);
3855	}
3856
3857	// Release the per-function state when we return, whether we succeeded or not.
3858	auto FinalizeOnReturn = make_scope_exit(F: [this]() { finalizeFunction(); });
3859
3860	// Setup a separate basic-block for the arguments and constants
3861	MachineBasicBlock *EntryBB = MF->CreateMachineBasicBlock();
3862	MF->push_back(MBB: EntryBB);
3863	EntryBuilder->setMBB(*EntryBB);
3864
3865	DebugLoc DbgLoc = F.getEntryBlock().getFirstNonPHI()->getDebugLoc();
3866	SwiftError.setFunction(CurMF);
3867	SwiftError.createEntriesInEntryBlock(DbgLoc);
3868
3869	bool IsVarArg = F.isVarArg();
3870	bool HasMustTailInVarArgFn = false;
3871
3872	// Create all blocks, in IR order, to preserve the layout.
3873	for (const BasicBlock &BB: F) {
3874	auto *&MBB = BBToMBB[&BB];
3875
3876	MBB = MF->CreateMachineBasicBlock(BB: &BB);
3877	MF->push_back(MBB);
3878
3879	if (BB.hasAddressTaken())
3880	MBB->setAddressTakenIRBlock(const_cast<BasicBlock *>(&BB));
3881
3882	if (!HasMustTailInVarArgFn)
3883	HasMustTailInVarArgFn = checkForMustTailInVarArgFn(IsVarArg, BB);
3884	}
3885
3886	MF->getFrameInfo().setHasMustTailInVarArgFunc(HasMustTailInVarArgFn);
3887
3888	// Make our arguments/constants entry block fallthrough to the IR entry block.
3889	EntryBB->addSuccessor(Succ: &getMBB(BB: F.front()));
3890
3891	if (CLI->fallBackToDAGISel(MF: *MF)) {
3892	OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
3893	F.getSubprogram(), &F.getEntryBlock());
3894	R << "unable to lower function: " << ore::NV("Prototype", F.getType());
3895	reportTranslationError(MF&: *MF, TPC: *TPC, ORE&: *ORE, R);
3896	return false;
3897	}
3898
3899	// Lower the actual args into this basic block.
3900	SmallVector<ArrayRef<Register>, 8> VRegArgs;
3901	for (const Argument &Arg: F.args()) {
3902	if (DL->getTypeStoreSize(Ty: Arg.getType()).isZero())
3903	continue; // Don't handle zero sized types.
3904	ArrayRef<Register> VRegs = getOrCreateVRegs(Val: Arg);
3905	VRegArgs.push_back(Elt: VRegs);
3906
3907	if (Arg.hasSwiftErrorAttr()) {
3908	assert(VRegs.size() == 1 && "Too many vregs for Swift error");
3909	SwiftError.setCurrentVReg(MBB: EntryBB, SwiftError.getFunctionArg(), VRegs[0]);
3910	}
3911	}
3912
3913	if (!CLI->lowerFormalArguments(MIRBuilder&: *EntryBuilder, F, VRegs: VRegArgs, FLI&: FuncInfo)) {
3914	OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
3915	F.getSubprogram(), &F.getEntryBlock());
3916	R << "unable to lower arguments: " << ore::NV("Prototype", F.getType());
3917	reportTranslationError(MF&: *MF, TPC: *TPC, ORE&: *ORE, R);
3918	return false;
3919	}
3920
3921	// Need to visit defs before uses when translating instructions.
3922	GISelObserverWrapper WrapperObserver;
3923	if (EnableCSE && CSEInfo)
3924	WrapperObserver.addObserver(O: CSEInfo);
3925	{
3926	ReversePostOrderTraversal<const Function *> RPOT(&F);
3927	#ifndef NDEBUG
3928	DILocationVerifier Verifier;
3929	WrapperObserver.addObserver(O: &Verifier);
3930	#endif // ifndef NDEBUG
3931	RAIIDelegateInstaller DelInstall(*MF, &WrapperObserver);
3932	RAIIMFObserverInstaller ObsInstall(*MF, WrapperObserver);
3933	for (const BasicBlock *BB : RPOT) {
3934	MachineBasicBlock &MBB = getMBB(BB: *BB);
3935	// Set the insertion point of all the following translations to
3936	// the end of this basic block.
3937	CurBuilder->setMBB(MBB);
3938	HasTailCall = false;
3939	for (const Instruction &Inst : *BB) {
3940	// If we translated a tail call in the last step, then we know
3941	// everything after the call is either a return, or something that is
3942	// handled by the call itself. (E.g. a lifetime marker or assume
3943	// intrinsic.) In this case, we should stop translating the block and
3944	// move on.
3945	if (HasTailCall)
3946	break;
3947	#ifndef NDEBUG
3948	Verifier.setCurrentInst(&Inst);
3949	#endif // ifndef NDEBUG
3950
3951	// Translate any debug-info attached to the instruction.
3952	translateDbgInfo(Inst, MIRBuilder&: *CurBuilder.get());
3953
3954	if (translate(Inst))
3955	continue;
3956
3957	OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
3958	Inst.getDebugLoc(), BB);
3959	R << "unable to translate instruction: " << ore::NV("Opcode", &Inst);
3960
3961	if (ORE->allowExtraAnalysis(PassName: "gisel-irtranslator")) {
3962	std::string InstStrStorage;
3963	raw_string_ostream InstStr(InstStrStorage);
3964	InstStr << Inst;
3965
3966	R << ": '" << InstStr.str() << "'";
3967	}
3968
3969	reportTranslationError(MF&: *MF, TPC: *TPC, ORE&: *ORE, R);
3970	return false;
3971	}
3972
3973	if (!finalizeBasicBlock(BB: *BB, MBB)) {
3974	OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
3975	BB->getTerminator()->getDebugLoc(), BB);
3976	R << "unable to translate basic block";
3977	reportTranslationError(MF&: *MF, TPC: *TPC, ORE&: *ORE, R);
3978	return false;
3979	}
3980	}
3981	#ifndef NDEBUG
3982	WrapperObserver.removeObserver(O: &Verifier);
3983	#endif
3984	}
3985
3986	finishPendingPhis();
3987
3988	SwiftError.propagateVRegs();
3989
3990	// Merge the argument lowering and constants block with its single
3991	// successor, the LLVM-IR entry block. We want the basic block to
3992	// be maximal.
3993	assert(EntryBB->succ_size() == 1 &&
3994	"Custom BB used for lowering should have only one successor");
3995	// Get the successor of the current entry block.
3996	MachineBasicBlock &NewEntryBB = **EntryBB->succ_begin();
3997	assert(NewEntryBB.pred_size() == 1 &&
3998	"LLVM-IR entry block has a predecessor!?");
3999	// Move all the instruction from the current entry block to the
4000	// new entry block.
4001	NewEntryBB.splice(Where: NewEntryBB.begin(), Other: EntryBB, From: EntryBB->begin(),
4002	To: EntryBB->end());
4003
4004	// Update the live-in information for the new entry block.
4005	for (const MachineBasicBlock::RegisterMaskPair &LiveIn : EntryBB->liveins())
4006	NewEntryBB.addLiveIn(RegMaskPair: LiveIn);
4007	NewEntryBB.sortUniqueLiveIns();
4008
4009	// Get rid of the now empty basic block.
4010	EntryBB->removeSuccessor(Succ: &NewEntryBB);
4011	MF->remove(MBBI: EntryBB);
4012	MF->deleteMachineBasicBlock(MBB: EntryBB);
4013
4014	assert(&MF->front() == &NewEntryBB &&
4015	"New entry wasn't next in the list of basic block!");
4016
4017	// Initialize stack protector information.
4018	StackProtector &SP = getAnalysis<StackProtector>();
4019	SP.copyToMachineFrameInfo(MFI&: MF->getFrameInfo());
4020
4021	return false;
4022	}
4023