1	//===- FastISel.cpp - Implementation of the FastISel class ----------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file contains the implementation of the FastISel class.
10	//
11	// "Fast" instruction selection is designed to emit very poor code quickly.
12	// Also, it is not designed to be able to do much lowering, so most illegal
13	// types (e.g. i64 on 32-bit targets) and operations are not supported. It is
14	// also not intended to be able to do much optimization, except in a few cases
15	// where doing optimizations reduces overall compile time. For example, folding
16	// constants into immediate fields is often done, because it's cheap and it
17	// reduces the number of instructions later phases have to examine.
18	//
19	// "Fast" instruction selection is able to fail gracefully and transfer
20	// control to the SelectionDAG selector for operations that it doesn't
21	// support. In many cases, this allows us to avoid duplicating a lot of
22	// the complicated lowering logic that SelectionDAG currently has.
23	//
24	// The intended use for "fast" instruction selection is "-O0" mode
25	// compilation, where the quality of the generated code is irrelevant when
26	// weighed against the speed at which the code can be generated. Also,
27	// at -O0, the LLVM optimizers are not running, and this makes the
28	// compile time of codegen a much higher portion of the overall compile
29	// time. Despite its limitations, "fast" instruction selection is able to
30	// handle enough code on its own to provide noticeable overall speedups
31	// in -O0 compiles.
32	//
33	// Basic operations are supported in a target-independent way, by reading
34	// the same instruction descriptions that the SelectionDAG selector reads,
35	// and identifying simple arithmetic operations that can be directly selected
36	// from simple operators. More complicated operations currently require
37	// target-specific code.
38	//
39	//===----------------------------------------------------------------------===//
40
41	#include "llvm/CodeGen/FastISel.h"
42	#include "llvm/ADT/APFloat.h"
43	#include "llvm/ADT/APSInt.h"
44	#include "llvm/ADT/DenseMap.h"
45	#include "llvm/ADT/SmallPtrSet.h"
46	#include "llvm/ADT/SmallString.h"
47	#include "llvm/ADT/SmallVector.h"
48	#include "llvm/ADT/Statistic.h"
49	#include "llvm/Analysis/BranchProbabilityInfo.h"
50	#include "llvm/Analysis/TargetLibraryInfo.h"
51	#include "llvm/CodeGen/Analysis.h"
52	#include "llvm/CodeGen/FunctionLoweringInfo.h"
53	#include "llvm/CodeGen/ISDOpcodes.h"
54	#include "llvm/CodeGen/MachineBasicBlock.h"
55	#include "llvm/CodeGen/MachineFrameInfo.h"
56	#include "llvm/CodeGen/MachineInstr.h"
57	#include "llvm/CodeGen/MachineInstrBuilder.h"
58	#include "llvm/CodeGen/MachineMemOperand.h"
59	#include "llvm/CodeGen/MachineModuleInfo.h"
60	#include "llvm/CodeGen/MachineOperand.h"
61	#include "llvm/CodeGen/MachineRegisterInfo.h"
62	#include "llvm/CodeGen/StackMaps.h"
63	#include "llvm/CodeGen/TargetInstrInfo.h"
64	#include "llvm/CodeGen/TargetLowering.h"
65	#include "llvm/CodeGen/TargetSubtargetInfo.h"
66	#include "llvm/CodeGen/ValueTypes.h"
67	#include "llvm/CodeGenTypes/MachineValueType.h"
68	#include "llvm/IR/Argument.h"
69	#include "llvm/IR/Attributes.h"
70	#include "llvm/IR/BasicBlock.h"
71	#include "llvm/IR/CallingConv.h"
72	#include "llvm/IR/Constant.h"
73	#include "llvm/IR/Constants.h"
74	#include "llvm/IR/DataLayout.h"
75	#include "llvm/IR/DebugLoc.h"
76	#include "llvm/IR/DerivedTypes.h"
77	#include "llvm/IR/DiagnosticInfo.h"
78	#include "llvm/IR/Function.h"
79	#include "llvm/IR/GetElementPtrTypeIterator.h"
80	#include "llvm/IR/GlobalValue.h"
81	#include "llvm/IR/InlineAsm.h"
82	#include "llvm/IR/InstrTypes.h"
83	#include "llvm/IR/Instruction.h"
84	#include "llvm/IR/Instructions.h"
85	#include "llvm/IR/IntrinsicInst.h"
86	#include "llvm/IR/LLVMContext.h"
87	#include "llvm/IR/Mangler.h"
88	#include "llvm/IR/Metadata.h"
89	#include "llvm/IR/Operator.h"
90	#include "llvm/IR/PatternMatch.h"
91	#include "llvm/IR/Type.h"
92	#include "llvm/IR/User.h"
93	#include "llvm/IR/Value.h"
94	#include "llvm/MC/MCContext.h"
95	#include "llvm/MC/MCInstrDesc.h"
96	#include "llvm/Support/Casting.h"
97	#include "llvm/Support/Debug.h"
98	#include "llvm/Support/ErrorHandling.h"
99	#include "llvm/Support/MathExtras.h"
100	#include "llvm/Support/raw_ostream.h"
101	#include "llvm/Target/TargetMachine.h"
102	#include "llvm/Target/TargetOptions.h"
103	#include <algorithm>
104	#include <cassert>
105	#include <cstdint>
106	#include <iterator>
107	#include <optional>
108	#include <utility>
109
110	using namespace llvm;
111	using namespace PatternMatch;
112
113	#define DEBUG_TYPE "isel"
114
115	STATISTIC(NumFastIselSuccessIndependent, "Number of insts selected by "
116	"target-independent selector");
117	STATISTIC(NumFastIselSuccessTarget, "Number of insts selected by "
118	"target-specific selector");
119	STATISTIC(NumFastIselDead, "Number of dead insts removed on failure");
120
121	/// Set the current block to which generated machine instructions will be
122	/// appended.
123	void FastISel::startNewBlock() {
124	assert(LocalValueMap.empty() &&
125	"local values should be cleared after finishing a BB");
126
127	// Instructions are appended to FuncInfo.MBB. If the basic block already
128	// contains labels or copies, use the last instruction as the last local
129	// value.
130	EmitStartPt = nullptr;
131	if (!FuncInfo.MBB->empty())
132	EmitStartPt = &FuncInfo.MBB->back();
133	LastLocalValue = EmitStartPt;
134	}
135
136	void FastISel::finishBasicBlock() { flushLocalValueMap(); }
137
138	bool FastISel::lowerArguments() {
139	if (!FuncInfo.CanLowerReturn)
140	// Fallback to SDISel argument lowering code to deal with sret pointer
141	// parameter.
142	return false;
143
144	if (!fastLowerArguments())
145	return false;
146
147	// Enter arguments into ValueMap for uses in non-entry BBs.
148	for (Function::const_arg_iterator I = FuncInfo.Fn->arg_begin(),
149	E = FuncInfo.Fn->arg_end();
150	I != E; ++I) {
151	DenseMap<const Value *, Register>::iterator VI = LocalValueMap.find(Val: &*I);
152	assert(VI != LocalValueMap.end() && "Missed an argument?");
153	FuncInfo.ValueMap[&*I] = VI->second;
154	}
155	return true;
156	}
157
158	/// Return the defined register if this instruction defines exactly one
159	/// virtual register and uses no other virtual registers. Otherwise return 0.
160	static Register findLocalRegDef(MachineInstr &MI) {
161	Register RegDef;
162	for (const MachineOperand &MO : MI.operands()) {
163	if (!MO.isReg())
164	continue;
165	if (MO.isDef()) {
166	if (RegDef)
167	return Register();
168	RegDef = MO.getReg();
169	} else if (MO.getReg().isVirtual()) {
170	// This is another use of a vreg. Don't delete it.
171	return Register();
172	}
173	}
174	return RegDef;
175	}
176
177	static bool isRegUsedByPhiNodes(Register DefReg,
178	FunctionLoweringInfo &FuncInfo) {
179	for (auto &P : FuncInfo.PHINodesToUpdate)
180	if (P.second == DefReg)
181	return true;
182	return false;
183	}
184
185	void FastISel::flushLocalValueMap() {
186	// If FastISel bails out, it could leave local value instructions behind
187	// that aren't used for anything. Detect and erase those.
188	if (LastLocalValue != EmitStartPt) {
189	// Save the first instruction after local values, for later.
190	MachineBasicBlock::iterator FirstNonValue(LastLocalValue);
191	++FirstNonValue;
192
193	MachineBasicBlock::reverse_iterator RE =
194	EmitStartPt ? MachineBasicBlock::reverse_iterator(EmitStartPt)
195	: FuncInfo.MBB->rend();
196	MachineBasicBlock::reverse_iterator RI(LastLocalValue);
197	for (MachineInstr &LocalMI :
198	llvm::make_early_inc_range(Range: llvm::make_range(x: RI, y: RE))) {
199	Register DefReg = findLocalRegDef(MI&: LocalMI);
200	if (!DefReg)
201	continue;
202	if (FuncInfo.RegsWithFixups.count(V: DefReg))
203	continue;
204	bool UsedByPHI = isRegUsedByPhiNodes(DefReg, FuncInfo);
205	if (!UsedByPHI && MRI.use_nodbg_empty(RegNo: DefReg)) {
206	if (EmitStartPt == &LocalMI)
207	EmitStartPt = EmitStartPt->getPrevNode();
208	LLVM_DEBUG(dbgs() << "removing dead local value materialization"
209	<< LocalMI);
210	LocalMI.eraseFromParent();
211	}
212	}
213
214	if (FirstNonValue != FuncInfo.MBB->end()) {
215	// See if there are any local value instructions left. If so, we want to
216	// make sure the first one has a debug location; if it doesn't, use the
217	// first non-value instruction's debug location.
218
219	// If EmitStartPt is non-null, this block had copies at the top before
220	// FastISel started doing anything; it points to the last one, so the
221	// first local value instruction is the one after EmitStartPt.
222	// If EmitStartPt is null, the first local value instruction is at the
223	// top of the block.
224	MachineBasicBlock::iterator FirstLocalValue =
225	EmitStartPt ? ++MachineBasicBlock::iterator(EmitStartPt)
226	: FuncInfo.MBB->begin();
227	if (FirstLocalValue != FirstNonValue && !FirstLocalValue->getDebugLoc())
228	FirstLocalValue->setDebugLoc(FirstNonValue->getDebugLoc());
229	}
230	}
231
232	LocalValueMap.clear();
233	LastLocalValue = EmitStartPt;
234	recomputeInsertPt();
235	SavedInsertPt = FuncInfo.InsertPt;
236	}
237
238	Register FastISel::getRegForValue(const Value *V) {
239	EVT RealVT = TLI.getValueType(DL, Ty: V->getType(), /*AllowUnknown=*/true);
240	// Don't handle non-simple values in FastISel.
241	if (!RealVT.isSimple())
242	return Register();
243
244	// Ignore illegal types. We must do this before looking up the value
245	// in ValueMap because Arguments are given virtual registers regardless
246	// of whether FastISel can handle them.
247	MVT VT = RealVT.getSimpleVT();
248	if (!TLI.isTypeLegal(VT)) {
249	// Handle integer promotions, though, because they're common and easy.
250	if (VT == MVT::i1 || VT == MVT::i8 || VT == MVT::i16)
251	VT = TLI.getTypeToTransformTo(Context&: V->getContext(), VT).getSimpleVT();
252	else
253	return Register();
254	}
255
256	// Look up the value to see if we already have a register for it.
257	Register Reg = lookUpRegForValue(V);
258	if (Reg)
259	return Reg;
260
261	// In bottom-up mode, just create the virtual register which will be used
262	// to hold the value. It will be materialized later.
263	if (isa<Instruction>(Val: V) &&
264	(!isa<AllocaInst>(Val: V) ||
265	!FuncInfo.StaticAllocaMap.count(Val: cast<AllocaInst>(Val: V))))
266	return FuncInfo.InitializeRegForValue(V);
267
268	SavePoint SaveInsertPt = enterLocalValueArea();
269
270	// Materialize the value in a register. Emit any instructions in the
271	// local value area.
272	Reg = materializeRegForValue(V, VT);
273
274	leaveLocalValueArea(Old: SaveInsertPt);
275
276	return Reg;
277	}
278
279	Register FastISel::materializeConstant(const Value *V, MVT VT) {
280	Register Reg;
281	if (const auto *CI = dyn_cast<ConstantInt>(Val: V)) {
282	if (CI->getValue().getActiveBits() <= 64)
283	Reg = fastEmit_i(VT, RetVT: VT, Opcode: ISD::Constant, Imm: CI->getZExtValue());
284	} else if (isa<AllocaInst>(Val: V))
285	Reg = fastMaterializeAlloca(C: cast<AllocaInst>(Val: V));
286	else if (isa<ConstantPointerNull>(Val: V))
287	// Translate this as an integer zero so that it can be
288	// local-CSE'd with actual integer zeros.
289	Reg =
290	getRegForValue(V: Constant::getNullValue(Ty: DL.getIntPtrType(V->getType())));
291	else if (const auto *CF = dyn_cast<ConstantFP>(Val: V)) {
292	if (CF->isNullValue())
293	Reg = fastMaterializeFloatZero(CF);
294	else
295	// Try to emit the constant directly.
296	Reg = fastEmit_f(VT, RetVT: VT, Opcode: ISD::ConstantFP, FPImm: CF);
297
298	if (!Reg) {
299	// Try to emit the constant by using an integer constant with a cast.
300	const APFloat &Flt = CF->getValueAPF();
301	EVT IntVT = TLI.getPointerTy(DL);
302	uint32_t IntBitWidth = IntVT.getSizeInBits();
303	APSInt SIntVal(IntBitWidth, /*isUnsigned=*/false);
304	bool isExact;
305	(void)Flt.convertToInteger(Result&: SIntVal, RM: APFloat::rmTowardZero, IsExact: &isExact);
306	if (isExact) {
307	Register IntegerReg =
308	getRegForValue(V: ConstantInt::get(Context&: V->getContext(), V: SIntVal));
309	if (IntegerReg)
310	Reg = fastEmit_r(VT: IntVT.getSimpleVT(), RetVT: VT, Opcode: ISD::SINT_TO_FP,
311	Op0: IntegerReg);
312	}
313	}
314	} else if (const auto *Op = dyn_cast<Operator>(Val: V)) {
315	if (!selectOperator(I: Op, Opcode: Op->getOpcode()))
316	if (!isa<Instruction>(Val: Op) ||
317	!fastSelectInstruction(I: cast<Instruction>(Val: Op)))
318	return 0;
319	Reg = lookUpRegForValue(V: Op);
320	} else if (isa<UndefValue>(Val: V)) {
321	Reg = createResultReg(RC: TLI.getRegClassFor(VT));
322	BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
323	MCID: TII.get(Opcode: TargetOpcode::IMPLICIT_DEF), DestReg: Reg);
324	}
325	return Reg;
326	}
327
328	/// Helper for getRegForValue. This function is called when the value isn't
329	/// already available in a register and must be materialized with new
330	/// instructions.
331	Register FastISel::materializeRegForValue(const Value *V, MVT VT) {
332	Register Reg;
333	// Give the target-specific code a try first.
334	if (isa<Constant>(Val: V))
335	Reg = fastMaterializeConstant(C: cast<Constant>(Val: V));
336
337	// If target-specific code couldn't or didn't want to handle the value, then
338	// give target-independent code a try.
339	if (!Reg)
340	Reg = materializeConstant(V, VT);
341
342	// Don't cache constant materializations in the general ValueMap.
343	// To do so would require tracking what uses they dominate.
344	if (Reg) {
345	LocalValueMap[V] = Reg;
346	LastLocalValue = MRI.getVRegDef(Reg);
347	}
348	return Reg;
349	}
350
351	Register FastISel::lookUpRegForValue(const Value *V) {
352	// Look up the value to see if we already have a register for it. We
353	// cache values defined by Instructions across blocks, and other values
354	// only locally. This is because Instructions already have the SSA
355	// def-dominates-use requirement enforced.
356	DenseMap<const Value *, Register>::iterator I = FuncInfo.ValueMap.find(Val: V);
357	if (I != FuncInfo.ValueMap.end())
358	return I->second;
359	return LocalValueMap[V];
360	}
361
362	void FastISel::updateValueMap(const Value *I, Register Reg, unsigned NumRegs) {
363	if (!isa<Instruction>(Val: I)) {
364	LocalValueMap[I] = Reg;
365	return;
366	}
367
368	Register &AssignedReg = FuncInfo.ValueMap[I];
369	if (!AssignedReg)
370	// Use the new register.
371	AssignedReg = Reg;
372	else if (Reg != AssignedReg) {
373	// Arrange for uses of AssignedReg to be replaced by uses of Reg.
374	for (unsigned i = 0; i < NumRegs; i++) {
375	FuncInfo.RegFixups[AssignedReg + i] = Reg + i;
376	FuncInfo.RegsWithFixups.insert(V: Reg + i);
377	}
378
379	AssignedReg = Reg;
380	}
381	}
382
383	Register FastISel::getRegForGEPIndex(const Value *Idx) {
384	Register IdxN = getRegForValue(V: Idx);
385	if (!IdxN)
386	// Unhandled operand. Halt "fast" selection and bail.
387	return Register();
388
389	// If the index is smaller or larger than intptr_t, truncate or extend it.
390	MVT PtrVT = TLI.getPointerTy(DL);
391	EVT IdxVT = EVT::getEVT(Ty: Idx->getType(), /*HandleUnknown=*/false);
392	if (IdxVT.bitsLT(VT: PtrVT)) {
393	IdxN = fastEmit_r(VT: IdxVT.getSimpleVT(), RetVT: PtrVT, Opcode: ISD::SIGN_EXTEND, Op0: IdxN);
394	} else if (IdxVT.bitsGT(VT: PtrVT)) {
395	IdxN =
396	fastEmit_r(VT: IdxVT.getSimpleVT(), RetVT: PtrVT, Opcode: ISD::TRUNCATE, Op0: IdxN);
397	}
398	return IdxN;
399	}
400
401	void FastISel::recomputeInsertPt() {
402	if (getLastLocalValue()) {
403	FuncInfo.InsertPt = getLastLocalValue();
404	FuncInfo.MBB = FuncInfo.InsertPt->getParent();
405	++FuncInfo.InsertPt;
406	} else
407	FuncInfo.InsertPt = FuncInfo.MBB->getFirstNonPHI();
408	}
409
410	void FastISel::removeDeadCode(MachineBasicBlock::iterator I,
411	MachineBasicBlock::iterator E) {
412	assert(I.isValid() && E.isValid() && std::distance(I, E) > 0 &&
413	"Invalid iterator!");
414	while (I != E) {
415	if (SavedInsertPt == I)
416	SavedInsertPt = E;
417	if (EmitStartPt == I)
418	EmitStartPt = E.isValid() ? &*E : nullptr;
419	if (LastLocalValue == I)
420	LastLocalValue = E.isValid() ? &*E : nullptr;
421
422	MachineInstr *Dead = &*I;
423	++I;
424	Dead->eraseFromParent();
425	++NumFastIselDead;
426	}
427	recomputeInsertPt();
428	}
429
430	FastISel::SavePoint FastISel::enterLocalValueArea() {
431	SavePoint OldInsertPt = FuncInfo.InsertPt;
432	recomputeInsertPt();
433	return OldInsertPt;
434	}
435
436	void FastISel::leaveLocalValueArea(SavePoint OldInsertPt) {
437	if (FuncInfo.InsertPt != FuncInfo.MBB->begin())
438	LastLocalValue = &*std::prev(x: FuncInfo.InsertPt);
439
440	// Restore the previous insert position.
441	FuncInfo.InsertPt = OldInsertPt;
442	}
443
444	bool FastISel::selectBinaryOp(const User *I, unsigned ISDOpcode) {
445	EVT VT = EVT::getEVT(Ty: I->getType(), /*HandleUnknown=*/true);
446	if (VT == MVT::Other || !VT.isSimple())
447	// Unhandled type. Halt "fast" selection and bail.
448	return false;
449
450	// We only handle legal types. For example, on x86-32 the instruction
451	// selector contains all of the 64-bit instructions from x86-64,
452	// under the assumption that i64 won't be used if the target doesn't
453	// support it.
454	if (!TLI.isTypeLegal(VT)) {
455	// MVT::i1 is special. Allow AND, OR, or XOR because they
456	// don't require additional zeroing, which makes them easy.
457	if (VT == MVT::i1 && ISD::isBitwiseLogicOp(Opcode: ISDOpcode))
458	VT = TLI.getTypeToTransformTo(Context&: I->getContext(), VT);
459	else
460	return false;
461	}
462
463	// Check if the first operand is a constant, and handle it as "ri". At -O0,
464	// we don't have anything that canonicalizes operand order.
465	if (const auto *CI = dyn_cast<ConstantInt>(Val: I->getOperand(i: 0)))
466	if (isa<Instruction>(Val: I) && cast<Instruction>(Val: I)->isCommutative()) {
467	Register Op1 = getRegForValue(V: I->getOperand(i: 1));
468	if (!Op1)
469	return false;
470
471	Register ResultReg =
472	fastEmit_ri_(VT: VT.getSimpleVT(), Opcode: ISDOpcode, Op0: Op1, Imm: CI->getZExtValue(),
473	ImmType: VT.getSimpleVT());
474	if (!ResultReg)
475	return false;
476
477	// We successfully emitted code for the given LLVM Instruction.
478	updateValueMap(I, Reg: ResultReg);
479	return true;
480	}
481
482	Register Op0 = getRegForValue(V: I->getOperand(i: 0));
483	if (!Op0) // Unhandled operand. Halt "fast" selection and bail.
484	return false;
485
486	// Check if the second operand is a constant and handle it appropriately.
487	if (const auto *CI = dyn_cast<ConstantInt>(Val: I->getOperand(i: 1))) {
488	uint64_t Imm = CI->getSExtValue();
489
490	// Transform "sdiv exact X, 8" -> "sra X, 3".
491	if (ISDOpcode == ISD::SDIV && isa<BinaryOperator>(Val: I) &&
492	cast<BinaryOperator>(Val: I)->isExact() && isPowerOf2_64(Value: Imm)) {
493	Imm = Log2_64(Value: Imm);
494	ISDOpcode = ISD::SRA;
495	}
496
497	// Transform "urem x, pow2" -> "and x, pow2-1".
498	if (ISDOpcode == ISD::UREM && isa<BinaryOperator>(Val: I) &&
499	isPowerOf2_64(Value: Imm)) {
500	--Imm;
501	ISDOpcode = ISD::AND;
502	}
503
504	Register ResultReg = fastEmit_ri_(VT: VT.getSimpleVT(), Opcode: ISDOpcode, Op0, Imm,
505	ImmType: VT.getSimpleVT());
506	if (!ResultReg)
507	return false;
508
509	// We successfully emitted code for the given LLVM Instruction.
510	updateValueMap(I, Reg: ResultReg);
511	return true;
512	}
513
514	Register Op1 = getRegForValue(V: I->getOperand(i: 1));
515	if (!Op1) // Unhandled operand. Halt "fast" selection and bail.
516	return false;
517
518	// Now we have both operands in registers. Emit the instruction.
519	Register ResultReg = fastEmit_rr(VT: VT.getSimpleVT(), RetVT: VT.getSimpleVT(),
520	Opcode: ISDOpcode, Op0, Op1);
521	if (!ResultReg)
522	// Target-specific code wasn't able to find a machine opcode for
523	// the given ISD opcode and type. Halt "fast" selection and bail.
524	return false;
525
526	// We successfully emitted code for the given LLVM Instruction.
527	updateValueMap(I, Reg: ResultReg);
528	return true;
529	}
530
531	bool FastISel::selectGetElementPtr(const User *I) {
532	Register N = getRegForValue(V: I->getOperand(i: 0));
533	if (!N) // Unhandled operand. Halt "fast" selection and bail.
534	return false;
535
536	// FIXME: The code below does not handle vector GEPs. Halt "fast" selection
537	// and bail.
538	if (isa<VectorType>(Val: I->getType()))
539	return false;
540
541	// Keep a running tab of the total offset to coalesce multiple N = N + Offset
542	// into a single N = N + TotalOffset.
543	uint64_t TotalOffs = 0;
544	// FIXME: What's a good SWAG number for MaxOffs?
545	uint64_t MaxOffs = 2048;
546	MVT VT = TLI.getPointerTy(DL);
547	for (gep_type_iterator GTI = gep_type_begin(GEP: I), E = gep_type_end(GEP: I);
548	GTI != E; ++GTI) {
549	const Value *Idx = GTI.getOperand();
550	if (StructType *StTy = GTI.getStructTypeOrNull()) {
551	uint64_t Field = cast<ConstantInt>(Val: Idx)->getZExtValue();
552	if (Field) {
553	// N = N + Offset
554	TotalOffs += DL.getStructLayout(Ty: StTy)->getElementOffset(Idx: Field);
555	if (TotalOffs >= MaxOffs) {
556	N = fastEmit_ri_(VT, Opcode: ISD::ADD, Op0: N, Imm: TotalOffs, ImmType: VT);
557	if (!N) // Unhandled operand. Halt "fast" selection and bail.
558	return false;
559	TotalOffs = 0;
560	}
561	}
562	} else {
563	// If this is a constant subscript, handle it quickly.
564	if (const auto *CI = dyn_cast<ConstantInt>(Val: Idx)) {
565	if (CI->isZero())
566	continue;
567	// N = N + Offset
568	uint64_t IdxN = CI->getValue().sextOrTrunc(width: 64).getSExtValue();
569	TotalOffs += GTI.getSequentialElementStride(DL) * IdxN;
570	if (TotalOffs >= MaxOffs) {
571	N = fastEmit_ri_(VT, Opcode: ISD::ADD, Op0: N, Imm: TotalOffs, ImmType: VT);
572	if (!N) // Unhandled operand. Halt "fast" selection and bail.
573	return false;
574	TotalOffs = 0;
575	}
576	continue;
577	}
578	if (TotalOffs) {
579	N = fastEmit_ri_(VT, Opcode: ISD::ADD, Op0: N, Imm: TotalOffs, ImmType: VT);
580	if (!N) // Unhandled operand. Halt "fast" selection and bail.
581	return false;
582	TotalOffs = 0;
583	}
584
585	// N = N + Idx * ElementSize;
586	uint64_t ElementSize = GTI.getSequentialElementStride(DL);
587	Register IdxN = getRegForGEPIndex(Idx);
588	if (!IdxN) // Unhandled operand. Halt "fast" selection and bail.
589	return false;
590
591	if (ElementSize != 1) {
592	IdxN = fastEmit_ri_(VT, Opcode: ISD::MUL, Op0: IdxN, Imm: ElementSize, ImmType: VT);
593	if (!IdxN) // Unhandled operand. Halt "fast" selection and bail.
594	return false;
595	}
596	N = fastEmit_rr(VT, RetVT: VT, Opcode: ISD::ADD, Op0: N, Op1: IdxN);
597	if (!N) // Unhandled operand. Halt "fast" selection and bail.
598	return false;
599	}
600	}
601	if (TotalOffs) {
602	N = fastEmit_ri_(VT, Opcode: ISD::ADD, Op0: N, Imm: TotalOffs, ImmType: VT);
603	if (!N) // Unhandled operand. Halt "fast" selection and bail.
604	return false;
605	}
606
607	// We successfully emitted code for the given LLVM Instruction.
608	updateValueMap(I, Reg: N);
609	return true;
610	}
611
612	bool FastISel::addStackMapLiveVars(SmallVectorImpl<MachineOperand> &Ops,
613	const CallInst *CI, unsigned StartIdx) {
614	for (unsigned i = StartIdx, e = CI->arg_size(); i != e; ++i) {
615	Value *Val = CI->getArgOperand(i);
616	// Check for constants and encode them with a StackMaps::ConstantOp prefix.
617	if (const auto *C = dyn_cast<ConstantInt>(Val)) {
618	Ops.push_back(Elt: MachineOperand::CreateImm(Val: StackMaps::ConstantOp));
619	Ops.push_back(Elt: MachineOperand::CreateImm(Val: C->getSExtValue()));
620	} else if (isa<ConstantPointerNull>(Val)) {
621	Ops.push_back(Elt: MachineOperand::CreateImm(Val: StackMaps::ConstantOp));
622	Ops.push_back(Elt: MachineOperand::CreateImm(Val: 0));
623	} else if (auto *AI = dyn_cast<AllocaInst>(Val)) {
624	// Values coming from a stack location also require a special encoding,
625	// but that is added later on by the target specific frame index
626	// elimination implementation.
627	auto SI = FuncInfo.StaticAllocaMap.find(Val: AI);
628	if (SI != FuncInfo.StaticAllocaMap.end())
629	Ops.push_back(Elt: MachineOperand::CreateFI(Idx: SI->second));
630	else
631	return false;
632	} else {
633	Register Reg = getRegForValue(V: Val);
634	if (!Reg)
635	return false;
636	Ops.push_back(Elt: MachineOperand::CreateReg(Reg, /*isDef=*/false));
637	}
638	}
639	return true;
640	}
641
642	bool FastISel::selectStackmap(const CallInst *I) {
643	// void @llvm.experimental.stackmap(i64 <id>, i32 <numShadowBytes>,
644	// [live variables...])
645	assert(I->getCalledFunction()->getReturnType()->isVoidTy() &&
646	"Stackmap cannot return a value.");
647
648	// The stackmap intrinsic only records the live variables (the arguments
649	// passed to it) and emits NOPS (if requested). Unlike the patchpoint
650	// intrinsic, this won't be lowered to a function call. This means we don't
651	// have to worry about calling conventions and target-specific lowering code.
652	// Instead we perform the call lowering right here.
653	//
654	// CALLSEQ_START(0, 0...)
655	// STACKMAP(id, nbytes, ...)
656	// CALLSEQ_END(0, 0)
657	//
658	SmallVector<MachineOperand, 32> Ops;
659
660	// Add the <id> and <numBytes> constants.
661	assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::IDPos)) &&
662	"Expected a constant integer.");
663	const auto *ID = cast<ConstantInt>(Val: I->getOperand(i_nocapture: PatchPointOpers::IDPos));
664	Ops.push_back(Elt: MachineOperand::CreateImm(Val: ID->getZExtValue()));
665
666	assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::NBytesPos)) &&
667	"Expected a constant integer.");
668	const auto *NumBytes =
669	cast<ConstantInt>(Val: I->getOperand(i_nocapture: PatchPointOpers::NBytesPos));
670	Ops.push_back(Elt: MachineOperand::CreateImm(Val: NumBytes->getZExtValue()));
671
672	// Push live variables for the stack map (skipping the first two arguments
673	// <id> and <numBytes>).
674	if (!addStackMapLiveVars(Ops, CI: I, StartIdx: 2))
675	return false;
676
677	// We are not adding any register mask info here, because the stackmap doesn't
678	// clobber anything.
679
680	// Add scratch registers as implicit def and early clobber.
681	CallingConv::ID CC = I->getCallingConv();
682	const MCPhysReg *ScratchRegs = TLI.getScratchRegisters(CC);
683	for (unsigned i = 0; ScratchRegs[i]; ++i)
684	Ops.push_back(Elt: MachineOperand::CreateReg(
685	Reg: ScratchRegs[i], /*isDef=*/true, /*isImp=*/true, /*isKill=*/false,
686	/*isDead=*/false, /*isUndef=*/false, /*isEarlyClobber=*/true));
687
688	// Issue CALLSEQ_START
689	unsigned AdjStackDown = TII.getCallFrameSetupOpcode();
690	auto Builder =
691	BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: AdjStackDown));
692	const MCInstrDesc &MCID = Builder.getInstr()->getDesc();
693	for (unsigned I = 0, E = MCID.getNumOperands(); I < E; ++I)
694	Builder.addImm(Val: 0);
695
696	// Issue STACKMAP.
697	MachineInstrBuilder MIB = BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
698	MCID: TII.get(Opcode: TargetOpcode::STACKMAP));
699	for (auto const &MO : Ops)
700	MIB.add(MO);
701
702	// Issue CALLSEQ_END
703	unsigned AdjStackUp = TII.getCallFrameDestroyOpcode();
704	BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: AdjStackUp))
705	.addImm(Val: 0)
706	.addImm(Val: 0);
707
708	// Inform the Frame Information that we have a stackmap in this function.
709	FuncInfo.MF->getFrameInfo().setHasStackMap();
710
711	return true;
712	}
713
714	/// Lower an argument list according to the target calling convention.
715	///
716	/// This is a helper for lowering intrinsics that follow a target calling
717	/// convention or require stack pointer adjustment. Only a subset of the
718	/// intrinsic's operands need to participate in the calling convention.
719	bool FastISel::lowerCallOperands(const CallInst *CI, unsigned ArgIdx,
720	unsigned NumArgs, const Value *Callee,
721	bool ForceRetVoidTy, CallLoweringInfo &CLI) {
722	ArgListTy Args;
723	Args.reserve(n: NumArgs);
724
725	// Populate the argument list.
726	for (unsigned ArgI = ArgIdx, ArgE = ArgIdx + NumArgs; ArgI != ArgE; ++ArgI) {
727	Value *V = CI->getOperand(i_nocapture: ArgI);
728
729	assert(!V->getType()->isEmptyTy() && "Empty type passed to intrinsic.");
730
731	ArgListEntry Entry;
732	Entry.Val = V;
733	Entry.Ty = V->getType();
734	Entry.setAttributes(Call: CI, ArgIdx: ArgI);
735	Args.push_back(x: Entry);
736	}
737
738	Type *RetTy = ForceRetVoidTy ? Type::getVoidTy(C&: CI->getType()->getContext())
739	: CI->getType();
740	CLI.setCallee(CC: CI->getCallingConv(), ResultTy: RetTy, Target: Callee, ArgsList: std::move(Args), FixedArgs: NumArgs);
741
742	return lowerCallTo(CLI);
743	}
744
745	FastISel::CallLoweringInfo &FastISel::CallLoweringInfo::setCallee(
746	const DataLayout &DL, MCContext &Ctx, CallingConv::ID CC, Type *ResultTy,
747	StringRef Target, ArgListTy &&ArgsList, unsigned FixedArgs) {
748	SmallString<32> MangledName;
749	Mangler::getNameWithPrefix(OutName&: MangledName, GVName: Target, DL);
750	MCSymbol *Sym = Ctx.getOrCreateSymbol(Name: MangledName);
751	return setCallee(CC, ResultTy, Target: Sym, ArgsList: std::move(ArgsList), FixedArgs);
752	}
753
754	bool FastISel::selectPatchpoint(const CallInst *I) {
755	// <ty> @llvm.experimental.patchpoint.<ty>(i64 <id>,
756	// i32 <numBytes>,
757	// i8* <target>,
758	// i32 <numArgs>,
759	// [Args...],
760	// [live variables...])
761	CallingConv::ID CC = I->getCallingConv();
762	bool IsAnyRegCC = CC == CallingConv::AnyReg;
763	bool HasDef = !I->getType()->isVoidTy();
764	Value *Callee = I->getOperand(i_nocapture: PatchPointOpers::TargetPos)->stripPointerCasts();
765
766	// Check if we can lower the return type when using anyregcc.
767	MVT ValueType;
768	if (IsAnyRegCC && HasDef) {
769	ValueType = TLI.getSimpleValueType(DL, Ty: I->getType(), /*AllowUnknown=*/true);
770	if (ValueType == MVT::Other)
771	return false;
772	}
773
774	// Get the real number of arguments participating in the call <numArgs>
775	assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::NArgPos)) &&
776	"Expected a constant integer.");
777	const auto *NumArgsVal =
778	cast<ConstantInt>(Val: I->getOperand(i_nocapture: PatchPointOpers::NArgPos));
779	unsigned NumArgs = NumArgsVal->getZExtValue();
780
781	// Skip the four meta args: <id>, <numNopBytes>, <target>, <numArgs>
782	// This includes all meta-operands up to but not including CC.
783	unsigned NumMetaOpers = PatchPointOpers::CCPos;
784	assert(I->arg_size() >= NumMetaOpers + NumArgs &&
785	"Not enough arguments provided to the patchpoint intrinsic");
786
787	// For AnyRegCC the arguments are lowered later on manually.
788	unsigned NumCallArgs = IsAnyRegCC ? 0 : NumArgs;
789	CallLoweringInfo CLI;
790	CLI.setIsPatchPoint();
791	if (!lowerCallOperands(CI: I, ArgIdx: NumMetaOpers, NumArgs: NumCallArgs, Callee, ForceRetVoidTy: IsAnyRegCC, CLI))
792	return false;
793
794	assert(CLI.Call && "No call instruction specified.");
795
796	SmallVector<MachineOperand, 32> Ops;
797
798	// Add an explicit result reg if we use the anyreg calling convention.
799	if (IsAnyRegCC && HasDef) {
800	assert(CLI.NumResultRegs == 0 && "Unexpected result register.");
801	assert(ValueType.isValid());
802	CLI.ResultReg = createResultReg(RC: TLI.getRegClassFor(VT: ValueType));
803	CLI.NumResultRegs = 1;
804	Ops.push_back(Elt: MachineOperand::CreateReg(Reg: CLI.ResultReg, /*isDef=*/true));
805	}
806
807	// Add the <id> and <numBytes> constants.
808	assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::IDPos)) &&
809	"Expected a constant integer.");
810	const auto *ID = cast<ConstantInt>(Val: I->getOperand(i_nocapture: PatchPointOpers::IDPos));
811	Ops.push_back(Elt: MachineOperand::CreateImm(Val: ID->getZExtValue()));
812
813	assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::NBytesPos)) &&
814	"Expected a constant integer.");
815	const auto *NumBytes =
816	cast<ConstantInt>(Val: I->getOperand(i_nocapture: PatchPointOpers::NBytesPos));
817	Ops.push_back(Elt: MachineOperand::CreateImm(Val: NumBytes->getZExtValue()));
818
819	// Add the call target.
820	if (const auto *C = dyn_cast<IntToPtrInst>(Val: Callee)) {
821	uint64_t CalleeConstAddr =
822	cast<ConstantInt>(Val: C->getOperand(i_nocapture: 0))->getZExtValue();
823	Ops.push_back(Elt: MachineOperand::CreateImm(Val: CalleeConstAddr));
824	} else if (const auto *C = dyn_cast<ConstantExpr>(Val: Callee)) {
825	if (C->getOpcode() == Instruction::IntToPtr) {
826	uint64_t CalleeConstAddr =
827	cast<ConstantInt>(Val: C->getOperand(i_nocapture: 0))->getZExtValue();
828	Ops.push_back(Elt: MachineOperand::CreateImm(Val: CalleeConstAddr));
829	} else
830	llvm_unreachable("Unsupported ConstantExpr.");
831	} else if (const auto *GV = dyn_cast<GlobalValue>(Val: Callee)) {
832	Ops.push_back(Elt: MachineOperand::CreateGA(GV, Offset: 0));
833	} else if (isa<ConstantPointerNull>(Val: Callee))
834	Ops.push_back(Elt: MachineOperand::CreateImm(Val: 0));
835	else
836	llvm_unreachable("Unsupported callee address.");
837
838	// Adjust <numArgs> to account for any arguments that have been passed on
839	// the stack instead.
840	unsigned NumCallRegArgs = IsAnyRegCC ? NumArgs : CLI.OutRegs.size();
841	Ops.push_back(Elt: MachineOperand::CreateImm(Val: NumCallRegArgs));
842
843	// Add the calling convention
844	Ops.push_back(Elt: MachineOperand::CreateImm(Val: (unsigned)CC));
845
846	// Add the arguments we omitted previously. The register allocator should
847	// place these in any free register.
848	if (IsAnyRegCC) {
849	for (unsigned i = NumMetaOpers, e = NumMetaOpers + NumArgs; i != e; ++i) {
850	Register Reg = getRegForValue(V: I->getArgOperand(i));
851	if (!Reg)
852	return false;
853	Ops.push_back(Elt: MachineOperand::CreateReg(Reg, /*isDef=*/false));
854	}
855	}
856
857	// Push the arguments from the call instruction.
858	for (auto Reg : CLI.OutRegs)
859	Ops.push_back(Elt: MachineOperand::CreateReg(Reg, /*isDef=*/false));
860
861	// Push live variables for the stack map.
862	if (!addStackMapLiveVars(Ops, CI: I, StartIdx: NumMetaOpers + NumArgs))
863	return false;
864
865	// Push the register mask info.
866	Ops.push_back(Elt: MachineOperand::CreateRegMask(
867	Mask: TRI.getCallPreservedMask(MF: *FuncInfo.MF, CC)));
868
869	// Add scratch registers as implicit def and early clobber.
870	const MCPhysReg *ScratchRegs = TLI.getScratchRegisters(CC);
871	for (unsigned i = 0; ScratchRegs[i]; ++i)
872	Ops.push_back(Elt: MachineOperand::CreateReg(
873	Reg: ScratchRegs[i], /*isDef=*/true, /*isImp=*/true, /*isKill=*/false,
874	/*isDead=*/false, /*isUndef=*/false, /*isEarlyClobber=*/true));
875
876	// Add implicit defs (return values).
877	for (auto Reg : CLI.InRegs)
878	Ops.push_back(Elt: MachineOperand::CreateReg(Reg, /*isDef=*/true,
879	/*isImp=*/true));
880
881	// Insert the patchpoint instruction before the call generated by the target.
882	MachineInstrBuilder MIB = BuildMI(BB&: *FuncInfo.MBB, I: CLI.Call, MIMD,
883	MCID: TII.get(Opcode: TargetOpcode::PATCHPOINT));
884
885	for (auto &MO : Ops)
886	MIB.add(MO);
887
888	MIB->setPhysRegsDeadExcept(UsedRegs: CLI.InRegs, TRI);
889
890	// Delete the original call instruction.
891	CLI.Call->eraseFromParent();
892
893	// Inform the Frame Information that we have a patchpoint in this function.
894	FuncInfo.MF->getFrameInfo().setHasPatchPoint();
895
896	if (CLI.NumResultRegs)
897	updateValueMap(I, Reg: CLI.ResultReg, NumRegs: CLI.NumResultRegs);
898	return true;
899	}
900
901	bool FastISel::selectXRayCustomEvent(const CallInst *I) {
902	const auto &Triple = TM.getTargetTriple();
903	if (Triple.isAArch64(PointerWidth: 64) && Triple.getArch() != Triple::x86_64)
904	return true; // don't do anything to this instruction.
905	SmallVector<MachineOperand, 8> Ops;
906	Ops.push_back(Elt: MachineOperand::CreateReg(Reg: getRegForValue(V: I->getArgOperand(i: 0)),
907	/*isDef=*/false));
908	Ops.push_back(Elt: MachineOperand::CreateReg(Reg: getRegForValue(V: I->getArgOperand(i: 1)),
909	/*isDef=*/false));
910	MachineInstrBuilder MIB =
911	BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
912	MCID: TII.get(Opcode: TargetOpcode::PATCHABLE_EVENT_CALL));
913	for (auto &MO : Ops)
914	MIB.add(MO);
915
916	// Insert the Patchable Event Call instruction, that gets lowered properly.
917	return true;
918	}
919
920	bool FastISel::selectXRayTypedEvent(const CallInst *I) {
921	const auto &Triple = TM.getTargetTriple();
922	if (Triple.isAArch64(PointerWidth: 64) && Triple.getArch() != Triple::x86_64)
923	return true; // don't do anything to this instruction.
924	SmallVector<MachineOperand, 8> Ops;
925	Ops.push_back(Elt: MachineOperand::CreateReg(Reg: getRegForValue(V: I->getArgOperand(i: 0)),
926	/*isDef=*/false));
927	Ops.push_back(Elt: MachineOperand::CreateReg(Reg: getRegForValue(V: I->getArgOperand(i: 1)),
928	/*isDef=*/false));
929	Ops.push_back(Elt: MachineOperand::CreateReg(Reg: getRegForValue(V: I->getArgOperand(i: 2)),
930	/*isDef=*/false));
931	MachineInstrBuilder MIB =
932	BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
933	MCID: TII.get(Opcode: TargetOpcode::PATCHABLE_TYPED_EVENT_CALL));
934	for (auto &MO : Ops)
935	MIB.add(MO);
936
937	// Insert the Patchable Typed Event Call instruction, that gets lowered properly.
938	return true;
939	}
940
941	/// Returns an AttributeList representing the attributes applied to the return
942	/// value of the given call.
943	static AttributeList getReturnAttrs(FastISel::CallLoweringInfo &CLI) {
944	SmallVector<Attribute::AttrKind, 2> Attrs;
945	if (CLI.RetSExt)
946	Attrs.push_back(Attribute::Elt: SExt);
947	if (CLI.RetZExt)
948	Attrs.push_back(Attribute::Elt: ZExt);
949	if (CLI.IsInReg)
950	Attrs.push_back(Attribute::Elt: InReg);
951
952	return AttributeList::get(C&: CLI.RetTy->getContext(), Index: AttributeList::ReturnIndex,
953	Kinds: Attrs);
954	}
955
956	bool FastISel::lowerCallTo(const CallInst *CI, const char *SymName,
957	unsigned NumArgs) {
958	MCContext &Ctx = MF->getContext();
959	SmallString<32> MangledName;
960	Mangler::getNameWithPrefix(OutName&: MangledName, GVName: SymName, DL);
961	MCSymbol *Sym = Ctx.getOrCreateSymbol(Name: MangledName);
962	return lowerCallTo(CI, Symbol: Sym, NumArgs);
963	}
964
965	bool FastISel::lowerCallTo(const CallInst *CI, MCSymbol *Symbol,
966	unsigned NumArgs) {
967	FunctionType *FTy = CI->getFunctionType();
968	Type *RetTy = CI->getType();
969
970	ArgListTy Args;
971	Args.reserve(n: NumArgs);
972
973	// Populate the argument list.
974	// Attributes for args start at offset 1, after the return attribute.
975	for (unsigned ArgI = 0; ArgI != NumArgs; ++ArgI) {
976	Value *V = CI->getOperand(i_nocapture: ArgI);
977
978	assert(!V->getType()->isEmptyTy() && "Empty type passed to intrinsic.");
979
980	ArgListEntry Entry;
981	Entry.Val = V;
982	Entry.Ty = V->getType();
983	Entry.setAttributes(Call: CI, ArgIdx: ArgI);
984	Args.push_back(x: Entry);
985	}
986	TLI.markLibCallAttributes(MF, CC: CI->getCallingConv(), Args);
987
988	CallLoweringInfo CLI;
989	CLI.setCallee(ResultTy: RetTy, FuncTy: FTy, Target: Symbol, ArgsList: std::move(Args), Call: *CI, FixedArgs: NumArgs);
990
991	return lowerCallTo(CLI);
992	}
993
994	bool FastISel::lowerCallTo(CallLoweringInfo &CLI) {
995	// Handle the incoming return values from the call.
996	CLI.clearIns();
997	SmallVector<EVT, 4> RetTys;
998	ComputeValueVTs(TLI, DL, Ty: CLI.RetTy, ValueVTs&: RetTys);
999
1000	SmallVector<ISD::OutputArg, 4> Outs;
1001	GetReturnInfo(CC: CLI.CallConv, ReturnType: CLI.RetTy, attr: getReturnAttrs(CLI), Outs, TLI, DL);
1002
1003	bool CanLowerReturn = TLI.CanLowerReturn(
1004	CLI.CallConv, *FuncInfo.MF, CLI.IsVarArg, Outs, CLI.RetTy->getContext());
1005
1006	// FIXME: sret demotion isn't supported yet - bail out.
1007	if (!CanLowerReturn)
1008	return false;
1009
1010	for (EVT VT : RetTys) {
1011	MVT RegisterVT = TLI.getRegisterType(Context&: CLI.RetTy->getContext(), VT);
1012	unsigned NumRegs = TLI.getNumRegisters(Context&: CLI.RetTy->getContext(), VT);
1013	for (unsigned i = 0; i != NumRegs; ++i) {
1014	ISD::InputArg MyFlags;
1015	MyFlags.VT = RegisterVT;
1016	MyFlags.ArgVT = VT;
1017	MyFlags.Used = CLI.IsReturnValueUsed;
1018	if (CLI.RetSExt)
1019	MyFlags.Flags.setSExt();
1020	if (CLI.RetZExt)
1021	MyFlags.Flags.setZExt();
1022	if (CLI.IsInReg)
1023	MyFlags.Flags.setInReg();
1024	CLI.Ins.push_back(Elt: MyFlags);
1025	}
1026	}
1027
1028	// Handle all of the outgoing arguments.
1029	CLI.clearOuts();
1030	for (auto &Arg : CLI.getArgs()) {
1031	Type *FinalType = Arg.Ty;
1032	if (Arg.IsByVal)
1033	FinalType = Arg.IndirectType;
1034	bool NeedsRegBlock = TLI.functionArgumentNeedsConsecutiveRegisters(
1035	Ty: FinalType, CallConv: CLI.CallConv, isVarArg: CLI.IsVarArg, DL);
1036
1037	ISD::ArgFlagsTy Flags;
1038	if (Arg.IsZExt)
1039	Flags.setZExt();
1040	if (Arg.IsSExt)
1041	Flags.setSExt();
1042	if (Arg.IsInReg)
1043	Flags.setInReg();
1044	if (Arg.IsSRet)
1045	Flags.setSRet();
1046	if (Arg.IsSwiftSelf)
1047	Flags.setSwiftSelf();
1048	if (Arg.IsSwiftAsync)
1049	Flags.setSwiftAsync();
1050	if (Arg.IsSwiftError)
1051	Flags.setSwiftError();
1052	if (Arg.IsCFGuardTarget)
1053	Flags.setCFGuardTarget();
1054	if (Arg.IsByVal)
1055	Flags.setByVal();
1056	if (Arg.IsInAlloca) {
1057	Flags.setInAlloca();
1058	// Set the byval flag for CCAssignFn callbacks that don't know about
1059	// inalloca. This way we can know how many bytes we should've allocated
1060	// and how many bytes a callee cleanup function will pop. If we port
1061	// inalloca to more targets, we'll have to add custom inalloca handling in
1062	// the various CC lowering callbacks.
1063	Flags.setByVal();
1064	}
1065	if (Arg.IsPreallocated) {
1066	Flags.setPreallocated();
1067	// Set the byval flag for CCAssignFn callbacks that don't know about
1068	// preallocated. This way we can know how many bytes we should've
1069	// allocated and how many bytes a callee cleanup function will pop. If we
1070	// port preallocated to more targets, we'll have to add custom
1071	// preallocated handling in the various CC lowering callbacks.
1072	Flags.setByVal();
1073	}
1074	MaybeAlign MemAlign = Arg.Alignment;
1075	if (Arg.IsByVal || Arg.IsInAlloca || Arg.IsPreallocated) {
1076	unsigned FrameSize = DL.getTypeAllocSize(Ty: Arg.IndirectType);
1077
1078	// For ByVal, alignment should come from FE. BE will guess if this info
1079	// is not there, but there are cases it cannot get right.
1080	if (!MemAlign)
1081	MemAlign = Align(TLI.getByValTypeAlignment(Ty: Arg.IndirectType, DL));
1082	Flags.setByValSize(FrameSize);
1083	} else if (!MemAlign) {
1084	MemAlign = DL.getABITypeAlign(Ty: Arg.Ty);
1085	}
1086	Flags.setMemAlign(*MemAlign);
1087	if (Arg.IsNest)
1088	Flags.setNest();
1089	if (NeedsRegBlock)
1090	Flags.setInConsecutiveRegs();
1091	Flags.setOrigAlign(DL.getABITypeAlign(Ty: Arg.Ty));
1092	CLI.OutVals.push_back(Elt: Arg.Val);
1093	CLI.OutFlags.push_back(Elt: Flags);
1094	}
1095
1096	if (!fastLowerCall(CLI))
1097	return false;
1098
1099	// Set all unused physreg defs as dead.
1100	assert(CLI.Call && "No call instruction specified.");
1101	CLI.Call->setPhysRegsDeadExcept(UsedRegs: CLI.InRegs, TRI);
1102
1103	if (CLI.NumResultRegs && CLI.CB)
1104	updateValueMap(I: CLI.CB, Reg: CLI.ResultReg, NumRegs: CLI.NumResultRegs);
1105
1106	// Set labels for heapallocsite call.
1107	if (CLI.CB)
1108	if (MDNode *MD = CLI.CB->getMetadata(Kind: "heapallocsite"))
1109	CLI.Call->setHeapAllocMarker(MF&: *MF, MD);
1110
1111	return true;
1112	}
1113
1114	bool FastISel::lowerCall(const CallInst *CI) {
1115	FunctionType *FuncTy = CI->getFunctionType();
1116	Type *RetTy = CI->getType();
1117
1118	ArgListTy Args;
1119	ArgListEntry Entry;
1120	Args.reserve(n: CI->arg_size());
1121
1122	for (auto i = CI->arg_begin(), e = CI->arg_end(); i != e; ++i) {
1123	Value *V = *i;
1124
1125	// Skip empty types
1126	if (V->getType()->isEmptyTy())
1127	continue;
1128
1129	Entry.Val = V;
1130	Entry.Ty = V->getType();
1131
1132	// Skip the first return-type Attribute to get to params.
1133	Entry.setAttributes(Call: CI, ArgIdx: i - CI->arg_begin());
1134	Args.push_back(x: Entry);
1135	}
1136
1137	// Check if target-independent constraints permit a tail call here.
1138	// Target-dependent constraints are checked within fastLowerCall.
1139	bool IsTailCall = CI->isTailCall();
1140	if (IsTailCall && !isInTailCallPosition(Call: *CI, TM))
1141	IsTailCall = false;
1142	if (IsTailCall && !CI->isMustTailCall() &&
1143	MF->getFunction().getFnAttribute(Kind: "disable-tail-calls").getValueAsBool())
1144	IsTailCall = false;
1145
1146	CallLoweringInfo CLI;
1147	CLI.setCallee(ResultTy: RetTy, FuncTy, Target: CI->getCalledOperand(), ArgsList: std::move(Args), Call: *CI)
1148	.setTailCall(IsTailCall);
1149
1150	diagnoseDontCall(CI: *CI);
1151
1152	return lowerCallTo(CLI);
1153	}
1154
1155	bool FastISel::selectCall(const User *I) {
1156	const CallInst *Call = cast<CallInst>(Val: I);
1157
1158	// Handle simple inline asms.
1159	if (const InlineAsm *IA = dyn_cast<InlineAsm>(Val: Call->getCalledOperand())) {
1160	// Don't attempt to handle constraints.
1161	if (!IA->getConstraintString().empty())
1162	return false;
1163
1164	unsigned ExtraInfo = 0;
1165	if (IA->hasSideEffects())
1166	ExtraInfo |= InlineAsm::Extra_HasSideEffects;
1167	if (IA->isAlignStack())
1168	ExtraInfo |= InlineAsm::Extra_IsAlignStack;
1169	if (Call->isConvergent())
1170	ExtraInfo |= InlineAsm::Extra_IsConvergent;
1171	ExtraInfo |= IA->getDialect() * InlineAsm::Extra_AsmDialect;
1172
1173	MachineInstrBuilder MIB = BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
1174	MCID: TII.get(Opcode: TargetOpcode::INLINEASM));
1175	MIB.addExternalSymbol(FnName: IA->getAsmString().c_str());
1176	MIB.addImm(Val: ExtraInfo);
1177
1178	const MDNode *SrcLoc = Call->getMetadata(Kind: "srcloc");
1179	if (SrcLoc)
1180	MIB.addMetadata(MD: SrcLoc);
1181
1182	return true;
1183	}
1184
1185	// Handle intrinsic function calls.
1186	if (const auto *II = dyn_cast<IntrinsicInst>(Val: Call))
1187	return selectIntrinsicCall(II);
1188
1189	return lowerCall(CI: Call);
1190	}
1191
1192	void FastISel::handleDbgInfo(const Instruction *II) {
1193	if (!II->hasDbgRecords())
1194	return;
1195
1196	// Clear any metadata.
1197	MIMD = MIMetadata();
1198
1199	// Reverse order of debug records, because fast-isel walks through backwards.
1200	for (DbgRecord &DR : llvm::reverse(C: II->getDbgRecordRange())) {
1201	flushLocalValueMap();
1202	recomputeInsertPt();
1203
1204	if (DbgLabelRecord *DLR = dyn_cast<DbgLabelRecord>(Val: &DR)) {
1205	assert(DLR->getLabel() && "Missing label");
1206	if (!FuncInfo.MF->getMMI().hasDebugInfo()) {
1207	LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DLR << "\n");
1208	continue;
1209	}
1210
1211	BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD: DLR->getDebugLoc(),
1212	MCID: TII.get(Opcode: TargetOpcode::DBG_LABEL))
1213	.addMetadata(MD: DLR->getLabel());
1214	continue;
1215	}
1216
1217	DbgVariableRecord &DVR = cast<DbgVariableRecord>(Val&: DR);
1218
1219	Value *V = nullptr;
1220	if (!DVR.hasArgList())
1221	V = DVR.getVariableLocationOp(OpIdx: 0);
1222
1223	bool Res = false;
1224	if (DVR.getType() == DbgVariableRecord::LocationType::Value ||
1225	DVR.getType() == DbgVariableRecord::LocationType::Assign) {
1226	Res = lowerDbgValue(V, Expr: DVR.getExpression(), Var: DVR.getVariable(),
1227	DL: DVR.getDebugLoc());
1228	} else {
1229	assert(DVR.getType() == DbgVariableRecord::LocationType::Declare);
1230	if (FuncInfo.PreprocessedDVRDeclares.contains(Ptr: &DVR))
1231	continue;
1232	Res = lowerDbgDeclare(V, Expr: DVR.getExpression(), Var: DVR.getVariable(),
1233	DL: DVR.getDebugLoc());
1234	}
1235
1236	if (!Res)
1237	LLVM_DEBUG(dbgs() << "Dropping debug-info for " << DVR << "\n";);
1238	}
1239	}
1240
1241	bool FastISel::lowerDbgValue(const Value *V, DIExpression *Expr,
1242	DILocalVariable *Var, const DebugLoc &DL) {
1243	// This form of DBG_VALUE is target-independent.
1244	const MCInstrDesc &II = TII.get(Opcode: TargetOpcode::DBG_VALUE);
1245	if (!V || isa<UndefValue>(Val: V)) {
1246	// DI is either undef or cannot produce a valid DBG_VALUE, so produce an
1247	// undef DBG_VALUE to terminate any prior location.
1248	BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, DL, MCID: II, IsIndirect: false, Reg: 0U, Variable: Var, Expr);
1249	return true;
1250	}
1251	if (const auto *CI = dyn_cast<ConstantInt>(Val: V)) {
1252	// See if there's an expression to constant-fold.
1253	if (Expr)
1254	std::tie(args&: Expr, args&: CI) = Expr->constantFold(CI);
1255	if (CI->getBitWidth() > 64)
1256	BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD: DL, MCID: II)
1257	.addCImm(Val: CI)
1258	.addImm(Val: 0U)
1259	.addMetadata(MD: Var)
1260	.addMetadata(MD: Expr);
1261	else
1262	BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD: DL, MCID: II)
1263	.addImm(Val: CI->getZExtValue())
1264	.addImm(Val: 0U)
1265	.addMetadata(MD: Var)
1266	.addMetadata(MD: Expr);
1267	return true;
1268	}
1269	if (const auto *CF = dyn_cast<ConstantFP>(Val: V)) {
1270	BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD: DL, MCID: II)
1271	.addFPImm(Val: CF)
1272	.addImm(Val: 0U)
1273	.addMetadata(MD: Var)
1274	.addMetadata(MD: Expr);
1275	return true;
1276	}
1277	if (const auto *Arg = dyn_cast<Argument>(Val: V);
1278	Arg && Expr && Expr->isEntryValue()) {
1279	// As per the Verifier, this case is only valid for swift async Args.
1280	assert(Arg->hasAttribute(Attribute::AttrKind::SwiftAsync));
1281
1282	Register Reg = getRegForValue(V: Arg);
1283	for (auto [PhysReg, VirtReg] : FuncInfo.RegInfo->liveins())
1284	if (Reg == VirtReg || Reg == PhysReg) {
1285	BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, DL, MCID: II, IsIndirect: false /*IsIndirect*/,
1286	Reg: PhysReg, Variable: Var, Expr);
1287	return true;
1288	}
1289
1290	LLVM_DEBUG(dbgs() << "Dropping dbg.value: expression is entry_value but "
1291	"couldn't find a physical register\n");
1292	return false;
1293	}
1294	if (auto SI = FuncInfo.StaticAllocaMap.find(Val: dyn_cast<AllocaInst>(Val: V));
1295	SI != FuncInfo.StaticAllocaMap.end()) {
1296	MachineOperand FrameIndexOp = MachineOperand::CreateFI(Idx: SI->second);
1297	bool IsIndirect = false;
1298	BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, DL, MCID: II, IsIndirect, MOs: FrameIndexOp,
1299	Variable: Var, Expr);
1300	return true;
1301	}
1302	if (Register Reg = lookUpRegForValue(V)) {
1303	// FIXME: This does not handle register-indirect values at offset 0.
1304	if (!FuncInfo.MF->useDebugInstrRef()) {
1305	bool IsIndirect = false;
1306	BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, DL, MCID: II, IsIndirect, Reg, Variable: Var,
1307	Expr);
1308	return true;
1309	}
1310	// If using instruction referencing, produce this as a DBG_INSTR_REF,
1311	// to be later patched up by finalizeDebugInstrRefs.
1312	SmallVector<MachineOperand, 1> MOs({MachineOperand::CreateReg(
1313	/* Reg */ Reg, /* isDef */ false, /* isImp */ false,
1314	/* isKill */ false, /* isDead */ false,
1315	/* isUndef */ false, /* isEarlyClobber */ false,
1316	/* SubReg */ 0, /* isDebug */ true)});
1317	SmallVector<uint64_t, 2> Ops({dwarf::DW_OP_LLVM_arg, 0});
1318	auto *NewExpr = DIExpression::prependOpcodes(Expr, Ops);
1319	BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, DL,
1320	MCID: TII.get(Opcode: TargetOpcode::DBG_INSTR_REF), /*IsIndirect*/ false, MOs,
1321	Variable: Var, Expr: NewExpr);
1322	return true;
1323	}
1324	return false;
1325	}
1326
1327	bool FastISel::lowerDbgDeclare(const Value *Address, DIExpression *Expr,
1328	DILocalVariable *Var, const DebugLoc &DL) {
1329	if (!Address || isa<UndefValue>(Val: Address)) {
1330	LLVM_DEBUG(dbgs() << "Dropping debug info (bad/undef address)\n");
1331	return false;
1332	}
1333
1334	std::optional<MachineOperand> Op;
1335	if (Register Reg = lookUpRegForValue(V: Address))
1336	Op = MachineOperand::CreateReg(Reg, isDef: false);
1337
1338	// If we have a VLA that has a "use" in a metadata node that's then used
1339	// here but it has no other uses, then we have a problem. E.g.,
1340	//
1341	// int foo (const int *x) {
1342	// char a[*x];
1343	// return 0;
1344	// }
1345	//
1346	// If we assign 'a' a vreg and fast isel later on has to use the selection
1347	// DAG isel, it will want to copy the value to the vreg. However, there are
1348	// no uses, which goes counter to what selection DAG isel expects.
1349	if (!Op && !Address->use_empty() && isa<Instruction>(Val: Address) &&
1350	(!isa<AllocaInst>(Val: Address) ||
1351	!FuncInfo.StaticAllocaMap.count(Val: cast<AllocaInst>(Val: Address))))
1352	Op = MachineOperand::CreateReg(Reg: FuncInfo.InitializeRegForValue(V: Address),
1353	isDef: false);
1354
1355	if (Op) {
1356	assert(Var->isValidLocationForIntrinsic(DL) &&
1357	"Expected inlined-at fields to agree");
1358	if (FuncInfo.MF->useDebugInstrRef() && Op->isReg()) {
1359	// If using instruction referencing, produce this as a DBG_INSTR_REF,
1360	// to be later patched up by finalizeDebugInstrRefs. Tack a deref onto
1361	// the expression, we don't have an "indirect" flag in DBG_INSTR_REF.
1362	SmallVector<uint64_t, 3> Ops(
1363	{dwarf::DW_OP_LLVM_arg, 0, dwarf::DW_OP_deref});
1364	auto *NewExpr = DIExpression::prependOpcodes(Expr, Ops);
1365	BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, DL,
1366	MCID: TII.get(Opcode: TargetOpcode::DBG_INSTR_REF), /*IsIndirect*/ false, MOs: *Op,
1367	Variable: Var, Expr: NewExpr);
1368	return true;
1369	}
1370
1371	// A dbg.declare describes the address of a source variable, so lower it
1372	// into an indirect DBG_VALUE.
1373	BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, DL,
1374	MCID: TII.get(Opcode: TargetOpcode::DBG_VALUE), /*IsIndirect*/ true, MOs: *Op, Variable: Var,
1375	Expr);
1376	return true;
1377	}
1378
1379	// We can't yet handle anything else here because it would require
1380	// generating code, thus altering codegen because of debug info.
1381	LLVM_DEBUG(
1382	dbgs() << "Dropping debug info (no materialized reg for address)\n");
1383	return false;
1384	}
1385
1386	bool FastISel::selectIntrinsicCall(const IntrinsicInst *II) {
1387	switch (II->getIntrinsicID()) {
1388	default:
1389	break;
1390	// At -O0 we don't care about the lifetime intrinsics.
1391	case Intrinsic::lifetime_start:
1392	case Intrinsic::lifetime_end:
1393	// The donothing intrinsic does, well, nothing.
1394	case Intrinsic::donothing:
1395	// Neither does the sideeffect intrinsic.
1396	case Intrinsic::sideeffect:
1397	// Neither does the assume intrinsic; it's also OK not to codegen its operand.
1398	case Intrinsic::assume:
1399	// Neither does the llvm.experimental.noalias.scope.decl intrinsic
1400	case Intrinsic::experimental_noalias_scope_decl:
1401	return true;
1402	case Intrinsic::dbg_declare: {
1403	const DbgDeclareInst *DI = cast<DbgDeclareInst>(Val: II);
1404	assert(DI->getVariable() && "Missing variable");
1405	if (!FuncInfo.MF->getMMI().hasDebugInfo()) {
1406	LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI
1407	<< " (!hasDebugInfo)\n");
1408	return true;
1409	}
1410
1411	if (FuncInfo.PreprocessedDbgDeclares.contains(Ptr: DI))
1412	return true;
1413
1414	const Value *Address = DI->getAddress();
1415	if (!lowerDbgDeclare(Address, Expr: DI->getExpression(), Var: DI->getVariable(),
1416	DL: MIMD.getDL()))
1417	LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI);
1418
1419	return true;
1420	}
1421	case Intrinsic::dbg_assign:
1422	// A dbg.assign is a dbg.value with more information, typically produced
1423	// during optimisation. If one reaches fastisel then something odd has
1424	// happened (such as an optimised function being always-inlined into an
1425	// optnone function). We will not be using the extra information in the
1426	// dbg.assign in that case, just use its dbg.value fields.
1427	LLVM_FALLTHROUGH;
1428	case Intrinsic::dbg_value: {
1429	// This form of DBG_VALUE is target-independent.
1430	const DbgValueInst *DI = cast<DbgValueInst>(Val: II);
1431	const Value *V = DI->getValue();
1432	DIExpression *Expr = DI->getExpression();
1433	DILocalVariable *Var = DI->getVariable();
1434	if (DI->hasArgList())
1435	// Signal that we don't have a location for this.
1436	V = nullptr;
1437
1438	assert(Var->isValidLocationForIntrinsic(MIMD.getDL()) &&
1439	"Expected inlined-at fields to agree");
1440
1441	if (!lowerDbgValue(V, Expr, Var, DL: MIMD.getDL()))
1442	LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
1443
1444	return true;
1445	}
1446	case Intrinsic::dbg_label: {
1447	const DbgLabelInst *DI = cast<DbgLabelInst>(Val: II);
1448	assert(DI->getLabel() && "Missing label");
1449	if (!FuncInfo.MF->getMMI().hasDebugInfo()) {
1450	LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
1451	return true;
1452	}
1453
1454	BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
1455	MCID: TII.get(Opcode: TargetOpcode::DBG_LABEL)).addMetadata(MD: DI->getLabel());
1456	return true;
1457	}
1458	case Intrinsic::objectsize:
1459	llvm_unreachable("llvm.objectsize.* should have been lowered already");
1460
1461	case Intrinsic::is_constant:
1462	llvm_unreachable("llvm.is.constant.* should have been lowered already");
1463
1464	case Intrinsic::allow_runtime_check:
1465	case Intrinsic::allow_ubsan_check: {
1466	Register ResultReg = getRegForValue(V: ConstantInt::getTrue(Ty: II->getType()));
1467	if (!ResultReg)
1468	return false;
1469	updateValueMap(I: II, Reg: ResultReg);
1470	return true;
1471	}
1472
1473	case Intrinsic::launder_invariant_group:
1474	case Intrinsic::strip_invariant_group:
1475	case Intrinsic::expect: {
1476	Register ResultReg = getRegForValue(V: II->getArgOperand(i: 0));
1477	if (!ResultReg)
1478	return false;
1479	updateValueMap(I: II, Reg: ResultReg);
1480	return true;
1481	}
1482	case Intrinsic::experimental_stackmap:
1483	return selectStackmap(I: II);
1484	case Intrinsic::experimental_patchpoint_void:
1485	case Intrinsic::experimental_patchpoint:
1486	return selectPatchpoint(I: II);
1487
1488	case Intrinsic::xray_customevent:
1489	return selectXRayCustomEvent(I: II);
1490	case Intrinsic::xray_typedevent:
1491	return selectXRayTypedEvent(I: II);
1492	}
1493
1494	return fastLowerIntrinsicCall(II);
1495	}
1496
1497	bool FastISel::selectCast(const User *I, unsigned Opcode) {
1498	EVT SrcVT = TLI.getValueType(DL, Ty: I->getOperand(i: 0)->getType());
1499	EVT DstVT = TLI.getValueType(DL, Ty: I->getType());
1500
1501	if (SrcVT == MVT::Other || !SrcVT.isSimple() || DstVT == MVT::Other ||
1502	!DstVT.isSimple())
1503	// Unhandled type. Halt "fast" selection and bail.
1504	return false;
1505
1506	// Check if the destination type is legal.
1507	if (!TLI.isTypeLegal(VT: DstVT))
1508	return false;
1509
1510	// Check if the source operand is legal.
1511	if (!TLI.isTypeLegal(VT: SrcVT))
1512	return false;
1513
1514	Register InputReg = getRegForValue(V: I->getOperand(i: 0));
1515	if (!InputReg)
1516	// Unhandled operand. Halt "fast" selection and bail.
1517	return false;
1518
1519	Register ResultReg = fastEmit_r(VT: SrcVT.getSimpleVT(), RetVT: DstVT.getSimpleVT(),
1520	Opcode, Op0: InputReg);
1521	if (!ResultReg)
1522	return false;
1523
1524	updateValueMap(I, Reg: ResultReg);
1525	return true;
1526	}
1527
1528	bool FastISel::selectBitCast(const User *I) {
1529	EVT SrcEVT = TLI.getValueType(DL, Ty: I->getOperand(i: 0)->getType());
1530	EVT DstEVT = TLI.getValueType(DL, Ty: I->getType());
1531	if (SrcEVT == MVT::Other || DstEVT == MVT::Other ||
1532	!TLI.isTypeLegal(VT: SrcEVT) || !TLI.isTypeLegal(VT: DstEVT))
1533	// Unhandled type. Halt "fast" selection and bail.
1534	return false;
1535
1536	MVT SrcVT = SrcEVT.getSimpleVT();
1537	MVT DstVT = DstEVT.getSimpleVT();
1538	Register Op0 = getRegForValue(V: I->getOperand(i: 0));
1539	if (!Op0) // Unhandled operand. Halt "fast" selection and bail.
1540	return false;
1541
1542	// If the bitcast doesn't change the type, just use the operand value.
1543	if (SrcVT == DstVT) {
1544	updateValueMap(I, Reg: Op0);
1545	return true;
1546	}
1547
1548	// Otherwise, select a BITCAST opcode.
1549	Register ResultReg = fastEmit_r(VT: SrcVT, RetVT: DstVT, Opcode: ISD::BITCAST, Op0);
1550	if (!ResultReg)
1551	return false;
1552
1553	updateValueMap(I, Reg: ResultReg);
1554	return true;
1555	}
1556
1557	bool FastISel::selectFreeze(const User *I) {
1558	Register Reg = getRegForValue(V: I->getOperand(i: 0));
1559	if (!Reg)
1560	// Unhandled operand.
1561	return false;
1562
1563	EVT ETy = TLI.getValueType(DL, Ty: I->getOperand(i: 0)->getType());
1564	if (ETy == MVT::Other || !TLI.isTypeLegal(VT: ETy))
1565	// Unhandled type, bail out.
1566	return false;
1567
1568	MVT Ty = ETy.getSimpleVT();
1569	const TargetRegisterClass *TyRegClass = TLI.getRegClassFor(VT: Ty);
1570	Register ResultReg = createResultReg(RC: TyRegClass);
1571	BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
1572	MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: ResultReg).addReg(RegNo: Reg);
1573
1574	updateValueMap(I, Reg: ResultReg);
1575	return true;
1576	}
1577
1578	// Remove local value instructions starting from the instruction after
1579	// SavedLastLocalValue to the current function insert point.
1580	void FastISel::removeDeadLocalValueCode(MachineInstr *SavedLastLocalValue)
1581	{
1582	MachineInstr *CurLastLocalValue = getLastLocalValue();
1583	if (CurLastLocalValue != SavedLastLocalValue) {
1584	// Find the first local value instruction to be deleted.
1585	// This is the instruction after SavedLastLocalValue if it is non-NULL.
1586	// Otherwise it's the first instruction in the block.
1587	MachineBasicBlock::iterator FirstDeadInst(SavedLastLocalValue);
1588	if (SavedLastLocalValue)
1589	++FirstDeadInst;
1590	else
1591	FirstDeadInst = FuncInfo.MBB->getFirstNonPHI();
1592	setLastLocalValue(SavedLastLocalValue);
1593	removeDeadCode(I: FirstDeadInst, E: FuncInfo.InsertPt);
1594	}
1595	}
1596
1597	bool FastISel::selectInstruction(const Instruction *I) {
1598	// Flush the local value map before starting each instruction.
1599	// This improves locality and debugging, and can reduce spills.
1600	// Reuse of values across IR instructions is relatively uncommon.
1601	flushLocalValueMap();
1602
1603	MachineInstr *SavedLastLocalValue = getLastLocalValue();
1604	// Just before the terminator instruction, insert instructions to
1605	// feed PHI nodes in successor blocks.
1606	if (I->isTerminator()) {
1607	if (!handlePHINodesInSuccessorBlocks(LLVMBB: I->getParent())) {
1608	// PHI node handling may have generated local value instructions,
1609	// even though it failed to handle all PHI nodes.
1610	// We remove these instructions because SelectionDAGISel will generate
1611	// them again.
1612	removeDeadLocalValueCode(SavedLastLocalValue);
1613	return false;
1614	}
1615	}
1616
1617	// FastISel does not handle any operand bundles except OB_funclet.
1618	if (auto *Call = dyn_cast<CallBase>(Val: I))
1619	for (unsigned i = 0, e = Call->getNumOperandBundles(); i != e; ++i)
1620	if (Call->getOperandBundleAt(Index: i).getTagID() != LLVMContext::OB_funclet)
1621	return false;
1622
1623	MIMD = MIMetadata(*I);
1624
1625	SavedInsertPt = FuncInfo.InsertPt;
1626
1627	if (const auto *Call = dyn_cast<CallInst>(Val: I)) {
1628	const Function *F = Call->getCalledFunction();
1629	LibFunc Func;
1630
1631	// As a special case, don't handle calls to builtin library functions that
1632	// may be translated directly to target instructions.
1633	if (F && !F->hasLocalLinkage() && F->hasName() &&
1634	LibInfo->getLibFunc(funcName: F->getName(), F&: Func) &&
1635	LibInfo->hasOptimizedCodeGen(F: Func))
1636	return false;
1637
1638	// Don't handle Intrinsic::trap if a trap function is specified.
1639	if (F && F->getIntrinsicID() == Intrinsic::trap &&
1640	Call->hasFnAttr(Kind: "trap-func-name"))
1641	return false;
1642	}
1643
1644	// First, try doing target-independent selection.
1645	if (!SkipTargetIndependentISel) {
1646	if (selectOperator(I, Opcode: I->getOpcode())) {
1647	++NumFastIselSuccessIndependent;
1648	MIMD = {};
1649	return true;
1650	}
1651	// Remove dead code.
1652	recomputeInsertPt();
1653	if (SavedInsertPt != FuncInfo.InsertPt)
1654	removeDeadCode(I: FuncInfo.InsertPt, E: SavedInsertPt);
1655	SavedInsertPt = FuncInfo.InsertPt;
1656	}
1657	// Next, try calling the target to attempt to handle the instruction.
1658	if (fastSelectInstruction(I)) {
1659	++NumFastIselSuccessTarget;
1660	MIMD = {};
1661	return true;
1662	}
1663	// Remove dead code.
1664	recomputeInsertPt();
1665	if (SavedInsertPt != FuncInfo.InsertPt)
1666	removeDeadCode(I: FuncInfo.InsertPt, E: SavedInsertPt);
1667
1668	MIMD = {};
1669	// Undo phi node updates, because they will be added again by SelectionDAG.
1670	if (I->isTerminator()) {
1671	// PHI node handling may have generated local value instructions.
1672	// We remove them because SelectionDAGISel will generate them again.
1673	removeDeadLocalValueCode(SavedLastLocalValue);
1674	FuncInfo.PHINodesToUpdate.resize(new_size: FuncInfo.OrigNumPHINodesToUpdate);
1675	}
1676	return false;
1677	}
1678
1679	/// Emit an unconditional branch to the given block, unless it is the immediate
1680	/// (fall-through) successor, and update the CFG.
1681	void FastISel::fastEmitBranch(MachineBasicBlock *MSucc,
1682	const DebugLoc &DbgLoc) {
1683	if (FuncInfo.MBB->getBasicBlock()->sizeWithoutDebug() > 1 &&
1684	FuncInfo.MBB->isLayoutSuccessor(MBB: MSucc)) {
1685	// For more accurate line information if this is the only non-debug
1686	// instruction in the block then emit it, otherwise we have the
1687	// unconditional fall-through case, which needs no instructions.
1688	} else {
1689	// The unconditional branch case.
1690	TII.insertBranch(MBB&: *FuncInfo.MBB, TBB: MSucc, FBB: nullptr,
1691	Cond: SmallVector<MachineOperand, 0>(), DL: DbgLoc);
1692	}
1693	if (FuncInfo.BPI) {
1694	auto BranchProbability = FuncInfo.BPI->getEdgeProbability(
1695	Src: FuncInfo.MBB->getBasicBlock(), Dst: MSucc->getBasicBlock());
1696	FuncInfo.MBB->addSuccessor(Succ: MSucc, Prob: BranchProbability);
1697	} else
1698	FuncInfo.MBB->addSuccessorWithoutProb(Succ: MSucc);
1699	}
1700
1701	void FastISel::finishCondBranch(const BasicBlock *BranchBB,
1702	MachineBasicBlock *TrueMBB,
1703	MachineBasicBlock *FalseMBB) {
1704	// Add TrueMBB as successor unless it is equal to the FalseMBB: This can
1705	// happen in degenerate IR and MachineIR forbids to have a block twice in the
1706	// successor/predecessor lists.
1707	if (TrueMBB != FalseMBB) {
1708	if (FuncInfo.BPI) {
1709	auto BranchProbability =
1710	FuncInfo.BPI->getEdgeProbability(Src: BranchBB, Dst: TrueMBB->getBasicBlock());
1711	FuncInfo.MBB->addSuccessor(Succ: TrueMBB, Prob: BranchProbability);
1712	} else
1713	FuncInfo.MBB->addSuccessorWithoutProb(Succ: TrueMBB);
1714	}
1715
1716	fastEmitBranch(MSucc: FalseMBB, DbgLoc: MIMD.getDL());
1717	}
1718
1719	/// Emit an FNeg operation.
1720	bool FastISel::selectFNeg(const User *I, const Value *In) {
1721	Register OpReg = getRegForValue(V: In);
1722	if (!OpReg)
1723	return false;
1724
1725	// If the target has ISD::FNEG, use it.
1726	EVT VT = TLI.getValueType(DL, Ty: I->getType());
1727	Register ResultReg = fastEmit_r(VT: VT.getSimpleVT(), RetVT: VT.getSimpleVT(), Opcode: ISD::FNEG,
1728	Op0: OpReg);
1729	if (ResultReg) {
1730	updateValueMap(I, Reg: ResultReg);
1731	return true;
1732	}
1733
1734	// Bitcast the value to integer, twiddle the sign bit with xor,
1735	// and then bitcast it back to floating-point.
1736	if (VT.getSizeInBits() > 64)
1737	return false;
1738	EVT IntVT = EVT::getIntegerVT(Context&: I->getContext(), BitWidth: VT.getSizeInBits());
1739	if (!TLI.isTypeLegal(VT: IntVT))
1740	return false;
1741
1742	Register IntReg = fastEmit_r(VT: VT.getSimpleVT(), RetVT: IntVT.getSimpleVT(),
1743	Opcode: ISD::BITCAST, Op0: OpReg);
1744	if (!IntReg)
1745	return false;
1746
1747	Register IntResultReg = fastEmit_ri_(
1748	VT: IntVT.getSimpleVT(), Opcode: ISD::XOR, Op0: IntReg,
1749	UINT64_C(1) << (VT.getSizeInBits() - 1), ImmType: IntVT.getSimpleVT());
1750	if (!IntResultReg)
1751	return false;
1752
1753	ResultReg = fastEmit_r(VT: IntVT.getSimpleVT(), RetVT: VT.getSimpleVT(), Opcode: ISD::BITCAST,
1754	Op0: IntResultReg);
1755	if (!ResultReg)
1756	return false;
1757
1758	updateValueMap(I, Reg: ResultReg);
1759	return true;
1760	}
1761
1762	bool FastISel::selectExtractValue(const User *U) {
1763	const ExtractValueInst *EVI = dyn_cast<ExtractValueInst>(Val: U);
1764	if (!EVI)
1765	return false;
1766
1767	// Make sure we only try to handle extracts with a legal result. But also
1768	// allow i1 because it's easy.
1769	EVT RealVT = TLI.getValueType(DL, Ty: EVI->getType(), /*AllowUnknown=*/true);
1770	if (!RealVT.isSimple())
1771	return false;
1772	MVT VT = RealVT.getSimpleVT();
1773	if (!TLI.isTypeLegal(VT) && VT != MVT::i1)
1774	return false;
1775
1776	const Value *Op0 = EVI->getOperand(i_nocapture: 0);
1777	Type *AggTy = Op0->getType();
1778
1779	// Get the base result register.
1780	unsigned ResultReg;
1781	DenseMap<const Value *, Register>::iterator I = FuncInfo.ValueMap.find(Val: Op0);
1782	if (I != FuncInfo.ValueMap.end())
1783	ResultReg = I->second;
1784	else if (isa<Instruction>(Val: Op0))
1785	ResultReg = FuncInfo.InitializeRegForValue(V: Op0);
1786	else
1787	return false; // fast-isel can't handle aggregate constants at the moment
1788
1789	// Get the actual result register, which is an offset from the base register.
1790	unsigned VTIndex = ComputeLinearIndex(Ty: AggTy, Indices: EVI->getIndices());
1791
1792	SmallVector<EVT, 4> AggValueVTs;
1793	ComputeValueVTs(TLI, DL, Ty: AggTy, ValueVTs&: AggValueVTs);
1794
1795	for (unsigned i = 0; i < VTIndex; i++)
1796	ResultReg += TLI.getNumRegisters(Context&: FuncInfo.Fn->getContext(), VT: AggValueVTs[i]);
1797
1798	updateValueMap(I: EVI, Reg: ResultReg);
1799	return true;
1800	}
1801
1802	bool FastISel::selectOperator(const User *I, unsigned Opcode) {
1803	switch (Opcode) {
1804	case Instruction::Add:
1805	return selectBinaryOp(I, ISDOpcode: ISD::ADD);
1806	case Instruction::FAdd:
1807	return selectBinaryOp(I, ISDOpcode: ISD::FADD);
1808	case Instruction::Sub:
1809	return selectBinaryOp(I, ISDOpcode: ISD::SUB);
1810	case Instruction::FSub:
1811	return selectBinaryOp(I, ISDOpcode: ISD::FSUB);
1812	case Instruction::Mul:
1813	return selectBinaryOp(I, ISDOpcode: ISD::MUL);
1814	case Instruction::FMul:
1815	return selectBinaryOp(I, ISDOpcode: ISD::FMUL);
1816	case Instruction::SDiv:
1817	return selectBinaryOp(I, ISDOpcode: ISD::SDIV);
1818	case Instruction::UDiv:
1819	return selectBinaryOp(I, ISDOpcode: ISD::UDIV);
1820	case Instruction::FDiv:
1821	return selectBinaryOp(I, ISDOpcode: ISD::FDIV);
1822	case Instruction::SRem:
1823	return selectBinaryOp(I, ISDOpcode: ISD::SREM);
1824	case Instruction::URem:
1825	return selectBinaryOp(I, ISDOpcode: ISD::UREM);
1826	case Instruction::FRem:
1827	return selectBinaryOp(I, ISDOpcode: ISD::FREM);
1828	case Instruction::Shl:
1829	return selectBinaryOp(I, ISDOpcode: ISD::SHL);
1830	case Instruction::LShr:
1831	return selectBinaryOp(I, ISDOpcode: ISD::SRL);
1832	case Instruction::AShr:
1833	return selectBinaryOp(I, ISDOpcode: ISD::SRA);
1834	case Instruction::And:
1835	return selectBinaryOp(I, ISDOpcode: ISD::AND);
1836	case Instruction::Or:
1837	return selectBinaryOp(I, ISDOpcode: ISD::OR);
1838	case Instruction::Xor:
1839	return selectBinaryOp(I, ISDOpcode: ISD::XOR);
1840
1841	case Instruction::FNeg:
1842	return selectFNeg(I, In: I->getOperand(i: 0));
1843
1844	case Instruction::GetElementPtr:
1845	return selectGetElementPtr(I);
1846
1847	case Instruction::Br: {
1848	const BranchInst *BI = cast<BranchInst>(Val: I);
1849
1850	if (BI->isUnconditional()) {
1851	const BasicBlock *LLVMSucc = BI->getSuccessor(i: 0);
1852	MachineBasicBlock *MSucc = FuncInfo.MBBMap[LLVMSucc];
1853	fastEmitBranch(MSucc, DbgLoc: BI->getDebugLoc());
1854	return true;
1855	}
1856
1857	// Conditional branches are not handed yet.
1858	// Halt "fast" selection and bail.
1859	return false;
1860	}
1861
1862	case Instruction::Unreachable:
1863	if (TM.Options.TrapUnreachable)
1864	return fastEmit_(MVT::VT: Other, MVT::RetVT: Other, Opcode: ISD::TRAP) != 0;
1865	else
1866	return true;
1867
1868	case Instruction::Alloca:
1869	// FunctionLowering has the static-sized case covered.
1870	if (FuncInfo.StaticAllocaMap.count(Val: cast<AllocaInst>(Val: I)))
1871	return true;
1872
1873	// Dynamic-sized alloca is not handled yet.
1874	return false;
1875
1876	case Instruction::Call:
1877	// On AIX, normal call lowering uses the DAG-ISEL path currently so that the
1878	// callee of the direct function call instruction will be mapped to the
1879	// symbol for the function's entry point, which is distinct from the
1880	// function descriptor symbol. The latter is the symbol whose XCOFF symbol
1881	// name is the C-linkage name of the source level function.
1882	// But fast isel still has the ability to do selection for intrinsics.
1883	if (TM.getTargetTriple().isOSAIX() && !isa<IntrinsicInst>(Val: I))
1884	return false;
1885	return selectCall(I);
1886
1887	case Instruction::BitCast:
1888	return selectBitCast(I);
1889
1890	case Instruction::FPToSI:
1891	return selectCast(I, Opcode: ISD::FP_TO_SINT);
1892	case Instruction::ZExt:
1893	return selectCast(I, Opcode: ISD::ZERO_EXTEND);
1894	case Instruction::SExt:
1895	return selectCast(I, Opcode: ISD::SIGN_EXTEND);
1896	case Instruction::Trunc:
1897	return selectCast(I, Opcode: ISD::TRUNCATE);
1898	case Instruction::SIToFP:
1899	return selectCast(I, Opcode: ISD::SINT_TO_FP);
1900
1901	case Instruction::IntToPtr: // Deliberate fall-through.
1902	case Instruction::PtrToInt: {
1903	EVT SrcVT = TLI.getValueType(DL, Ty: I->getOperand(i: 0)->getType());
1904	EVT DstVT = TLI.getValueType(DL, Ty: I->getType());
1905	if (DstVT.bitsGT(VT: SrcVT))
1906	return selectCast(I, Opcode: ISD::ZERO_EXTEND);
1907	if (DstVT.bitsLT(VT: SrcVT))
1908	return selectCast(I, Opcode: ISD::TRUNCATE);
1909	Register Reg = getRegForValue(V: I->getOperand(i: 0));
1910	if (!Reg)
1911	return false;
1912	updateValueMap(I, Reg);
1913	return true;
1914	}
1915
1916	case Instruction::ExtractValue:
1917	return selectExtractValue(U: I);
1918
1919	case Instruction::Freeze:
1920	return selectFreeze(I);
1921
1922	case Instruction::PHI:
1923	llvm_unreachable("FastISel shouldn't visit PHI nodes!");
1924
1925	default:
1926	// Unhandled instruction. Halt "fast" selection and bail.
1927	return false;
1928	}
1929	}
1930
1931	FastISel::FastISel(FunctionLoweringInfo &FuncInfo,
1932	const TargetLibraryInfo *LibInfo,
1933	bool SkipTargetIndependentISel)
1934	: FuncInfo(FuncInfo), MF(FuncInfo.MF), MRI(FuncInfo.MF->getRegInfo()),
1935	MFI(FuncInfo.MF->getFrameInfo()), MCP(*FuncInfo.MF->getConstantPool()),
1936	TM(FuncInfo.MF->getTarget()), DL(MF->getDataLayout()),
1937	TII(*MF->getSubtarget().getInstrInfo()),
1938	TLI(*MF->getSubtarget().getTargetLowering()),
1939	TRI(*MF->getSubtarget().getRegisterInfo()), LibInfo(LibInfo),
1940	SkipTargetIndependentISel(SkipTargetIndependentISel) {}
1941
1942	FastISel::~FastISel() = default;
1943
1944	bool FastISel::fastLowerArguments() { return false; }
1945
1946	bool FastISel::fastLowerCall(CallLoweringInfo & /*CLI*/) { return false; }
1947
1948	bool FastISel::fastLowerIntrinsicCall(const IntrinsicInst * /*II*/) {
1949	return false;
1950	}
1951
1952	unsigned FastISel::fastEmit_(MVT, MVT, unsigned) { return 0; }
1953
1954	unsigned FastISel::fastEmit_r(MVT, MVT, unsigned, unsigned /*Op0*/) {
1955	return 0;
1956	}
1957
1958	unsigned FastISel::fastEmit_rr(MVT, MVT, unsigned, unsigned /*Op0*/,
1959	unsigned /*Op1*/) {
1960	return 0;
1961	}
1962
1963	unsigned FastISel::fastEmit_i(MVT, MVT, unsigned, uint64_t /*Imm*/) {
1964	return 0;
1965	}
1966
1967	unsigned FastISel::fastEmit_f(MVT, MVT, unsigned,
1968	const ConstantFP * /*FPImm*/) {
1969	return 0;
1970	}
1971
1972	unsigned FastISel::fastEmit_ri(MVT, MVT, unsigned, unsigned /*Op0*/,
1973	uint64_t /*Imm*/) {
1974	return 0;
1975	}
1976
1977	/// This method is a wrapper of fastEmit_ri. It first tries to emit an
1978	/// instruction with an immediate operand using fastEmit_ri.
1979	/// If that fails, it materializes the immediate into a register and try
1980	/// fastEmit_rr instead.
1981	Register FastISel::fastEmit_ri_(MVT VT, unsigned Opcode, unsigned Op0,
1982	uint64_t Imm, MVT ImmType) {
1983	// If this is a multiply by a power of two, emit this as a shift left.
1984	if (Opcode == ISD::MUL && isPowerOf2_64(Value: Imm)) {
1985	Opcode = ISD::SHL;
1986	Imm = Log2_64(Value: Imm);
1987	} else if (Opcode == ISD::UDIV && isPowerOf2_64(Value: Imm)) {
1988	// div x, 8 -> srl x, 3
1989	Opcode = ISD::SRL;
1990	Imm = Log2_64(Value: Imm);
1991	}
1992
1993	// Horrible hack (to be removed), check to make sure shift amounts are
1994	// in-range.
1995	if ((Opcode == ISD::SHL || Opcode == ISD::SRA || Opcode == ISD::SRL) &&
1996	Imm >= VT.getSizeInBits())
1997	return 0;
1998
1999	// First check if immediate type is legal. If not, we can't use the ri form.
2000	Register ResultReg = fastEmit_ri(VT, VT, Opcode, Op0, Imm);
2001	if (ResultReg)
2002	return ResultReg;
2003	Register MaterialReg = fastEmit_i(ImmType, ImmType, ISD::Constant, Imm);
2004	if (!MaterialReg) {
2005	// This is a bit ugly/slow, but failing here means falling out of
2006	// fast-isel, which would be very slow.
2007	IntegerType *ITy =
2008	IntegerType::get(C&: FuncInfo.Fn->getContext(), NumBits: VT.getSizeInBits());
2009	MaterialReg = getRegForValue(V: ConstantInt::get(Ty: ITy, V: Imm));
2010	if (!MaterialReg)
2011	return 0;
2012	}
2013	return fastEmit_rr(VT, VT, Opcode, Op0, MaterialReg);
2014	}
2015
2016	Register FastISel::createResultReg(const TargetRegisterClass *RC) {
2017	return MRI.createVirtualRegister(RegClass: RC);
2018	}
2019
2020	Register FastISel::constrainOperandRegClass(const MCInstrDesc &II, Register Op,
2021	unsigned OpNum) {
2022	if (Op.isVirtual()) {
2023	const TargetRegisterClass *RegClass =
2024	TII.getRegClass(MCID: II, OpNum, TRI: &TRI, MF: *FuncInfo.MF);
2025	if (!MRI.constrainRegClass(Reg: Op, RC: RegClass)) {
2026	// If it's not legal to COPY between the register classes, something
2027	// has gone very wrong before we got here.
2028	Register NewOp = createResultReg(RC: RegClass);
2029	BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
2030	MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: NewOp).addReg(RegNo: Op);
2031	return NewOp;
2032	}
2033	}
2034	return Op;
2035	}
2036
2037	Register FastISel::fastEmitInst_(unsigned MachineInstOpcode,
2038	const TargetRegisterClass *RC) {
2039	Register ResultReg = createResultReg(RC);
2040	const MCInstrDesc &II = TII.get(Opcode: MachineInstOpcode);
2041
2042	BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II, DestReg: ResultReg);
2043	return ResultReg;
2044	}
2045
2046	Register FastISel::fastEmitInst_r(unsigned MachineInstOpcode,
2047	const TargetRegisterClass *RC, unsigned Op0) {
2048	const MCInstrDesc &II = TII.get(Opcode: MachineInstOpcode);
2049
2050	Register ResultReg = createResultReg(RC);
2051	Op0 = constrainOperandRegClass(II, Op: Op0, OpNum: II.getNumDefs());
2052
2053	if (II.getNumDefs() >= 1)
2054	BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II, DestReg: ResultReg)
2055	.addReg(RegNo: Op0);
2056	else {
2057	BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II)
2058	.addReg(RegNo: Op0);
2059	BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY),
2060	DestReg: ResultReg)
2061	.addReg(RegNo: II.implicit_defs()[0]);
2062	}
2063
2064	return ResultReg;
2065	}
2066
2067	Register FastISel::fastEmitInst_rr(unsigned MachineInstOpcode,
2068	const TargetRegisterClass *RC, unsigned Op0,
2069	unsigned Op1) {
2070	const MCInstrDesc &II = TII.get(Opcode: MachineInstOpcode);
2071
2072	Register ResultReg = createResultReg(RC);
2073	Op0 = constrainOperandRegClass(II, Op: Op0, OpNum: II.getNumDefs());
2074	Op1 = constrainOperandRegClass(II, Op: Op1, OpNum: II.getNumDefs() + 1);
2075
2076	if (II.getNumDefs() >= 1)
2077	BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II, DestReg: ResultReg)
2078	.addReg(RegNo: Op0)
2079	.addReg(RegNo: Op1);
2080	else {
2081	BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II)
2082	.addReg(RegNo: Op0)
2083	.addReg(RegNo: Op1);
2084	BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY),
2085	DestReg: ResultReg)
2086	.addReg(RegNo: II.implicit_defs()[0]);
2087	}
2088	return ResultReg;
2089	}
2090
2091	Register FastISel::fastEmitInst_rrr(unsigned MachineInstOpcode,
2092	const TargetRegisterClass *RC, unsigned Op0,
2093	unsigned Op1, unsigned Op2) {
2094	const MCInstrDesc &II = TII.get(Opcode: MachineInstOpcode);
2095
2096	Register ResultReg = createResultReg(RC);
2097	Op0 = constrainOperandRegClass(II, Op: Op0, OpNum: II.getNumDefs());
2098	Op1 = constrainOperandRegClass(II, Op: Op1, OpNum: II.getNumDefs() + 1);
2099	Op2 = constrainOperandRegClass(II, Op: Op2, OpNum: II.getNumDefs() + 2);
2100
2101	if (II.getNumDefs() >= 1)
2102	BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II, DestReg: ResultReg)
2103	.addReg(RegNo: Op0)
2104	.addReg(RegNo: Op1)
2105	.addReg(RegNo: Op2);
2106	else {
2107	BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II)
2108	.addReg(RegNo: Op0)
2109	.addReg(RegNo: Op1)
2110	.addReg(RegNo: Op2);
2111	BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY),
2112	DestReg: ResultReg)
2113	.addReg(RegNo: II.implicit_defs()[0]);
2114	}
2115	return ResultReg;
2116	}
2117
2118	Register FastISel::fastEmitInst_ri(unsigned MachineInstOpcode,
2119	const TargetRegisterClass *RC, unsigned Op0,
2120	uint64_t Imm) {
2121	const MCInstrDesc &II = TII.get(Opcode: MachineInstOpcode);
2122
2123	Register ResultReg = createResultReg(RC);
2124	Op0 = constrainOperandRegClass(II, Op: Op0, OpNum: II.getNumDefs());
2125
2126	if (II.getNumDefs() >= 1)
2127	BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II, DestReg: ResultReg)
2128	.addReg(RegNo: Op0)
2129	.addImm(Val: Imm);
2130	else {
2131	BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II)
2132	.addReg(RegNo: Op0)
2133	.addImm(Val: Imm);
2134	BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY),
2135	DestReg: ResultReg)
2136	.addReg(RegNo: II.implicit_defs()[0]);
2137	}
2138	return ResultReg;
2139	}
2140
2141	Register FastISel::fastEmitInst_rii(unsigned MachineInstOpcode,
2142	const TargetRegisterClass *RC, unsigned Op0,
2143	uint64_t Imm1, uint64_t Imm2) {
2144	const MCInstrDesc &II = TII.get(Opcode: MachineInstOpcode);
2145
2146	Register ResultReg = createResultReg(RC);
2147	Op0 = constrainOperandRegClass(II, Op: Op0, OpNum: II.getNumDefs());
2148
2149	if (II.getNumDefs() >= 1)
2150	BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II, DestReg: ResultReg)
2151	.addReg(RegNo: Op0)
2152	.addImm(Val: Imm1)
2153	.addImm(Val: Imm2);
2154	else {
2155	BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II)
2156	.addReg(RegNo: Op0)
2157	.addImm(Val: Imm1)
2158	.addImm(Val: Imm2);
2159	BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY),
2160	DestReg: ResultReg)
2161	.addReg(RegNo: II.implicit_defs()[0]);
2162	}
2163	return ResultReg;
2164	}
2165
2166	Register FastISel::fastEmitInst_f(unsigned MachineInstOpcode,
2167	const TargetRegisterClass *RC,
2168	const ConstantFP *FPImm) {
2169	const MCInstrDesc &II = TII.get(Opcode: MachineInstOpcode);
2170
2171	Register ResultReg = createResultReg(RC);
2172
2173	if (II.getNumDefs() >= 1)
2174	BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II, DestReg: ResultReg)
2175	.addFPImm(Val: FPImm);
2176	else {
2177	BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II)
2178	.addFPImm(Val: FPImm);
2179	BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY),
2180	DestReg: ResultReg)
2181	.addReg(RegNo: II.implicit_defs()[0]);
2182	}
2183	return ResultReg;
2184	}
2185
2186	Register FastISel::fastEmitInst_rri(unsigned MachineInstOpcode,
2187	const TargetRegisterClass *RC, unsigned Op0,
2188	unsigned Op1, uint64_t Imm) {
2189	const MCInstrDesc &II = TII.get(Opcode: MachineInstOpcode);
2190
2191	Register ResultReg = createResultReg(RC);
2192	Op0 = constrainOperandRegClass(II, Op: Op0, OpNum: II.getNumDefs());
2193	Op1 = constrainOperandRegClass(II, Op: Op1, OpNum: II.getNumDefs() + 1);
2194
2195	if (II.getNumDefs() >= 1)
2196	BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II, DestReg: ResultReg)
2197	.addReg(RegNo: Op0)
2198	.addReg(RegNo: Op1)
2199	.addImm(Val: Imm);
2200	else {
2201	BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II)
2202	.addReg(RegNo: Op0)
2203	.addReg(RegNo: Op1)
2204	.addImm(Val: Imm);
2205	BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY),
2206	DestReg: ResultReg)
2207	.addReg(RegNo: II.implicit_defs()[0]);
2208	}
2209	return ResultReg;
2210	}
2211
2212	Register FastISel::fastEmitInst_i(unsigned MachineInstOpcode,
2213	const TargetRegisterClass *RC, uint64_t Imm) {
2214	Register ResultReg = createResultReg(RC);
2215	const MCInstrDesc &II = TII.get(Opcode: MachineInstOpcode);
2216
2217	if (II.getNumDefs() >= 1)
2218	BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II, DestReg: ResultReg)
2219	.addImm(Val: Imm);
2220	else {
2221	BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II).addImm(Val: Imm);
2222	BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY),
2223	DestReg: ResultReg)
2224	.addReg(RegNo: II.implicit_defs()[0]);
2225	}
2226	return ResultReg;
2227	}
2228
2229	Register FastISel::fastEmitInst_extractsubreg(MVT RetVT, unsigned Op0,
2230	uint32_t Idx) {
2231	Register ResultReg = createResultReg(RC: TLI.getRegClassFor(VT: RetVT));
2232	assert(Register::isVirtualRegister(Op0) &&
2233	"Cannot yet extract from physregs");
2234	const TargetRegisterClass *RC = MRI.getRegClass(Reg: Op0);
2235	MRI.constrainRegClass(Reg: Op0, RC: TRI.getSubClassWithSubReg(RC, Idx));
2236	BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY),
2237	DestReg: ResultReg).addReg(RegNo: Op0, flags: 0, SubReg: Idx);
2238	return ResultReg;
2239	}
2240
2241	/// Emit MachineInstrs to compute the value of Op with all but the least
2242	/// significant bit set to zero.
2243	Register FastISel::fastEmitZExtFromI1(MVT VT, unsigned Op0) {
2244	return fastEmit_ri(VT, VT, ISD::AND, Op0, 1);
2245	}
2246
2247	/// HandlePHINodesInSuccessorBlocks - Handle PHI nodes in successor blocks.
2248	/// Emit code to ensure constants are copied into registers when needed.
2249	/// Remember the virtual registers that need to be added to the Machine PHI
2250	/// nodes as input. We cannot just directly add them, because expansion
2251	/// might result in multiple MBB's for one BB. As such, the start of the
2252	/// BB might correspond to a different MBB than the end.
2253	bool FastISel::handlePHINodesInSuccessorBlocks(const BasicBlock *LLVMBB) {
2254	SmallPtrSet<MachineBasicBlock *, 4> SuccsHandled;
2255	FuncInfo.OrigNumPHINodesToUpdate = FuncInfo.PHINodesToUpdate.size();
2256
2257	// Check successor nodes' PHI nodes that expect a constant to be available
2258	// from this block.
2259	for (const BasicBlock *SuccBB : successors(BB: LLVMBB)) {
2260	if (!isa<PHINode>(Val: SuccBB->begin()))
2261	continue;
2262	MachineBasicBlock *SuccMBB = FuncInfo.MBBMap[SuccBB];
2263
2264	// If this terminator has multiple identical successors (common for
2265	// switches), only handle each succ once.
2266	if (!SuccsHandled.insert(Ptr: SuccMBB).second)
2267	continue;
2268
2269	MachineBasicBlock::iterator MBBI = SuccMBB->begin();
2270
2271	// At this point we know that there is a 1-1 correspondence between LLVM PHI
2272	// nodes and Machine PHI nodes, but the incoming operands have not been
2273	// emitted yet.
2274	for (const PHINode &PN : SuccBB->phis()) {
2275	// Ignore dead phi's.
2276	if (PN.use_empty())
2277	continue;
2278
2279	// Only handle legal types. Two interesting things to note here. First,
2280	// by bailing out early, we may leave behind some dead instructions,
2281	// since SelectionDAG's HandlePHINodesInSuccessorBlocks will insert its
2282	// own moves. Second, this check is necessary because FastISel doesn't
2283	// use CreateRegs to create registers, so it always creates
2284	// exactly one register for each non-void instruction.
2285	EVT VT = TLI.getValueType(DL, Ty: PN.getType(), /*AllowUnknown=*/true);
2286	if (VT == MVT::Other || !TLI.isTypeLegal(VT)) {
2287	// Handle integer promotions, though, because they're common and easy.
2288	if (!(VT == MVT::i1 || VT == MVT::i8 || VT == MVT::i16)) {
2289	FuncInfo.PHINodesToUpdate.resize(new_size: FuncInfo.OrigNumPHINodesToUpdate);
2290	return false;
2291	}
2292	}
2293
2294	const Value *PHIOp = PN.getIncomingValueForBlock(BB: LLVMBB);
2295
2296	// Set the DebugLoc for the copy. Use the location of the operand if
2297	// there is one; otherwise no location, flushLocalValueMap will fix it.
2298	MIMD = {};
2299	if (const auto *Inst = dyn_cast<Instruction>(Val: PHIOp))
2300	MIMD = MIMetadata(*Inst);
2301
2302	Register Reg = getRegForValue(V: PHIOp);
2303	if (!Reg) {
2304	FuncInfo.PHINodesToUpdate.resize(new_size: FuncInfo.OrigNumPHINodesToUpdate);
2305	return false;
2306	}
2307	FuncInfo.PHINodesToUpdate.push_back(x: std::make_pair(x: &*MBBI++, y&: Reg));
2308	MIMD = {};
2309	}
2310	}
2311
2312	return true;
2313	}
2314
2315	bool FastISel::tryToFoldLoad(const LoadInst *LI, const Instruction *FoldInst) {
2316	assert(LI->hasOneUse() &&
2317	"tryToFoldLoad expected a LoadInst with a single use");
2318	// We know that the load has a single use, but don't know what it is. If it
2319	// isn't one of the folded instructions, then we can't succeed here. Handle
2320	// this by scanning the single-use users of the load until we get to FoldInst.
2321	unsigned MaxUsers = 6; // Don't scan down huge single-use chains of instrs.
2322
2323	const Instruction *TheUser = LI->user_back();
2324	while (TheUser != FoldInst && // Scan up until we find FoldInst.
2325	// Stay in the right block.
2326	TheUser->getParent() == FoldInst->getParent() &&
2327	--MaxUsers) { // Don't scan too far.
2328	// If there are multiple or no uses of this instruction, then bail out.
2329	if (!TheUser->hasOneUse())
2330	return false;
2331
2332	TheUser = TheUser->user_back();
2333	}
2334
2335	// If we didn't find the fold instruction, then we failed to collapse the
2336	// sequence.
2337	if (TheUser != FoldInst)
2338	return false;
2339
2340	// Don't try to fold volatile loads. Target has to deal with alignment
2341	// constraints.
2342	if (LI->isVolatile())
2343	return false;
2344
2345	// Figure out which vreg this is going into. If there is no assigned vreg yet
2346	// then there actually was no reference to it. Perhaps the load is referenced
2347	// by a dead instruction.
2348	Register LoadReg = getRegForValue(V: LI);
2349	if (!LoadReg)
2350	return false;
2351
2352	// We can't fold if this vreg has no uses or more than one use. Multiple uses
2353	// may mean that the instruction got lowered to multiple MIs, or the use of
2354	// the loaded value ended up being multiple operands of the result.
2355	if (!MRI.hasOneUse(RegNo: LoadReg))
2356	return false;
2357
2358	// If the register has fixups, there may be additional uses through a
2359	// different alias of the register.
2360	if (FuncInfo.RegsWithFixups.contains(V: LoadReg))
2361	return false;
2362
2363	MachineRegisterInfo::reg_iterator RI = MRI.reg_begin(RegNo: LoadReg);
2364	MachineInstr *User = RI->getParent();
2365
2366	// Set the insertion point properly. Folding the load can cause generation of
2367	// other random instructions (like sign extends) for addressing modes; make
2368	// sure they get inserted in a logical place before the new instruction.
2369	FuncInfo.InsertPt = User;
2370	FuncInfo.MBB = User->getParent();
2371
2372	// Ask the target to try folding the load.
2373	return tryToFoldLoadIntoMI(User, RI.getOperandNo(), LI);
2374	}
2375
2376	bool FastISel::canFoldAddIntoGEP(const User *GEP, const Value *Add) {
2377	// Must be an add.
2378	if (!isa<AddOperator>(Val: Add))
2379	return false;
2380	// Type size needs to match.
2381	if (DL.getTypeSizeInBits(Ty: GEP->getType()) !=
2382	DL.getTypeSizeInBits(Ty: Add->getType()))
2383	return false;
2384	// Must be in the same basic block.
2385	if (isa<Instruction>(Val: Add) &&
2386	FuncInfo.MBBMap[cast<Instruction>(Val: Add)->getParent()] != FuncInfo.MBB)
2387	return false;
2388	// Must have a constant operand.
2389	return isa<ConstantInt>(Val: cast<AddOperator>(Val: Add)->getOperand(i_nocapture: 1));
2390	}
2391
2392	MachineMemOperand *
2393	FastISel::createMachineMemOperandFor(const Instruction *I) const {
2394	const Value *Ptr;
2395	Type *ValTy;
2396	MaybeAlign Alignment;
2397	MachineMemOperand::Flags Flags;
2398	bool IsVolatile;
2399
2400	if (const auto *LI = dyn_cast<LoadInst>(Val: I)) {
2401	Alignment = LI->getAlign();
2402	IsVolatile = LI->isVolatile();
2403	Flags = MachineMemOperand::MOLoad;
2404	Ptr = LI->getPointerOperand();
2405	ValTy = LI->getType();
2406	} else if (const auto *SI = dyn_cast<StoreInst>(Val: I)) {
2407	Alignment = SI->getAlign();
2408	IsVolatile = SI->isVolatile();
2409	Flags = MachineMemOperand::MOStore;
2410	Ptr = SI->getPointerOperand();
2411	ValTy = SI->getValueOperand()->getType();
2412	} else
2413	return nullptr;
2414
2415	bool IsNonTemporal = I->hasMetadata(KindID: LLVMContext::MD_nontemporal);
2416	bool IsInvariant = I->hasMetadata(KindID: LLVMContext::MD_invariant_load);
2417	bool IsDereferenceable = I->hasMetadata(KindID: LLVMContext::MD_dereferenceable);
2418	const MDNode *Ranges = I->getMetadata(KindID: LLVMContext::MD_range);
2419
2420	AAMDNodes AAInfo = I->getAAMetadata();
2421
2422	if (!Alignment) // Ensure that codegen never sees alignment 0.
2423	Alignment = DL.getABITypeAlign(Ty: ValTy);
2424
2425	unsigned Size = DL.getTypeStoreSize(Ty: ValTy);
2426
2427	if (IsVolatile)
2428	Flags |= MachineMemOperand::MOVolatile;
2429	if (IsNonTemporal)
2430	Flags |= MachineMemOperand::MONonTemporal;
2431	if (IsDereferenceable)
2432	Flags |= MachineMemOperand::MODereferenceable;
2433	if (IsInvariant)
2434	Flags |= MachineMemOperand::MOInvariant;
2435
2436	return FuncInfo.MF->getMachineMemOperand(PtrInfo: MachinePointerInfo(Ptr), F: Flags, Size,
2437	BaseAlignment: *Alignment, AAInfo, Ranges);
2438	}
2439
2440	CmpInst::Predicate FastISel::optimizeCmpPredicate(const CmpInst *CI) const {
2441	// If both operands are the same, then try to optimize or fold the cmp.
2442	CmpInst::Predicate Predicate = CI->getPredicate();
2443	if (CI->getOperand(i_nocapture: 0) != CI->getOperand(i_nocapture: 1))
2444	return Predicate;
2445
2446	switch (Predicate) {
2447	default: llvm_unreachable("Invalid predicate!");
2448	case CmpInst::FCMP_FALSE: Predicate = CmpInst::FCMP_FALSE; break;
2449	case CmpInst::FCMP_OEQ: Predicate = CmpInst::FCMP_ORD; break;
2450	case CmpInst::FCMP_OGT: Predicate = CmpInst::FCMP_FALSE; break;
2451	case CmpInst::FCMP_OGE: Predicate = CmpInst::FCMP_ORD; break;
2452	case CmpInst::FCMP_OLT: Predicate = CmpInst::FCMP_FALSE; break;
2453	case CmpInst::FCMP_OLE: Predicate = CmpInst::FCMP_ORD; break;
2454	case CmpInst::FCMP_ONE: Predicate = CmpInst::FCMP_FALSE; break;
2455	case CmpInst::FCMP_ORD: Predicate = CmpInst::FCMP_ORD; break;
2456	case CmpInst::FCMP_UNO: Predicate = CmpInst::FCMP_UNO; break;
2457	case CmpInst::FCMP_UEQ: Predicate = CmpInst::FCMP_TRUE; break;
2458	case CmpInst::FCMP_UGT: Predicate = CmpInst::FCMP_UNO; break;
2459	case CmpInst::FCMP_UGE: Predicate = CmpInst::FCMP_TRUE; break;
2460	case CmpInst::FCMP_ULT: Predicate = CmpInst::FCMP_UNO; break;
2461	case CmpInst::FCMP_ULE: Predicate = CmpInst::FCMP_TRUE; break;
2462	case CmpInst::FCMP_UNE: Predicate = CmpInst::FCMP_UNO; break;
2463	case CmpInst::FCMP_TRUE: Predicate = CmpInst::FCMP_TRUE; break;
2464
2465	case CmpInst::ICMP_EQ: Predicate = CmpInst::FCMP_TRUE; break;
2466	case CmpInst::ICMP_NE: Predicate = CmpInst::FCMP_FALSE; break;
2467	case CmpInst::ICMP_UGT: Predicate = CmpInst::FCMP_FALSE; break;
2468	case CmpInst::ICMP_UGE: Predicate = CmpInst::FCMP_TRUE; break;
2469	case CmpInst::ICMP_ULT: Predicate = CmpInst::FCMP_FALSE; break;
2470	case CmpInst::ICMP_ULE: Predicate = CmpInst::FCMP_TRUE; break;
2471	case CmpInst::ICMP_SGT: Predicate = CmpInst::FCMP_FALSE; break;
2472	case CmpInst::ICMP_SGE: Predicate = CmpInst::FCMP_TRUE; break;
2473	case CmpInst::ICMP_SLT: Predicate = CmpInst::FCMP_FALSE; break;
2474	case CmpInst::ICMP_SLE: Predicate = CmpInst::FCMP_TRUE; break;
2475	}
2476
2477	return Predicate;
2478	}
2479