1//===- SelectionDAGBuilder.cpp - Selection-DAG building -------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This implements routines for translating from LLVM IR into SelectionDAG IR.
10//
11//===----------------------------------------------------------------------===//
12
13#include "SelectionDAGBuilder.h"
14#include "SDNodeDbgValue.h"
15#include "llvm/ADT/APFloat.h"
16#include "llvm/ADT/APInt.h"
17#include "llvm/ADT/BitVector.h"
18#include "llvm/ADT/STLExtras.h"
19#include "llvm/ADT/SmallPtrSet.h"
20#include "llvm/ADT/SmallSet.h"
21#include "llvm/ADT/StringRef.h"
22#include "llvm/ADT/Twine.h"
23#include "llvm/Analysis/AliasAnalysis.h"
24#include "llvm/Analysis/BranchProbabilityInfo.h"
25#include "llvm/Analysis/ConstantFolding.h"
26#include "llvm/Analysis/Loads.h"
27#include "llvm/Analysis/MemoryLocation.h"
28#include "llvm/Analysis/TargetLibraryInfo.h"
29#include "llvm/Analysis/TargetTransformInfo.h"
30#include "llvm/Analysis/ValueTracking.h"
31#include "llvm/Analysis/VectorUtils.h"
32#include "llvm/CodeGen/Analysis.h"
33#include "llvm/CodeGen/AssignmentTrackingAnalysis.h"
34#include "llvm/CodeGen/CodeGenCommonISel.h"
35#include "llvm/CodeGen/FunctionLoweringInfo.h"
36#include "llvm/CodeGen/GCMetadata.h"
37#include "llvm/CodeGen/ISDOpcodes.h"
38#include "llvm/CodeGen/MachineBasicBlock.h"
39#include "llvm/CodeGen/MachineFrameInfo.h"
40#include "llvm/CodeGen/MachineFunction.h"
41#include "llvm/CodeGen/MachineInstrBuilder.h"
42#include "llvm/CodeGen/MachineInstrBundleIterator.h"
43#include "llvm/CodeGen/MachineMemOperand.h"
44#include "llvm/CodeGen/MachineModuleInfo.h"
45#include "llvm/CodeGen/MachineOperand.h"
46#include "llvm/CodeGen/MachineRegisterInfo.h"
47#include "llvm/CodeGen/SelectionDAG.h"
48#include "llvm/CodeGen/SelectionDAGTargetInfo.h"
49#include "llvm/CodeGen/StackMaps.h"
50#include "llvm/CodeGen/SwiftErrorValueTracking.h"
51#include "llvm/CodeGen/TargetFrameLowering.h"
52#include "llvm/CodeGen/TargetInstrInfo.h"
53#include "llvm/CodeGen/TargetOpcodes.h"
54#include "llvm/CodeGen/TargetRegisterInfo.h"
55#include "llvm/CodeGen/TargetSubtargetInfo.h"
56#include "llvm/CodeGen/WinEHFuncInfo.h"
57#include "llvm/IR/Argument.h"
58#include "llvm/IR/Attributes.h"
59#include "llvm/IR/BasicBlock.h"
60#include "llvm/IR/CFG.h"
61#include "llvm/IR/CallingConv.h"
62#include "llvm/IR/Constant.h"
63#include "llvm/IR/ConstantRange.h"
64#include "llvm/IR/Constants.h"
65#include "llvm/IR/DataLayout.h"
66#include "llvm/IR/DebugInfo.h"
67#include "llvm/IR/DebugInfoMetadata.h"
68#include "llvm/IR/DerivedTypes.h"
69#include "llvm/IR/DiagnosticInfo.h"
70#include "llvm/IR/EHPersonalities.h"
71#include "llvm/IR/Function.h"
72#include "llvm/IR/GetElementPtrTypeIterator.h"
73#include "llvm/IR/InlineAsm.h"
74#include "llvm/IR/InstrTypes.h"
75#include "llvm/IR/Instructions.h"
76#include "llvm/IR/IntrinsicInst.h"
77#include "llvm/IR/Intrinsics.h"
78#include "llvm/IR/IntrinsicsAArch64.h"
79#include "llvm/IR/IntrinsicsAMDGPU.h"
80#include "llvm/IR/IntrinsicsWebAssembly.h"
81#include "llvm/IR/LLVMContext.h"
82#include "llvm/IR/MemoryModelRelaxationAnnotations.h"
83#include "llvm/IR/Metadata.h"
84#include "llvm/IR/Module.h"
85#include "llvm/IR/Operator.h"
86#include "llvm/IR/PatternMatch.h"
87#include "llvm/IR/Statepoint.h"
88#include "llvm/IR/Type.h"
89#include "llvm/IR/User.h"
90#include "llvm/IR/Value.h"
91#include "llvm/MC/MCContext.h"
92#include "llvm/Support/AtomicOrdering.h"
93#include "llvm/Support/Casting.h"
94#include "llvm/Support/CommandLine.h"
95#include "llvm/Support/Compiler.h"
96#include "llvm/Support/Debug.h"
97#include "llvm/Support/InstructionCost.h"
98#include "llvm/Support/MathExtras.h"
99#include "llvm/Support/raw_ostream.h"
100#include "llvm/Target/TargetMachine.h"
101#include "llvm/Target/TargetOptions.h"
102#include "llvm/TargetParser/Triple.h"
103#include "llvm/Transforms/Utils/Local.h"
104#include <cstddef>
105#include <limits>
106#include <optional>
107#include <tuple>
108
109using namespace llvm;
110using namespace PatternMatch;
111using namespace SwitchCG;
112
113#define DEBUG_TYPE "isel"
114
115/// LimitFloatPrecision - Generate low-precision inline sequences for
116/// some float libcalls (6, 8 or 12 bits).
117static unsigned LimitFloatPrecision;
118
119static cl::opt<bool>
120 InsertAssertAlign("insert-assert-align", cl::init(Val: true),
121 cl::desc("Insert the experimental `assertalign` node."),
122 cl::ReallyHidden);
123
124static cl::opt<unsigned, true>
125 LimitFPPrecision("limit-float-precision",
126 cl::desc("Generate low-precision inline sequences "
127 "for some float libcalls"),
128 cl::location(L&: LimitFloatPrecision), cl::Hidden,
129 cl::init(Val: 0));
130
131static cl::opt<unsigned> SwitchPeelThreshold(
132 "switch-peel-threshold", cl::Hidden, cl::init(Val: 66),
133 cl::desc("Set the case probability threshold for peeling the case from a "
134 "switch statement. A value greater than 100 will void this "
135 "optimization"));
136
137// Limit the width of DAG chains. This is important in general to prevent
138// DAG-based analysis from blowing up. For example, alias analysis and
139// load clustering may not complete in reasonable time. It is difficult to
140// recognize and avoid this situation within each individual analysis, and
141// future analyses are likely to have the same behavior. Limiting DAG width is
142// the safe approach and will be especially important with global DAGs.
143//
144// MaxParallelChains default is arbitrarily high to avoid affecting
145// optimization, but could be lowered to improve compile time. Any ld-ld-st-st
146// sequence over this should have been converted to llvm.memcpy by the
147// frontend. It is easy to induce this behavior with .ll code such as:
148// %buffer = alloca [4096 x i8]
149// %data = load [4096 x i8]* %argPtr
150// store [4096 x i8] %data, [4096 x i8]* %buffer
151static const unsigned MaxParallelChains = 64;
152
153static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL,
154 const SDValue *Parts, unsigned NumParts,
155 MVT PartVT, EVT ValueVT, const Value *V,
156 SDValue InChain,
157 std::optional<CallingConv::ID> CC);
158
159/// getCopyFromParts - Create a value that contains the specified legal parts
160/// combined into the value they represent. If the parts combine to a type
161/// larger than ValueVT then AssertOp can be used to specify whether the extra
162/// bits are known to be zero (ISD::AssertZext) or sign extended from ValueVT
163/// (ISD::AssertSext).
164static SDValue
165getCopyFromParts(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts,
166 unsigned NumParts, MVT PartVT, EVT ValueVT, const Value *V,
167 SDValue InChain,
168 std::optional<CallingConv::ID> CC = std::nullopt,
169 std::optional<ISD::NodeType> AssertOp = std::nullopt) {
170 // Let the target assemble the parts if it wants to
171 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
172 if (SDValue Val = TLI.joinRegisterPartsIntoValue(DAG, DL, Parts, NumParts,
173 PartVT, ValueVT, CC))
174 return Val;
175
176 if (ValueVT.isVector())
177 return getCopyFromPartsVector(DAG, DL, Parts, NumParts, PartVT, ValueVT, V,
178 InChain, CC);
179
180 assert(NumParts > 0 && "No parts to assemble!");
181 SDValue Val = Parts[0];
182
183 if (NumParts > 1) {
184 // Assemble the value from multiple parts.
185 if (ValueVT.isInteger()) {
186 unsigned PartBits = PartVT.getSizeInBits();
187 unsigned ValueBits = ValueVT.getSizeInBits();
188
189 // Assemble the power of 2 part.
190 unsigned RoundParts = llvm::bit_floor(Value: NumParts);
191 unsigned RoundBits = PartBits * RoundParts;
192 EVT RoundVT = RoundBits == ValueBits ?
193 ValueVT : EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: RoundBits);
194 SDValue Lo, Hi;
195
196 EVT HalfVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: RoundBits/2);
197
198 if (RoundParts > 2) {
199 Lo = getCopyFromParts(DAG, DL, Parts, NumParts: RoundParts / 2, PartVT, ValueVT: HalfVT, V,
200 InChain);
201 Hi = getCopyFromParts(DAG, DL, Parts: Parts + RoundParts / 2, NumParts: RoundParts / 2,
202 PartVT, ValueVT: HalfVT, V, InChain);
203 } else {
204 Lo = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: HalfVT, Operand: Parts[0]);
205 Hi = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: HalfVT, Operand: Parts[1]);
206 }
207
208 if (DAG.getDataLayout().isBigEndian())
209 std::swap(a&: Lo, b&: Hi);
210
211 Val = DAG.getNode(Opcode: ISD::BUILD_PAIR, DL, VT: RoundVT, N1: Lo, N2: Hi);
212
213 if (RoundParts < NumParts) {
214 // Assemble the trailing non-power-of-2 part.
215 unsigned OddParts = NumParts - RoundParts;
216 EVT OddVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: OddParts * PartBits);
217 Hi = getCopyFromParts(DAG, DL, Parts: Parts + RoundParts, NumParts: OddParts, PartVT,
218 ValueVT: OddVT, V, InChain, CC);
219
220 // Combine the round and odd parts.
221 Lo = Val;
222 if (DAG.getDataLayout().isBigEndian())
223 std::swap(a&: Lo, b&: Hi);
224 EVT TotalVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: NumParts * PartBits);
225 Hi = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: TotalVT, Operand: Hi);
226 Hi = DAG.getNode(Opcode: ISD::SHL, DL, VT: TotalVT, N1: Hi,
227 N2: DAG.getConstant(Val: Lo.getValueSizeInBits(), DL,
228 VT: TLI.getShiftAmountTy(
229 LHSTy: TotalVT, DL: DAG.getDataLayout())));
230 Lo = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: TotalVT, Operand: Lo);
231 Val = DAG.getNode(Opcode: ISD::OR, DL, VT: TotalVT, N1: Lo, N2: Hi);
232 }
233 } else if (PartVT.isFloatingPoint()) {
234 // FP split into multiple FP parts (for ppcf128)
235 assert(ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 &&
236 "Unexpected split");
237 SDValue Lo, Hi;
238 Lo = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[0]);
239 Hi = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[1]);
240 if (TLI.hasBigEndianPartOrdering(VT: ValueVT, DL: DAG.getDataLayout()))
241 std::swap(a&: Lo, b&: Hi);
242 Val = DAG.getNode(Opcode: ISD::BUILD_PAIR, DL, VT: ValueVT, N1: Lo, N2: Hi);
243 } else {
244 // FP split into integer parts (soft fp)
245 assert(ValueVT.isFloatingPoint() && PartVT.isInteger() &&
246 !PartVT.isVector() && "Unexpected split");
247 EVT IntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ValueVT.getSizeInBits());
248 Val = getCopyFromParts(DAG, DL, Parts, NumParts, PartVT, ValueVT: IntVT, V,
249 InChain, CC);
250 }
251 }
252
253 // There is now one part, held in Val. Correct it to match ValueVT.
254 // PartEVT is the type of the register class that holds the value.
255 // ValueVT is the type of the inline asm operation.
256 EVT PartEVT = Val.getValueType();
257
258 if (PartEVT == ValueVT)
259 return Val;
260
261 if (PartEVT.isInteger() && ValueVT.isFloatingPoint() &&
262 ValueVT.bitsLT(VT: PartEVT)) {
263 // For an FP value in an integer part, we need to truncate to the right
264 // width first.
265 PartEVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ValueVT.getSizeInBits());
266 Val = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: PartEVT, Operand: Val);
267 }
268
269 // Handle types that have the same size.
270 if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits())
271 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: ValueVT, Operand: Val);
272
273 // Handle types with different sizes.
274 if (PartEVT.isInteger() && ValueVT.isInteger()) {
275 if (ValueVT.bitsLT(VT: PartEVT)) {
276 // For a truncate, see if we have any information to
277 // indicate whether the truncated bits will always be
278 // zero or sign-extension.
279 if (AssertOp)
280 Val = DAG.getNode(Opcode: *AssertOp, DL, VT: PartEVT, N1: Val,
281 N2: DAG.getValueType(ValueVT));
282 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: ValueVT, Operand: Val);
283 }
284 return DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: ValueVT, Operand: Val);
285 }
286
287 if (PartEVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
288 // FP_ROUND's are always exact here.
289 if (ValueVT.bitsLT(VT: Val.getValueType())) {
290
291 SDValue NoChange =
292 DAG.getTargetConstant(Val: 1, DL, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
293
294 if (DAG.getMachineFunction().getFunction().getAttributes().hasFnAttr(
295 llvm::Attribute::StrictFP)) {
296 return DAG.getNode(ISD::STRICT_FP_ROUND, DL,
297 DAG.getVTList(ValueVT, MVT::Other), InChain, Val,
298 NoChange);
299 }
300
301 return DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: ValueVT, N1: Val, N2: NoChange);
302 }
303
304 return DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: ValueVT, Operand: Val);
305 }
306
307 // Handle MMX to a narrower integer type by bitcasting MMX to integer and
308 // then truncating.
309 if (PartEVT == MVT::x86mmx && ValueVT.isInteger() &&
310 ValueVT.bitsLT(VT: PartEVT)) {
311 Val = DAG.getNode(ISD::BITCAST, DL, MVT::i64, Val);
312 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: ValueVT, Operand: Val);
313 }
314
315 report_fatal_error(reason: "Unknown mismatch in getCopyFromParts!");
316}
317
318static void diagnosePossiblyInvalidConstraint(LLVMContext &Ctx, const Value *V,
319 const Twine &ErrMsg) {
320 const Instruction *I = dyn_cast_or_null<Instruction>(Val: V);
321 if (!I)
322 return Ctx.emitError(ErrorStr: ErrMsg);
323
324 if (const CallInst *CI = dyn_cast<CallInst>(Val: I))
325 if (CI->isInlineAsm()) {
326 return Ctx.diagnose(DI: DiagnosticInfoInlineAsm(
327 *CI, ErrMsg + ", possible invalid constraint for vector type"));
328 }
329
330 return Ctx.emitError(I, ErrorStr: ErrMsg);
331}
332
333/// getCopyFromPartsVector - Create a value that contains the specified legal
334/// parts combined into the value they represent. If the parts combine to a
335/// type larger than ValueVT then AssertOp can be used to specify whether the
336/// extra bits are known to be zero (ISD::AssertZext) or sign extended from
337/// ValueVT (ISD::AssertSext).
338static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL,
339 const SDValue *Parts, unsigned NumParts,
340 MVT PartVT, EVT ValueVT, const Value *V,
341 SDValue InChain,
342 std::optional<CallingConv::ID> CallConv) {
343 assert(ValueVT.isVector() && "Not a vector value");
344 assert(NumParts > 0 && "No parts to assemble!");
345 const bool IsABIRegCopy = CallConv.has_value();
346
347 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
348 SDValue Val = Parts[0];
349
350 // Handle a multi-element vector.
351 if (NumParts > 1) {
352 EVT IntermediateVT;
353 MVT RegisterVT;
354 unsigned NumIntermediates;
355 unsigned NumRegs;
356
357 if (IsABIRegCopy) {
358 NumRegs = TLI.getVectorTypeBreakdownForCallingConv(
359 Context&: *DAG.getContext(), CC: *CallConv, VT: ValueVT, IntermediateVT,
360 NumIntermediates, RegisterVT);
361 } else {
362 NumRegs =
363 TLI.getVectorTypeBreakdown(Context&: *DAG.getContext(), VT: ValueVT, IntermediateVT,
364 NumIntermediates, RegisterVT);
365 }
366
367 assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
368 NumParts = NumRegs; // Silence a compiler warning.
369 assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
370 assert(RegisterVT.getSizeInBits() ==
371 Parts[0].getSimpleValueType().getSizeInBits() &&
372 "Part type sizes don't match!");
373
374 // Assemble the parts into intermediate operands.
375 SmallVector<SDValue, 8> Ops(NumIntermediates);
376 if (NumIntermediates == NumParts) {
377 // If the register was not expanded, truncate or copy the value,
378 // as appropriate.
379 for (unsigned i = 0; i != NumParts; ++i)
380 Ops[i] = getCopyFromParts(DAG, DL, Parts: &Parts[i], NumParts: 1, PartVT, ValueVT: IntermediateVT,
381 V, InChain, CC: CallConv);
382 } else if (NumParts > 0) {
383 // If the intermediate type was expanded, build the intermediate
384 // operands from the parts.
385 assert(NumParts % NumIntermediates == 0 &&
386 "Must expand into a divisible number of parts!");
387 unsigned Factor = NumParts / NumIntermediates;
388 for (unsigned i = 0; i != NumIntermediates; ++i)
389 Ops[i] = getCopyFromParts(DAG, DL, Parts: &Parts[i * Factor], NumParts: Factor, PartVT,
390 ValueVT: IntermediateVT, V, InChain, CC: CallConv);
391 }
392
393 // Build a vector with BUILD_VECTOR or CONCAT_VECTORS from the
394 // intermediate operands.
395 EVT BuiltVectorTy =
396 IntermediateVT.isVector()
397 ? EVT::getVectorVT(
398 Context&: *DAG.getContext(), VT: IntermediateVT.getScalarType(),
399 EC: IntermediateVT.getVectorElementCount() * NumParts)
400 : EVT::getVectorVT(Context&: *DAG.getContext(),
401 VT: IntermediateVT.getScalarType(),
402 NumElements: NumIntermediates);
403 Val = DAG.getNode(Opcode: IntermediateVT.isVector() ? ISD::CONCAT_VECTORS
404 : ISD::BUILD_VECTOR,
405 DL, VT: BuiltVectorTy, Ops);
406 }
407
408 // There is now one part, held in Val. Correct it to match ValueVT.
409 EVT PartEVT = Val.getValueType();
410
411 if (PartEVT == ValueVT)
412 return Val;
413
414 if (PartEVT.isVector()) {
415 // Vector/Vector bitcast.
416 if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits())
417 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: ValueVT, Operand: Val);
418
419 // If the parts vector has more elements than the value vector, then we
420 // have a vector widening case (e.g. <2 x float> -> <4 x float>).
421 // Extract the elements we want.
422 if (PartEVT.getVectorElementCount() != ValueVT.getVectorElementCount()) {
423 assert((PartEVT.getVectorElementCount().getKnownMinValue() >
424 ValueVT.getVectorElementCount().getKnownMinValue()) &&
425 (PartEVT.getVectorElementCount().isScalable() ==
426 ValueVT.getVectorElementCount().isScalable()) &&
427 "Cannot narrow, it would be a lossy transformation");
428 PartEVT =
429 EVT::getVectorVT(Context&: *DAG.getContext(), VT: PartEVT.getVectorElementType(),
430 EC: ValueVT.getVectorElementCount());
431 Val = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: PartEVT, N1: Val,
432 N2: DAG.getVectorIdxConstant(Val: 0, DL));
433 if (PartEVT == ValueVT)
434 return Val;
435 if (PartEVT.isInteger() && ValueVT.isFloatingPoint())
436 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: ValueVT, Operand: Val);
437
438 // Vector/Vector bitcast (e.g. <2 x bfloat> -> <2 x half>).
439 if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits())
440 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: ValueVT, Operand: Val);
441 }
442
443 // Promoted vector extract
444 return DAG.getAnyExtOrTrunc(Op: Val, DL, VT: ValueVT);
445 }
446
447 // Trivial bitcast if the types are the same size and the destination
448 // vector type is legal.
449 if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits() &&
450 TLI.isTypeLegal(VT: ValueVT))
451 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: ValueVT, Operand: Val);
452
453 if (ValueVT.getVectorNumElements() != 1) {
454 // Certain ABIs require that vectors are passed as integers. For vectors
455 // are the same size, this is an obvious bitcast.
456 if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits()) {
457 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: ValueVT, Operand: Val);
458 } else if (ValueVT.bitsLT(VT: PartEVT)) {
459 const uint64_t ValueSize = ValueVT.getFixedSizeInBits();
460 EVT IntermediateType = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ValueSize);
461 // Drop the extra bits.
462 Val = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: IntermediateType, Operand: Val);
463 return DAG.getBitcast(VT: ValueVT, V: Val);
464 }
465
466 diagnosePossiblyInvalidConstraint(
467 Ctx&: *DAG.getContext(), V, ErrMsg: "non-trivial scalar-to-vector conversion");
468 return DAG.getUNDEF(VT: ValueVT);
469 }
470
471 // Handle cases such as i8 -> <1 x i1>
472 EVT ValueSVT = ValueVT.getVectorElementType();
473 if (ValueVT.getVectorNumElements() == 1 && ValueSVT != PartEVT) {
474 unsigned ValueSize = ValueSVT.getSizeInBits();
475 if (ValueSize == PartEVT.getSizeInBits()) {
476 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: ValueSVT, Operand: Val);
477 } else if (ValueSVT.isFloatingPoint() && PartEVT.isInteger()) {
478 // It's possible a scalar floating point type gets softened to integer and
479 // then promoted to a larger integer. If PartEVT is the larger integer
480 // we need to truncate it and then bitcast to the FP type.
481 assert(ValueSVT.bitsLT(PartEVT) && "Unexpected types");
482 EVT IntermediateType = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ValueSize);
483 Val = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: IntermediateType, Operand: Val);
484 Val = DAG.getBitcast(VT: ValueSVT, V: Val);
485 } else {
486 Val = ValueVT.isFloatingPoint()
487 ? DAG.getFPExtendOrRound(Op: Val, DL, VT: ValueSVT)
488 : DAG.getAnyExtOrTrunc(Op: Val, DL, VT: ValueSVT);
489 }
490 }
491
492 return DAG.getBuildVector(VT: ValueVT, DL, Ops: Val);
493}
494
495static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &dl,
496 SDValue Val, SDValue *Parts, unsigned NumParts,
497 MVT PartVT, const Value *V,
498 std::optional<CallingConv::ID> CallConv);
499
500/// getCopyToParts - Create a series of nodes that contain the specified value
501/// split into legal parts. If the parts contain more bits than Val, then, for
502/// integers, ExtendKind can be used to specify how to generate the extra bits.
503static void
504getCopyToParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts,
505 unsigned NumParts, MVT PartVT, const Value *V,
506 std::optional<CallingConv::ID> CallConv = std::nullopt,
507 ISD::NodeType ExtendKind = ISD::ANY_EXTEND) {
508 // Let the target split the parts if it wants to
509 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
510 if (TLI.splitValueIntoRegisterParts(DAG, DL, Val, Parts, NumParts, PartVT,
511 CC: CallConv))
512 return;
513 EVT ValueVT = Val.getValueType();
514
515 // Handle the vector case separately.
516 if (ValueVT.isVector())
517 return getCopyToPartsVector(DAG, dl: DL, Val, Parts, NumParts, PartVT, V,
518 CallConv);
519
520 unsigned OrigNumParts = NumParts;
521 assert(DAG.getTargetLoweringInfo().isTypeLegal(PartVT) &&
522 "Copying to an illegal type!");
523
524 if (NumParts == 0)
525 return;
526
527 assert(!ValueVT.isVector() && "Vector case handled elsewhere");
528 EVT PartEVT = PartVT;
529 if (PartEVT == ValueVT) {
530 assert(NumParts == 1 && "No-op copy with multiple parts!");
531 Parts[0] = Val;
532 return;
533 }
534
535 unsigned PartBits = PartVT.getSizeInBits();
536 if (NumParts * PartBits > ValueVT.getSizeInBits()) {
537 // If the parts cover more bits than the value has, promote the value.
538 if (PartVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
539 assert(NumParts == 1 && "Do not know what to promote to!");
540 Val = DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: PartVT, Operand: Val);
541 } else {
542 if (ValueVT.isFloatingPoint()) {
543 // FP values need to be bitcast, then extended if they are being put
544 // into a larger container.
545 ValueVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ValueVT.getSizeInBits());
546 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: ValueVT, Operand: Val);
547 }
548 assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
549 ValueVT.isInteger() &&
550 "Unknown mismatch!");
551 ValueVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: NumParts * PartBits);
552 Val = DAG.getNode(Opcode: ExtendKind, DL, VT: ValueVT, Operand: Val);
553 if (PartVT == MVT::x86mmx)
554 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: PartVT, Operand: Val);
555 }
556 } else if (PartBits == ValueVT.getSizeInBits()) {
557 // Different types of the same size.
558 assert(NumParts == 1 && PartEVT != ValueVT);
559 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: PartVT, Operand: Val);
560 } else if (NumParts * PartBits < ValueVT.getSizeInBits()) {
561 // If the parts cover less bits than value has, truncate the value.
562 assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
563 ValueVT.isInteger() &&
564 "Unknown mismatch!");
565 ValueVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: NumParts * PartBits);
566 Val = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: ValueVT, Operand: Val);
567 if (PartVT == MVT::x86mmx)
568 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: PartVT, Operand: Val);
569 }
570
571 // The value may have changed - recompute ValueVT.
572 ValueVT = Val.getValueType();
573 assert(NumParts * PartBits == ValueVT.getSizeInBits() &&
574 "Failed to tile the value with PartVT!");
575
576 if (NumParts == 1) {
577 if (PartEVT != ValueVT) {
578 diagnosePossiblyInvalidConstraint(Ctx&: *DAG.getContext(), V,
579 ErrMsg: "scalar-to-vector conversion failed");
580 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: PartVT, Operand: Val);
581 }
582
583 Parts[0] = Val;
584 return;
585 }
586
587 // Expand the value into multiple parts.
588 if (NumParts & (NumParts - 1)) {
589 // The number of parts is not a power of 2. Split off and copy the tail.
590 assert(PartVT.isInteger() && ValueVT.isInteger() &&
591 "Do not know what to expand to!");
592 unsigned RoundParts = llvm::bit_floor(Value: NumParts);
593 unsigned RoundBits = RoundParts * PartBits;
594 unsigned OddParts = NumParts - RoundParts;
595 SDValue OddVal = DAG.getNode(Opcode: ISD::SRL, DL, VT: ValueVT, N1: Val,
596 N2: DAG.getShiftAmountConstant(Val: RoundBits, VT: ValueVT, DL));
597
598 getCopyToParts(DAG, DL, Val: OddVal, Parts: Parts + RoundParts, NumParts: OddParts, PartVT, V,
599 CallConv);
600
601 if (DAG.getDataLayout().isBigEndian())
602 // The odd parts were reversed by getCopyToParts - unreverse them.
603 std::reverse(first: Parts + RoundParts, last: Parts + NumParts);
604
605 NumParts = RoundParts;
606 ValueVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: NumParts * PartBits);
607 Val = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: ValueVT, Operand: Val);
608 }
609
610 // The number of parts is a power of 2. Repeatedly bisect the value using
611 // EXTRACT_ELEMENT.
612 Parts[0] = DAG.getNode(Opcode: ISD::BITCAST, DL,
613 VT: EVT::getIntegerVT(Context&: *DAG.getContext(),
614 BitWidth: ValueVT.getSizeInBits()),
615 Operand: Val);
616
617 for (unsigned StepSize = NumParts; StepSize > 1; StepSize /= 2) {
618 for (unsigned i = 0; i < NumParts; i += StepSize) {
619 unsigned ThisBits = StepSize * PartBits / 2;
620 EVT ThisVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ThisBits);
621 SDValue &Part0 = Parts[i];
622 SDValue &Part1 = Parts[i+StepSize/2];
623
624 Part1 = DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL,
625 VT: ThisVT, N1: Part0, N2: DAG.getIntPtrConstant(Val: 1, DL));
626 Part0 = DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL,
627 VT: ThisVT, N1: Part0, N2: DAG.getIntPtrConstant(Val: 0, DL));
628
629 if (ThisBits == PartBits && ThisVT != PartVT) {
630 Part0 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: PartVT, Operand: Part0);
631 Part1 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: PartVT, Operand: Part1);
632 }
633 }
634 }
635
636 if (DAG.getDataLayout().isBigEndian())
637 std::reverse(first: Parts, last: Parts + OrigNumParts);
638}
639
640static SDValue widenVectorToPartType(SelectionDAG &DAG, SDValue Val,
641 const SDLoc &DL, EVT PartVT) {
642 if (!PartVT.isVector())
643 return SDValue();
644
645 EVT ValueVT = Val.getValueType();
646 EVT PartEVT = PartVT.getVectorElementType();
647 EVT ValueEVT = ValueVT.getVectorElementType();
648 ElementCount PartNumElts = PartVT.getVectorElementCount();
649 ElementCount ValueNumElts = ValueVT.getVectorElementCount();
650
651 // We only support widening vectors with equivalent element types and
652 // fixed/scalable properties. If a target needs to widen a fixed-length type
653 // to a scalable one, it should be possible to use INSERT_SUBVECTOR below.
654 if (ElementCount::isKnownLE(LHS: PartNumElts, RHS: ValueNumElts) ||
655 PartNumElts.isScalable() != ValueNumElts.isScalable())
656 return SDValue();
657
658 // Have a try for bf16 because some targets share its ABI with fp16.
659 if (ValueEVT == MVT::bf16 && PartEVT == MVT::f16) {
660 assert(DAG.getTargetLoweringInfo().isTypeLegal(PartVT) &&
661 "Cannot widen to illegal type");
662 Val = DAG.getNode(ISD::BITCAST, DL,
663 ValueVT.changeVectorElementType(MVT::EltVT: f16), Val);
664 } else if (PartEVT != ValueEVT) {
665 return SDValue();
666 }
667
668 // Widening a scalable vector to another scalable vector is done by inserting
669 // the vector into a larger undef one.
670 if (PartNumElts.isScalable())
671 return DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT: PartVT, N1: DAG.getUNDEF(VT: PartVT),
672 N2: Val, N3: DAG.getVectorIdxConstant(Val: 0, DL));
673
674 // Vector widening case, e.g. <2 x float> -> <4 x float>. Shuffle in
675 // undef elements.
676 SmallVector<SDValue, 16> Ops;
677 DAG.ExtractVectorElements(Op: Val, Args&: Ops);
678 SDValue EltUndef = DAG.getUNDEF(VT: PartEVT);
679 Ops.append(NumInputs: (PartNumElts - ValueNumElts).getFixedValue(), Elt: EltUndef);
680
681 // FIXME: Use CONCAT for 2x -> 4x.
682 return DAG.getBuildVector(VT: PartVT, DL, Ops);
683}
684
685/// getCopyToPartsVector - Create a series of nodes that contain the specified
686/// value split into legal parts.
687static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &DL,
688 SDValue Val, SDValue *Parts, unsigned NumParts,
689 MVT PartVT, const Value *V,
690 std::optional<CallingConv::ID> CallConv) {
691 EVT ValueVT = Val.getValueType();
692 assert(ValueVT.isVector() && "Not a vector");
693 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
694 const bool IsABIRegCopy = CallConv.has_value();
695
696 if (NumParts == 1) {
697 EVT PartEVT = PartVT;
698 if (PartEVT == ValueVT) {
699 // Nothing to do.
700 } else if (PartVT.getSizeInBits() == ValueVT.getSizeInBits()) {
701 // Bitconvert vector->vector case.
702 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: PartVT, Operand: Val);
703 } else if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, PartVT)) {
704 Val = Widened;
705 } else if (PartVT.isVector() &&
706 PartEVT.getVectorElementType().bitsGE(
707 VT: ValueVT.getVectorElementType()) &&
708 PartEVT.getVectorElementCount() ==
709 ValueVT.getVectorElementCount()) {
710
711 // Promoted vector extract
712 Val = DAG.getAnyExtOrTrunc(Op: Val, DL, VT: PartVT);
713 } else if (PartEVT.isVector() &&
714 PartEVT.getVectorElementType() !=
715 ValueVT.getVectorElementType() &&
716 TLI.getTypeAction(Context&: *DAG.getContext(), VT: ValueVT) ==
717 TargetLowering::TypeWidenVector) {
718 // Combination of widening and promotion.
719 EVT WidenVT =
720 EVT::getVectorVT(Context&: *DAG.getContext(), VT: ValueVT.getVectorElementType(),
721 EC: PartVT.getVectorElementCount());
722 SDValue Widened = widenVectorToPartType(DAG, Val, DL, PartVT: WidenVT);
723 Val = DAG.getAnyExtOrTrunc(Op: Widened, DL, VT: PartVT);
724 } else {
725 // Don't extract an integer from a float vector. This can happen if the
726 // FP type gets softened to integer and then promoted. The promotion
727 // prevents it from being picked up by the earlier bitcast case.
728 if (ValueVT.getVectorElementCount().isScalar() &&
729 (!ValueVT.isFloatingPoint() || !PartVT.isInteger())) {
730 // If we reach this condition and PartVT is FP, this means that
731 // ValueVT is also FP and both have a different size, otherwise we
732 // would have bitcasted them. Producing an EXTRACT_VECTOR_ELT here
733 // would be invalid since that would mean the smaller FP type has to
734 // be extended to the larger one.
735 if (PartVT.isFloatingPoint()) {
736 Val = DAG.getBitcast(VT: ValueVT.getScalarType(), V: Val);
737 Val = DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: PartVT, Operand: Val);
738 } else
739 Val = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: PartVT, N1: Val,
740 N2: DAG.getVectorIdxConstant(Val: 0, DL));
741 } else {
742 uint64_t ValueSize = ValueVT.getFixedSizeInBits();
743 assert(PartVT.getFixedSizeInBits() > ValueSize &&
744 "lossy conversion of vector to scalar type");
745 EVT IntermediateType = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ValueSize);
746 Val = DAG.getBitcast(VT: IntermediateType, V: Val);
747 Val = DAG.getAnyExtOrTrunc(Op: Val, DL, VT: PartVT);
748 }
749 }
750
751 assert(Val.getValueType() == PartVT && "Unexpected vector part value type");
752 Parts[0] = Val;
753 return;
754 }
755
756 // Handle a multi-element vector.
757 EVT IntermediateVT;
758 MVT RegisterVT;
759 unsigned NumIntermediates;
760 unsigned NumRegs;
761 if (IsABIRegCopy) {
762 NumRegs = TLI.getVectorTypeBreakdownForCallingConv(
763 Context&: *DAG.getContext(), CC: *CallConv, VT: ValueVT, IntermediateVT, NumIntermediates,
764 RegisterVT);
765 } else {
766 NumRegs =
767 TLI.getVectorTypeBreakdown(Context&: *DAG.getContext(), VT: ValueVT, IntermediateVT,
768 NumIntermediates, RegisterVT);
769 }
770
771 assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
772 NumParts = NumRegs; // Silence a compiler warning.
773 assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
774
775 assert(IntermediateVT.isScalableVector() == ValueVT.isScalableVector() &&
776 "Mixing scalable and fixed vectors when copying in parts");
777
778 std::optional<ElementCount> DestEltCnt;
779
780 if (IntermediateVT.isVector())
781 DestEltCnt = IntermediateVT.getVectorElementCount() * NumIntermediates;
782 else
783 DestEltCnt = ElementCount::getFixed(MinVal: NumIntermediates);
784
785 EVT BuiltVectorTy = EVT::getVectorVT(
786 Context&: *DAG.getContext(), VT: IntermediateVT.getScalarType(), EC: *DestEltCnt);
787
788 if (ValueVT == BuiltVectorTy) {
789 // Nothing to do.
790 } else if (ValueVT.getSizeInBits() == BuiltVectorTy.getSizeInBits()) {
791 // Bitconvert vector->vector case.
792 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: BuiltVectorTy, Operand: Val);
793 } else {
794 if (BuiltVectorTy.getVectorElementType().bitsGT(
795 VT: ValueVT.getVectorElementType())) {
796 // Integer promotion.
797 ValueVT = EVT::getVectorVT(Context&: *DAG.getContext(),
798 VT: BuiltVectorTy.getVectorElementType(),
799 EC: ValueVT.getVectorElementCount());
800 Val = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: ValueVT, Operand: Val);
801 }
802
803 if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, PartVT: BuiltVectorTy)) {
804 Val = Widened;
805 }
806 }
807
808 assert(Val.getValueType() == BuiltVectorTy && "Unexpected vector value type");
809
810 // Split the vector into intermediate operands.
811 SmallVector<SDValue, 8> Ops(NumIntermediates);
812 for (unsigned i = 0; i != NumIntermediates; ++i) {
813 if (IntermediateVT.isVector()) {
814 // This does something sensible for scalable vectors - see the
815 // definition of EXTRACT_SUBVECTOR for further details.
816 unsigned IntermediateNumElts = IntermediateVT.getVectorMinNumElements();
817 Ops[i] =
818 DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: IntermediateVT, N1: Val,
819 N2: DAG.getVectorIdxConstant(Val: i * IntermediateNumElts, DL));
820 } else {
821 Ops[i] = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: IntermediateVT, N1: Val,
822 N2: DAG.getVectorIdxConstant(Val: i, DL));
823 }
824 }
825
826 // Split the intermediate operands into legal parts.
827 if (NumParts == NumIntermediates) {
828 // If the register was not expanded, promote or copy the value,
829 // as appropriate.
830 for (unsigned i = 0; i != NumParts; ++i)
831 getCopyToParts(DAG, DL, Val: Ops[i], Parts: &Parts[i], NumParts: 1, PartVT, V, CallConv);
832 } else if (NumParts > 0) {
833 // If the intermediate type was expanded, split each the value into
834 // legal parts.
835 assert(NumIntermediates != 0 && "division by zero");
836 assert(NumParts % NumIntermediates == 0 &&
837 "Must expand into a divisible number of parts!");
838 unsigned Factor = NumParts / NumIntermediates;
839 for (unsigned i = 0; i != NumIntermediates; ++i)
840 getCopyToParts(DAG, DL, Val: Ops[i], Parts: &Parts[i * Factor], NumParts: Factor, PartVT, V,
841 CallConv);
842 }
843}
844
845RegsForValue::RegsForValue(const SmallVector<Register, 4> &regs, MVT regvt,
846 EVT valuevt, std::optional<CallingConv::ID> CC)
847 : ValueVTs(1, valuevt), RegVTs(1, regvt), Regs(regs),
848 RegCount(1, regs.size()), CallConv(CC) {}
849
850RegsForValue::RegsForValue(LLVMContext &Context, const TargetLowering &TLI,
851 const DataLayout &DL, Register Reg, Type *Ty,
852 std::optional<CallingConv::ID> CC) {
853 ComputeValueVTs(TLI, DL, Ty, ValueVTs);
854
855 CallConv = CC;
856
857 for (EVT ValueVT : ValueVTs) {
858 unsigned NumRegs =
859 isABIMangled()
860 ? TLI.getNumRegistersForCallingConv(Context, CC: *CC, VT: ValueVT)
861 : TLI.getNumRegisters(Context, VT: ValueVT);
862 MVT RegisterVT =
863 isABIMangled()
864 ? TLI.getRegisterTypeForCallingConv(Context, CC: *CC, VT: ValueVT)
865 : TLI.getRegisterType(Context, VT: ValueVT);
866 for (unsigned i = 0; i != NumRegs; ++i)
867 Regs.push_back(Elt: Reg + i);
868 RegVTs.push_back(Elt: RegisterVT);
869 RegCount.push_back(Elt: NumRegs);
870 Reg = Reg.id() + NumRegs;
871 }
872}
873
874SDValue RegsForValue::getCopyFromRegs(SelectionDAG &DAG,
875 FunctionLoweringInfo &FuncInfo,
876 const SDLoc &dl, SDValue &Chain,
877 SDValue *Glue, const Value *V) const {
878 // A Value with type {} or [0 x %t] needs no registers.
879 if (ValueVTs.empty())
880 return SDValue();
881
882 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
883
884 // Assemble the legal parts into the final values.
885 SmallVector<SDValue, 4> Values(ValueVTs.size());
886 SmallVector<SDValue, 8> Parts;
887 for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
888 // Copy the legal parts from the registers.
889 EVT ValueVT = ValueVTs[Value];
890 unsigned NumRegs = RegCount[Value];
891 MVT RegisterVT = isABIMangled()
892 ? TLI.getRegisterTypeForCallingConv(
893 Context&: *DAG.getContext(), CC: *CallConv, VT: RegVTs[Value])
894 : RegVTs[Value];
895
896 Parts.resize(N: NumRegs);
897 for (unsigned i = 0; i != NumRegs; ++i) {
898 SDValue P;
899 if (!Glue) {
900 P = DAG.getCopyFromReg(Chain, dl, Reg: Regs[Part+i], VT: RegisterVT);
901 } else {
902 P = DAG.getCopyFromReg(Chain, dl, Reg: Regs[Part+i], VT: RegisterVT, Glue: *Glue);
903 *Glue = P.getValue(R: 2);
904 }
905
906 Chain = P.getValue(R: 1);
907 Parts[i] = P;
908
909 // If the source register was virtual and if we know something about it,
910 // add an assert node.
911 if (!Regs[Part + i].isVirtual() || !RegisterVT.isInteger())
912 continue;
913
914 const FunctionLoweringInfo::LiveOutInfo *LOI =
915 FuncInfo.GetLiveOutRegInfo(Reg: Regs[Part+i]);
916 if (!LOI)
917 continue;
918
919 unsigned RegSize = RegisterVT.getScalarSizeInBits();
920 unsigned NumSignBits = LOI->NumSignBits;
921 unsigned NumZeroBits = LOI->Known.countMinLeadingZeros();
922
923 if (NumZeroBits == RegSize) {
924 // The current value is a zero.
925 // Explicitly express that as it would be easier for
926 // optimizations to kick in.
927 Parts[i] = DAG.getConstant(Val: 0, DL: dl, VT: RegisterVT);
928 continue;
929 }
930
931 // FIXME: We capture more information than the dag can represent. For
932 // now, just use the tightest assertzext/assertsext possible.
933 bool isSExt;
934 EVT FromVT(MVT::Other);
935 if (NumZeroBits) {
936 FromVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: RegSize - NumZeroBits);
937 isSExt = false;
938 } else if (NumSignBits > 1) {
939 FromVT =
940 EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: RegSize - NumSignBits + 1);
941 isSExt = true;
942 } else {
943 continue;
944 }
945 // Add an assertion node.
946 assert(FromVT != MVT::Other);
947 Parts[i] = DAG.getNode(isSExt ? ISD::AssertSext : ISD::AssertZext, dl,
948 RegisterVT, P, DAG.getValueType(FromVT));
949 }
950
951 Values[Value] = getCopyFromParts(DAG, DL: dl, Parts: Parts.begin(), NumParts: NumRegs,
952 PartVT: RegisterVT, ValueVT, V, InChain: Chain, CC: CallConv);
953 Part += NumRegs;
954 Parts.clear();
955 }
956
957 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: dl, VTList: DAG.getVTList(VTs: ValueVTs), Ops: Values);
958}
959
960void RegsForValue::getCopyToRegs(SDValue Val, SelectionDAG &DAG,
961 const SDLoc &dl, SDValue &Chain, SDValue *Glue,
962 const Value *V,
963 ISD::NodeType PreferredExtendType) const {
964 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
965 ISD::NodeType ExtendKind = PreferredExtendType;
966
967 // Get the list of the values's legal parts.
968 unsigned NumRegs = Regs.size();
969 SmallVector<SDValue, 8> Parts(NumRegs);
970 for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
971 unsigned NumParts = RegCount[Value];
972
973 MVT RegisterVT = isABIMangled()
974 ? TLI.getRegisterTypeForCallingConv(
975 Context&: *DAG.getContext(), CC: *CallConv, VT: RegVTs[Value])
976 : RegVTs[Value];
977
978 if (ExtendKind == ISD::ANY_EXTEND && TLI.isZExtFree(Val, VT2: RegisterVT))
979 ExtendKind = ISD::ZERO_EXTEND;
980
981 getCopyToParts(DAG, DL: dl, Val: Val.getValue(R: Val.getResNo() + Value), Parts: &Parts[Part],
982 NumParts, PartVT: RegisterVT, V, CallConv, ExtendKind);
983 Part += NumParts;
984 }
985
986 // Copy the parts into the registers.
987 SmallVector<SDValue, 8> Chains(NumRegs);
988 for (unsigned i = 0; i != NumRegs; ++i) {
989 SDValue Part;
990 if (!Glue) {
991 Part = DAG.getCopyToReg(Chain, dl, Reg: Regs[i], N: Parts[i]);
992 } else {
993 Part = DAG.getCopyToReg(Chain, dl, Reg: Regs[i], N: Parts[i], Glue: *Glue);
994 *Glue = Part.getValue(R: 1);
995 }
996
997 Chains[i] = Part.getValue(R: 0);
998 }
999
1000 if (NumRegs == 1 || Glue)
1001 // If NumRegs > 1 && Glue is used then the use of the last CopyToReg is
1002 // flagged to it. That is the CopyToReg nodes and the user are considered
1003 // a single scheduling unit. If we create a TokenFactor and return it as
1004 // chain, then the TokenFactor is both a predecessor (operand) of the
1005 // user as well as a successor (the TF operands are flagged to the user).
1006 // c1, f1 = CopyToReg
1007 // c2, f2 = CopyToReg
1008 // c3 = TokenFactor c1, c2
1009 // ...
1010 // = op c3, ..., f2
1011 Chain = Chains[NumRegs-1];
1012 else
1013 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
1014}
1015
1016void RegsForValue::AddInlineAsmOperands(InlineAsm::Kind Code, bool HasMatching,
1017 unsigned MatchingIdx, const SDLoc &dl,
1018 SelectionDAG &DAG,
1019 std::vector<SDValue> &Ops) const {
1020 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1021
1022 InlineAsm::Flag Flag(Code, Regs.size());
1023 if (HasMatching)
1024 Flag.setMatchingOp(MatchingIdx);
1025 else if (!Regs.empty() && Regs.front().isVirtual()) {
1026 // Put the register class of the virtual registers in the flag word. That
1027 // way, later passes can recompute register class constraints for inline
1028 // assembly as well as normal instructions.
1029 // Don't do this for tied operands that can use the regclass information
1030 // from the def.
1031 const MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
1032 const TargetRegisterClass *RC = MRI.getRegClass(Reg: Regs.front());
1033 Flag.setRegClass(RC->getID());
1034 }
1035
1036 SDValue Res = DAG.getTargetConstant(Flag, dl, MVT::i32);
1037 Ops.push_back(x: Res);
1038
1039 if (Code == InlineAsm::Kind::Clobber) {
1040 // Clobbers should always have a 1:1 mapping with registers, and may
1041 // reference registers that have illegal (e.g. vector) types. Hence, we
1042 // shouldn't try to apply any sort of splitting logic to them.
1043 assert(Regs.size() == RegVTs.size() && Regs.size() == ValueVTs.size() &&
1044 "No 1:1 mapping from clobbers to regs?");
1045 Register SP = TLI.getStackPointerRegisterToSaveRestore();
1046 (void)SP;
1047 for (unsigned I = 0, E = ValueVTs.size(); I != E; ++I) {
1048 Ops.push_back(x: DAG.getRegister(Reg: Regs[I], VT: RegVTs[I]));
1049 assert(
1050 (Regs[I] != SP ||
1051 DAG.getMachineFunction().getFrameInfo().hasOpaqueSPAdjustment()) &&
1052 "If we clobbered the stack pointer, MFI should know about it.");
1053 }
1054 return;
1055 }
1056
1057 for (unsigned Value = 0, Reg = 0, e = ValueVTs.size(); Value != e; ++Value) {
1058 MVT RegisterVT = RegVTs[Value];
1059 unsigned NumRegs = TLI.getNumRegisters(Context&: *DAG.getContext(), VT: ValueVTs[Value],
1060 RegisterVT);
1061 for (unsigned i = 0; i != NumRegs; ++i) {
1062 assert(Reg < Regs.size() && "Mismatch in # registers expected");
1063 Register TheReg = Regs[Reg++];
1064 Ops.push_back(x: DAG.getRegister(Reg: TheReg, VT: RegisterVT));
1065 }
1066 }
1067}
1068
1069SmallVector<std::pair<Register, TypeSize>, 4>
1070RegsForValue::getRegsAndSizes() const {
1071 SmallVector<std::pair<Register, TypeSize>, 4> OutVec;
1072 unsigned I = 0;
1073 for (auto CountAndVT : zip_first(t: RegCount, u: RegVTs)) {
1074 unsigned RegCount = std::get<0>(t&: CountAndVT);
1075 MVT RegisterVT = std::get<1>(t&: CountAndVT);
1076 TypeSize RegisterSize = RegisterVT.getSizeInBits();
1077 for (unsigned E = I + RegCount; I != E; ++I)
1078 OutVec.push_back(Elt: std::make_pair(x: Regs[I], y&: RegisterSize));
1079 }
1080 return OutVec;
1081}
1082
1083void SelectionDAGBuilder::init(GCFunctionInfo *gfi, BatchAAResults *aa,
1084 AssumptionCache *ac,
1085 const TargetLibraryInfo *li) {
1086 BatchAA = aa;
1087 AC = ac;
1088 GFI = gfi;
1089 LibInfo = li;
1090 Context = DAG.getContext();
1091 LPadToCallSiteMap.clear();
1092 SL->init(tli: DAG.getTargetLoweringInfo(), tm: TM, dl: DAG.getDataLayout());
1093 AssignmentTrackingEnabled = isAssignmentTrackingEnabled(
1094 M: *DAG.getMachineFunction().getFunction().getParent());
1095}
1096
1097void SelectionDAGBuilder::clear() {
1098 NodeMap.clear();
1099 UnusedArgNodeMap.clear();
1100 PendingLoads.clear();
1101 PendingExports.clear();
1102 PendingConstrainedFP.clear();
1103 PendingConstrainedFPStrict.clear();
1104 CurInst = nullptr;
1105 HasTailCall = false;
1106 SDNodeOrder = LowestSDNodeOrder;
1107 StatepointLowering.clear();
1108}
1109
1110void SelectionDAGBuilder::clearDanglingDebugInfo() {
1111 DanglingDebugInfoMap.clear();
1112}
1113
1114// Update DAG root to include dependencies on Pending chains.
1115SDValue SelectionDAGBuilder::updateRoot(SmallVectorImpl<SDValue> &Pending) {
1116 SDValue Root = DAG.getRoot();
1117
1118 if (Pending.empty())
1119 return Root;
1120
1121 // Add current root to PendingChains, unless we already indirectly
1122 // depend on it.
1123 if (Root.getOpcode() != ISD::EntryToken) {
1124 unsigned i = 0, e = Pending.size();
1125 for (; i != e; ++i) {
1126 assert(Pending[i].getNode()->getNumOperands() > 1);
1127 if (Pending[i].getNode()->getOperand(Num: 0) == Root)
1128 break; // Don't add the root if we already indirectly depend on it.
1129 }
1130
1131 if (i == e)
1132 Pending.push_back(Elt: Root);
1133 }
1134
1135 if (Pending.size() == 1)
1136 Root = Pending[0];
1137 else
1138 Root = DAG.getTokenFactor(DL: getCurSDLoc(), Vals&: Pending);
1139
1140 DAG.setRoot(Root);
1141 Pending.clear();
1142 return Root;
1143}
1144
1145SDValue SelectionDAGBuilder::getMemoryRoot() {
1146 return updateRoot(Pending&: PendingLoads);
1147}
1148
1149SDValue SelectionDAGBuilder::getRoot() {
1150 // Chain up all pending constrained intrinsics together with all
1151 // pending loads, by simply appending them to PendingLoads and
1152 // then calling getMemoryRoot().
1153 PendingLoads.reserve(N: PendingLoads.size() +
1154 PendingConstrainedFP.size() +
1155 PendingConstrainedFPStrict.size());
1156 PendingLoads.append(in_start: PendingConstrainedFP.begin(),
1157 in_end: PendingConstrainedFP.end());
1158 PendingLoads.append(in_start: PendingConstrainedFPStrict.begin(),
1159 in_end: PendingConstrainedFPStrict.end());
1160 PendingConstrainedFP.clear();
1161 PendingConstrainedFPStrict.clear();
1162 return getMemoryRoot();
1163}
1164
1165SDValue SelectionDAGBuilder::getControlRoot() {
1166 // We need to emit pending fpexcept.strict constrained intrinsics,
1167 // so append them to the PendingExports list.
1168 PendingExports.append(in_start: PendingConstrainedFPStrict.begin(),
1169 in_end: PendingConstrainedFPStrict.end());
1170 PendingConstrainedFPStrict.clear();
1171 return updateRoot(Pending&: PendingExports);
1172}
1173
1174void SelectionDAGBuilder::handleDebugDeclare(Value *Address,
1175 DILocalVariable *Variable,
1176 DIExpression *Expression,
1177 DebugLoc DL) {
1178 assert(Variable && "Missing variable");
1179
1180 // Check if address has undef value.
1181 if (!Address || isa<UndefValue>(Val: Address) ||
1182 (Address->use_empty() && !isa<Argument>(Val: Address))) {
1183 LLVM_DEBUG(
1184 dbgs()
1185 << "dbg_declare: Dropping debug info (bad/undef/unused-arg address)\n");
1186 return;
1187 }
1188
1189 bool IsParameter = Variable->isParameter() || isa<Argument>(Val: Address);
1190
1191 SDValue &N = NodeMap[Address];
1192 if (!N.getNode() && isa<Argument>(Val: Address))
1193 // Check unused arguments map.
1194 N = UnusedArgNodeMap[Address];
1195 SDDbgValue *SDV;
1196 if (N.getNode()) {
1197 if (const BitCastInst *BCI = dyn_cast<BitCastInst>(Val: Address))
1198 Address = BCI->getOperand(i_nocapture: 0);
1199 // Parameters are handled specially.
1200 auto *FINode = dyn_cast<FrameIndexSDNode>(Val: N.getNode());
1201 if (IsParameter && FINode) {
1202 // Byval parameter. We have a frame index at this point.
1203 SDV = DAG.getFrameIndexDbgValue(Var: Variable, Expr: Expression, FI: FINode->getIndex(),
1204 /*IsIndirect*/ true, DL, O: SDNodeOrder);
1205 } else if (isa<Argument>(Val: Address)) {
1206 // Address is an argument, so try to emit its dbg value using
1207 // virtual register info from the FuncInfo.ValueMap.
1208 EmitFuncArgumentDbgValue(V: Address, Variable, Expr: Expression, DL,
1209 Kind: FuncArgumentDbgValueKind::Declare, N);
1210 return;
1211 } else {
1212 SDV = DAG.getDbgValue(Var: Variable, Expr: Expression, N: N.getNode(), R: N.getResNo(),
1213 IsIndirect: true, DL, O: SDNodeOrder);
1214 }
1215 DAG.AddDbgValue(DB: SDV, isParameter: IsParameter);
1216 } else {
1217 // If Address is an argument then try to emit its dbg value using
1218 // virtual register info from the FuncInfo.ValueMap.
1219 if (!EmitFuncArgumentDbgValue(V: Address, Variable, Expr: Expression, DL,
1220 Kind: FuncArgumentDbgValueKind::Declare, N)) {
1221 LLVM_DEBUG(dbgs() << "dbg_declare: Dropping debug info"
1222 << " (could not emit func-arg dbg_value)\n");
1223 }
1224 }
1225}
1226
1227void SelectionDAGBuilder::visitDbgInfo(const Instruction &I) {
1228 // Add SDDbgValue nodes for any var locs here. Do so before updating
1229 // SDNodeOrder, as this mapping is {Inst -> Locs BEFORE Inst}.
1230 if (FunctionVarLocs const *FnVarLocs = DAG.getFunctionVarLocs()) {
1231 // Add SDDbgValue nodes for any var locs here. Do so before updating
1232 // SDNodeOrder, as this mapping is {Inst -> Locs BEFORE Inst}.
1233 for (auto It = FnVarLocs->locs_begin(Before: &I), End = FnVarLocs->locs_end(Before: &I);
1234 It != End; ++It) {
1235 auto *Var = FnVarLocs->getDILocalVariable(ID: It->VariableID);
1236 dropDanglingDebugInfo(Variable: Var, Expr: It->Expr);
1237 if (It->Values.isKillLocation(Expression: It->Expr)) {
1238 handleKillDebugValue(Var, Expr: It->Expr, DbgLoc: It->DL, Order: SDNodeOrder);
1239 continue;
1240 }
1241 SmallVector<Value *> Values(It->Values.location_ops());
1242 if (!handleDebugValue(Values, Var, Expr: It->Expr, DbgLoc: It->DL, Order: SDNodeOrder,
1243 IsVariadic: It->Values.hasArgList())) {
1244 SmallVector<Value *, 4> Vals(It->Values.location_ops());
1245 addDanglingDebugInfo(Values&: Vals,
1246 Var: FnVarLocs->getDILocalVariable(ID: It->VariableID),
1247 Expr: It->Expr, IsVariadic: Vals.size() > 1, DL: It->DL, Order: SDNodeOrder);
1248 }
1249 }
1250 }
1251
1252 // We must skip DbgVariableRecords if they've already been processed above as
1253 // we have just emitted the debug values resulting from assignment tracking
1254 // analysis, making any existing DbgVariableRecords redundant (and probably
1255 // less correct). We still need to process DbgLabelRecords. This does sink
1256 // DbgLabelRecords to the bottom of the group of debug records. That sholdn't
1257 // be important as it does so deterministcally and ordering between
1258 // DbgLabelRecords and DbgVariableRecords is immaterial (other than for MIR/IR
1259 // printing).
1260 bool SkipDbgVariableRecords = DAG.getFunctionVarLocs();
1261 // Is there is any debug-info attached to this instruction, in the form of
1262 // DbgRecord non-instruction debug-info records.
1263 for (DbgRecord &DR : I.getDbgRecordRange()) {
1264 if (DbgLabelRecord *DLR = dyn_cast<DbgLabelRecord>(Val: &DR)) {
1265 assert(DLR->getLabel() && "Missing label");
1266 SDDbgLabel *SDV =
1267 DAG.getDbgLabel(Label: DLR->getLabel(), DL: DLR->getDebugLoc(), O: SDNodeOrder);
1268 DAG.AddDbgLabel(DB: SDV);
1269 continue;
1270 }
1271
1272 if (SkipDbgVariableRecords)
1273 continue;
1274 DbgVariableRecord &DVR = cast<DbgVariableRecord>(Val&: DR);
1275 DILocalVariable *Variable = DVR.getVariable();
1276 DIExpression *Expression = DVR.getExpression();
1277 dropDanglingDebugInfo(Variable, Expr: Expression);
1278
1279 if (DVR.getType() == DbgVariableRecord::LocationType::Declare) {
1280 if (FuncInfo.PreprocessedDVRDeclares.contains(Ptr: &DVR))
1281 continue;
1282 LLVM_DEBUG(dbgs() << "SelectionDAG visiting dbg_declare: " << DVR
1283 << "\n");
1284 handleDebugDeclare(Address: DVR.getVariableLocationOp(OpIdx: 0), Variable, Expression,
1285 DL: DVR.getDebugLoc());
1286 continue;
1287 }
1288
1289 // A DbgVariableRecord with no locations is a kill location.
1290 SmallVector<Value *, 4> Values(DVR.location_ops());
1291 if (Values.empty()) {
1292 handleKillDebugValue(Var: Variable, Expr: Expression, DbgLoc: DVR.getDebugLoc(),
1293 Order: SDNodeOrder);
1294 continue;
1295 }
1296
1297 // A DbgVariableRecord with an undef or absent location is also a kill
1298 // location.
1299 if (llvm::any_of(Range&: Values,
1300 P: [](Value *V) { return !V || isa<UndefValue>(Val: V); })) {
1301 handleKillDebugValue(Var: Variable, Expr: Expression, DbgLoc: DVR.getDebugLoc(),
1302 Order: SDNodeOrder);
1303 continue;
1304 }
1305
1306 bool IsVariadic = DVR.hasArgList();
1307 if (!handleDebugValue(Values, Var: Variable, Expr: Expression, DbgLoc: DVR.getDebugLoc(),
1308 Order: SDNodeOrder, IsVariadic)) {
1309 addDanglingDebugInfo(Values, Var: Variable, Expr: Expression, IsVariadic,
1310 DL: DVR.getDebugLoc(), Order: SDNodeOrder);
1311 }
1312 }
1313}
1314
1315void SelectionDAGBuilder::visit(const Instruction &I) {
1316 visitDbgInfo(I);
1317
1318 // Set up outgoing PHI node register values before emitting the terminator.
1319 if (I.isTerminator()) {
1320 HandlePHINodesInSuccessorBlocks(LLVMBB: I.getParent());
1321 }
1322
1323 // Increase the SDNodeOrder if dealing with a non-debug instruction.
1324 if (!isa<DbgInfoIntrinsic>(Val: I))
1325 ++SDNodeOrder;
1326
1327 CurInst = &I;
1328
1329 // Set inserted listener only if required.
1330 bool NodeInserted = false;
1331 std::unique_ptr<SelectionDAG::DAGNodeInsertedListener> InsertedListener;
1332 MDNode *PCSectionsMD = I.getMetadata(KindID: LLVMContext::MD_pcsections);
1333 MDNode *MMRA = I.getMetadata(KindID: LLVMContext::MD_mmra);
1334 if (PCSectionsMD || MMRA) {
1335 InsertedListener = std::make_unique<SelectionDAG::DAGNodeInsertedListener>(
1336 args&: DAG, args: [&](SDNode *) { NodeInserted = true; });
1337 }
1338
1339 visit(Opcode: I.getOpcode(), I);
1340
1341 if (!I.isTerminator() && !HasTailCall &&
1342 !isa<GCStatepointInst>(Val: I)) // statepoints handle their exports internally
1343 CopyToExportRegsIfNeeded(V: &I);
1344
1345 // Handle metadata.
1346 if (PCSectionsMD || MMRA) {
1347 auto It = NodeMap.find(Val: &I);
1348 if (It != NodeMap.end()) {
1349 if (PCSectionsMD)
1350 DAG.addPCSections(Node: It->second.getNode(), MD: PCSectionsMD);
1351 if (MMRA)
1352 DAG.addMMRAMetadata(Node: It->second.getNode(), MMRA);
1353 } else if (NodeInserted) {
1354 // This should not happen; if it does, don't let it go unnoticed so we can
1355 // fix it. Relevant visit*() function is probably missing a setValue().
1356 errs() << "warning: loosing !pcsections and/or !mmra metadata ["
1357 << I.getModule()->getName() << "]\n";
1358 LLVM_DEBUG(I.dump());
1359 assert(false);
1360 }
1361 }
1362
1363 CurInst = nullptr;
1364}
1365
1366void SelectionDAGBuilder::visitPHI(const PHINode &) {
1367 llvm_unreachable("SelectionDAGBuilder shouldn't visit PHI nodes!");
1368}
1369
1370void SelectionDAGBuilder::visit(unsigned Opcode, const User &I) {
1371 // Note: this doesn't use InstVisitor, because it has to work with
1372 // ConstantExpr's in addition to instructions.
1373 switch (Opcode) {
1374 default: llvm_unreachable("Unknown instruction type encountered!");
1375 // Build the switch statement using the Instruction.def file.
1376#define HANDLE_INST(NUM, OPCODE, CLASS) \
1377 case Instruction::OPCODE: visit##OPCODE((const CLASS&)I); break;
1378#include "llvm/IR/Instruction.def"
1379 }
1380}
1381
1382static bool handleDanglingVariadicDebugInfo(SelectionDAG &DAG,
1383 DILocalVariable *Variable,
1384 DebugLoc DL, unsigned Order,
1385 SmallVectorImpl<Value *> &Values,
1386 DIExpression *Expression) {
1387 // For variadic dbg_values we will now insert poison.
1388 // FIXME: We can potentially recover these!
1389 SmallVector<SDDbgOperand, 2> Locs;
1390 for (const Value *V : Values) {
1391 auto *Poison = PoisonValue::get(T: V->getType());
1392 Locs.push_back(Elt: SDDbgOperand::fromConst(Const: Poison));
1393 }
1394 SDDbgValue *SDV = DAG.getDbgValueList(Var: Variable, Expr: Expression, Locs, Dependencies: {},
1395 /*IsIndirect=*/false, DL, O: Order,
1396 /*IsVariadic=*/true);
1397 DAG.AddDbgValue(DB: SDV, /*isParameter=*/false);
1398 return true;
1399}
1400
1401void SelectionDAGBuilder::addDanglingDebugInfo(SmallVectorImpl<Value *> &Values,
1402 DILocalVariable *Var,
1403 DIExpression *Expr,
1404 bool IsVariadic, DebugLoc DL,
1405 unsigned Order) {
1406 if (IsVariadic) {
1407 handleDanglingVariadicDebugInfo(DAG, Variable: Var, DL, Order, Values, Expression: Expr);
1408 return;
1409 }
1410 // TODO: Dangling debug info will eventually either be resolved or produce
1411 // a poison DBG_VALUE. However in the resolution case, a gap may appear
1412 // between the original dbg.value location and its resolved DBG_VALUE,
1413 // which we should ideally fill with an extra poison DBG_VALUE.
1414 assert(Values.size() == 1);
1415 DanglingDebugInfoMap[Values[0]].emplace_back(args&: Var, args&: Expr, args&: DL, args&: Order);
1416}
1417
1418void SelectionDAGBuilder::dropDanglingDebugInfo(const DILocalVariable *Variable,
1419 const DIExpression *Expr) {
1420 auto isMatchingDbgValue = [&](DanglingDebugInfo &DDI) {
1421 DIVariable *DanglingVariable = DDI.getVariable();
1422 DIExpression *DanglingExpr = DDI.getExpression();
1423 if (DanglingVariable == Variable && Expr->fragmentsOverlap(Other: DanglingExpr)) {
1424 LLVM_DEBUG(dbgs() << "Dropping dangling debug info for "
1425 << printDDI(nullptr, DDI) << "\n");
1426 return true;
1427 }
1428 return false;
1429 };
1430
1431 for (auto &DDIMI : DanglingDebugInfoMap) {
1432 DanglingDebugInfoVector &DDIV = DDIMI.second;
1433
1434 // If debug info is to be dropped, run it through final checks to see
1435 // whether it can be salvaged.
1436 for (auto &DDI : DDIV)
1437 if (isMatchingDbgValue(DDI))
1438 salvageUnresolvedDbgValue(V: DDIMI.first, DDI);
1439
1440 erase_if(C&: DDIV, P: isMatchingDbgValue);
1441 }
1442}
1443
1444// resolveDanglingDebugInfo - if we saw an earlier dbg_value referring to V,
1445// generate the debug data structures now that we've seen its definition.
1446void SelectionDAGBuilder::resolveDanglingDebugInfo(const Value *V,
1447 SDValue Val) {
1448 auto DanglingDbgInfoIt = DanglingDebugInfoMap.find(Key: V);
1449 if (DanglingDbgInfoIt == DanglingDebugInfoMap.end())
1450 return;
1451
1452 DanglingDebugInfoVector &DDIV = DanglingDbgInfoIt->second;
1453 for (auto &DDI : DDIV) {
1454 DebugLoc DL = DDI.getDebugLoc();
1455 unsigned ValSDNodeOrder = Val.getNode()->getIROrder();
1456 unsigned DbgSDNodeOrder = DDI.getSDNodeOrder();
1457 DILocalVariable *Variable = DDI.getVariable();
1458 DIExpression *Expr = DDI.getExpression();
1459 assert(Variable->isValidLocationForIntrinsic(DL) &&
1460 "Expected inlined-at fields to agree");
1461 SDDbgValue *SDV;
1462 if (Val.getNode()) {
1463 // FIXME: I doubt that it is correct to resolve a dangling DbgValue as a
1464 // FuncArgumentDbgValue (it would be hoisted to the function entry, and if
1465 // we couldn't resolve it directly when examining the DbgValue intrinsic
1466 // in the first place we should not be more successful here). Unless we
1467 // have some test case that prove this to be correct we should avoid
1468 // calling EmitFuncArgumentDbgValue here.
1469 if (!EmitFuncArgumentDbgValue(V, Variable, Expr, DL,
1470 Kind: FuncArgumentDbgValueKind::Value, N: Val)) {
1471 LLVM_DEBUG(dbgs() << "Resolve dangling debug info for "
1472 << printDDI(V, DDI) << "\n");
1473 LLVM_DEBUG(dbgs() << " By mapping to:\n "; Val.dump());
1474 // Increase the SDNodeOrder for the DbgValue here to make sure it is
1475 // inserted after the definition of Val when emitting the instructions
1476 // after ISel. An alternative could be to teach
1477 // ScheduleDAGSDNodes::EmitSchedule to delay the insertion properly.
1478 LLVM_DEBUG(if (ValSDNodeOrder > DbgSDNodeOrder) dbgs()
1479 << "changing SDNodeOrder from " << DbgSDNodeOrder << " to "
1480 << ValSDNodeOrder << "\n");
1481 SDV = getDbgValue(N: Val, Variable, Expr, dl: DL,
1482 DbgSDNodeOrder: std::max(a: DbgSDNodeOrder, b: ValSDNodeOrder));
1483 DAG.AddDbgValue(DB: SDV, isParameter: false);
1484 } else
1485 LLVM_DEBUG(dbgs() << "Resolved dangling debug info for "
1486 << printDDI(V, DDI)
1487 << " in EmitFuncArgumentDbgValue\n");
1488 } else {
1489 LLVM_DEBUG(dbgs() << "Dropping debug info for " << printDDI(V, DDI)
1490 << "\n");
1491 auto Poison = PoisonValue::get(T: V->getType());
1492 auto SDV =
1493 DAG.getConstantDbgValue(Var: Variable, Expr, C: Poison, DL, O: DbgSDNodeOrder);
1494 DAG.AddDbgValue(DB: SDV, isParameter: false);
1495 }
1496 }
1497 DDIV.clear();
1498}
1499
1500void SelectionDAGBuilder::salvageUnresolvedDbgValue(const Value *V,
1501 DanglingDebugInfo &DDI) {
1502 // TODO: For the variadic implementation, instead of only checking the fail
1503 // state of `handleDebugValue`, we need know specifically which values were
1504 // invalid, so that we attempt to salvage only those values when processing
1505 // a DIArgList.
1506 const Value *OrigV = V;
1507 DILocalVariable *Var = DDI.getVariable();
1508 DIExpression *Expr = DDI.getExpression();
1509 DebugLoc DL = DDI.getDebugLoc();
1510 unsigned SDOrder = DDI.getSDNodeOrder();
1511
1512 // Currently we consider only dbg.value intrinsics -- we tell the salvager
1513 // that DW_OP_stack_value is desired.
1514 bool StackValue = true;
1515
1516 // Can this Value can be encoded without any further work?
1517 if (handleDebugValue(Values: V, Var, Expr, DbgLoc: DL, Order: SDOrder, /*IsVariadic=*/false))
1518 return;
1519
1520 // Attempt to salvage back through as many instructions as possible. Bail if
1521 // a non-instruction is seen, such as a constant expression or global
1522 // variable. FIXME: Further work could recover those too.
1523 while (isa<Instruction>(Val: V)) {
1524 const Instruction &VAsInst = *cast<const Instruction>(Val: V);
1525 // Temporary "0", awaiting real implementation.
1526 SmallVector<uint64_t, 16> Ops;
1527 SmallVector<Value *, 4> AdditionalValues;
1528 V = salvageDebugInfoImpl(I&: const_cast<Instruction &>(VAsInst),
1529 CurrentLocOps: Expr->getNumLocationOperands(), Ops,
1530 AdditionalValues);
1531 // If we cannot salvage any further, and haven't yet found a suitable debug
1532 // expression, bail out.
1533 if (!V)
1534 break;
1535
1536 // TODO: If AdditionalValues isn't empty, then the salvage can only be
1537 // represented with a DBG_VALUE_LIST, so we give up. When we have support
1538 // here for variadic dbg_values, remove that condition.
1539 if (!AdditionalValues.empty())
1540 break;
1541
1542 // New value and expr now represent this debuginfo.
1543 Expr = DIExpression::appendOpsToArg(Expr, Ops, ArgNo: 0, StackValue);
1544
1545 // Some kind of simplification occurred: check whether the operand of the
1546 // salvaged debug expression can be encoded in this DAG.
1547 if (handleDebugValue(Values: V, Var, Expr, DbgLoc: DL, Order: SDOrder, /*IsVariadic=*/false)) {
1548 LLVM_DEBUG(
1549 dbgs() << "Salvaged debug location info for:\n " << *Var << "\n"
1550 << *OrigV << "\nBy stripping back to:\n " << *V << "\n");
1551 return;
1552 }
1553 }
1554
1555 // This was the final opportunity to salvage this debug information, and it
1556 // couldn't be done. Place a poison DBG_VALUE at this location to terminate
1557 // any earlier variable location.
1558 assert(OrigV && "V shouldn't be null");
1559 auto *Poison = PoisonValue::get(T: OrigV->getType());
1560 auto *SDV = DAG.getConstantDbgValue(Var, Expr, C: Poison, DL, O: SDNodeOrder);
1561 DAG.AddDbgValue(DB: SDV, isParameter: false);
1562 LLVM_DEBUG(dbgs() << "Dropping debug value info for:\n "
1563 << printDDI(OrigV, DDI) << "\n");
1564}
1565
1566void SelectionDAGBuilder::handleKillDebugValue(DILocalVariable *Var,
1567 DIExpression *Expr,
1568 DebugLoc DbgLoc,
1569 unsigned Order) {
1570 Value *Poison = PoisonValue::get(T: Type::getInt1Ty(C&: *Context));
1571 DIExpression *NewExpr =
1572 const_cast<DIExpression *>(DIExpression::convertToUndefExpression(Expr));
1573 handleDebugValue(Values: Poison, Var, Expr: NewExpr, DbgLoc, Order,
1574 /*IsVariadic*/ false);
1575}
1576
1577bool SelectionDAGBuilder::handleDebugValue(ArrayRef<const Value *> Values,
1578 DILocalVariable *Var,
1579 DIExpression *Expr, DebugLoc DbgLoc,
1580 unsigned Order, bool IsVariadic) {
1581 if (Values.empty())
1582 return true;
1583
1584 // Filter EntryValue locations out early.
1585 if (visitEntryValueDbgValue(Values, Variable: Var, Expr, DbgLoc))
1586 return true;
1587
1588 SmallVector<SDDbgOperand> LocationOps;
1589 SmallVector<SDNode *> Dependencies;
1590 for (const Value *V : Values) {
1591 // Constant value.
1592 if (isa<ConstantInt>(Val: V) || isa<ConstantFP>(Val: V) || isa<UndefValue>(Val: V) ||
1593 isa<ConstantPointerNull>(Val: V)) {
1594 LocationOps.emplace_back(Args: SDDbgOperand::fromConst(Const: V));
1595 continue;
1596 }
1597
1598 // Look through IntToPtr constants.
1599 if (auto *CE = dyn_cast<ConstantExpr>(Val: V))
1600 if (CE->getOpcode() == Instruction::IntToPtr) {
1601 LocationOps.emplace_back(Args: SDDbgOperand::fromConst(Const: CE->getOperand(i_nocapture: 0)));
1602 continue;
1603 }
1604
1605 // If the Value is a frame index, we can create a FrameIndex debug value
1606 // without relying on the DAG at all.
1607 if (const AllocaInst *AI = dyn_cast<AllocaInst>(Val: V)) {
1608 auto SI = FuncInfo.StaticAllocaMap.find(Val: AI);
1609 if (SI != FuncInfo.StaticAllocaMap.end()) {
1610 LocationOps.emplace_back(Args: SDDbgOperand::fromFrameIdx(FrameIdx: SI->second));
1611 continue;
1612 }
1613 }
1614
1615 // Do not use getValue() in here; we don't want to generate code at
1616 // this point if it hasn't been done yet.
1617 SDValue N = NodeMap[V];
1618 if (!N.getNode() && isa<Argument>(Val: V)) // Check unused arguments map.
1619 N = UnusedArgNodeMap[V];
1620
1621 if (N.getNode()) {
1622 // Only emit func arg dbg value for non-variadic dbg.values for now.
1623 if (!IsVariadic &&
1624 EmitFuncArgumentDbgValue(V, Variable: Var, Expr, DL: DbgLoc,
1625 Kind: FuncArgumentDbgValueKind::Value, N))
1626 return true;
1627 if (auto *FISDN = dyn_cast<FrameIndexSDNode>(Val: N.getNode())) {
1628 // Construct a FrameIndexDbgValue for FrameIndexSDNodes so we can
1629 // describe stack slot locations.
1630 //
1631 // Consider "int x = 0; int *px = &x;". There are two kinds of
1632 // interesting debug values here after optimization:
1633 //
1634 // dbg.value(i32* %px, !"int *px", !DIExpression()), and
1635 // dbg.value(i32* %px, !"int x", !DIExpression(DW_OP_deref))
1636 //
1637 // Both describe the direct values of their associated variables.
1638 Dependencies.push_back(Elt: N.getNode());
1639 LocationOps.emplace_back(Args: SDDbgOperand::fromFrameIdx(FrameIdx: FISDN->getIndex()));
1640 continue;
1641 }
1642 LocationOps.emplace_back(
1643 Args: SDDbgOperand::fromNode(Node: N.getNode(), ResNo: N.getResNo()));
1644 continue;
1645 }
1646
1647 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1648 // Special rules apply for the first dbg.values of parameter variables in a
1649 // function. Identify them by the fact they reference Argument Values, that
1650 // they're parameters, and they are parameters of the current function. We
1651 // need to let them dangle until they get an SDNode.
1652 bool IsParamOfFunc =
1653 isa<Argument>(Val: V) && Var->isParameter() && !DbgLoc.getInlinedAt();
1654 if (IsParamOfFunc)
1655 return false;
1656
1657 // The value is not used in this block yet (or it would have an SDNode).
1658 // We still want the value to appear for the user if possible -- if it has
1659 // an associated VReg, we can refer to that instead.
1660 auto VMI = FuncInfo.ValueMap.find(Val: V);
1661 if (VMI != FuncInfo.ValueMap.end()) {
1662 Register Reg = VMI->second;
1663 // If this is a PHI node, it may be split up into several MI PHI nodes
1664 // (in FunctionLoweringInfo::set).
1665 RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), Reg,
1666 V->getType(), std::nullopt);
1667 if (RFV.occupiesMultipleRegs()) {
1668 // FIXME: We could potentially support variadic dbg_values here.
1669 if (IsVariadic)
1670 return false;
1671 unsigned Offset = 0;
1672 unsigned BitsToDescribe = 0;
1673 if (auto VarSize = Var->getSizeInBits())
1674 BitsToDescribe = *VarSize;
1675 if (auto Fragment = Expr->getFragmentInfo())
1676 BitsToDescribe = Fragment->SizeInBits;
1677 for (const auto &RegAndSize : RFV.getRegsAndSizes()) {
1678 // Bail out if all bits are described already.
1679 if (Offset >= BitsToDescribe)
1680 break;
1681 // TODO: handle scalable vectors.
1682 unsigned RegisterSize = RegAndSize.second;
1683 unsigned FragmentSize = (Offset + RegisterSize > BitsToDescribe)
1684 ? BitsToDescribe - Offset
1685 : RegisterSize;
1686 auto FragmentExpr = DIExpression::createFragmentExpression(
1687 Expr, OffsetInBits: Offset, SizeInBits: FragmentSize);
1688 if (!FragmentExpr)
1689 continue;
1690 SDDbgValue *SDV = DAG.getVRegDbgValue(
1691 Var, Expr: *FragmentExpr, VReg: RegAndSize.first, IsIndirect: false, DL: DbgLoc, O: Order);
1692 DAG.AddDbgValue(DB: SDV, isParameter: false);
1693 Offset += RegisterSize;
1694 }
1695 return true;
1696 }
1697 // We can use simple vreg locations for variadic dbg_values as well.
1698 LocationOps.emplace_back(Args: SDDbgOperand::fromVReg(VReg: Reg));
1699 continue;
1700 }
1701 // We failed to create a SDDbgOperand for V.
1702 return false;
1703 }
1704
1705 // We have created a SDDbgOperand for each Value in Values.
1706 assert(!LocationOps.empty());
1707 SDDbgValue *SDV =
1708 DAG.getDbgValueList(Var, Expr, Locs: LocationOps, Dependencies,
1709 /*IsIndirect=*/false, DL: DbgLoc, O: Order, IsVariadic);
1710 DAG.AddDbgValue(DB: SDV, /*isParameter=*/false);
1711 return true;
1712}
1713
1714void SelectionDAGBuilder::resolveOrClearDbgInfo() {
1715 // Try to fixup any remaining dangling debug info -- and drop it if we can't.
1716 for (auto &Pair : DanglingDebugInfoMap)
1717 for (auto &DDI : Pair.second)
1718 salvageUnresolvedDbgValue(V: const_cast<Value *>(Pair.first), DDI);
1719 clearDanglingDebugInfo();
1720}
1721
1722/// getCopyFromRegs - If there was virtual register allocated for the value V
1723/// emit CopyFromReg of the specified type Ty. Return empty SDValue() otherwise.
1724SDValue SelectionDAGBuilder::getCopyFromRegs(const Value *V, Type *Ty) {
1725 DenseMap<const Value *, Register>::iterator It = FuncInfo.ValueMap.find(Val: V);
1726 SDValue Result;
1727
1728 if (It != FuncInfo.ValueMap.end()) {
1729 Register InReg = It->second;
1730
1731 RegsForValue RFV(*DAG.getContext(), DAG.getTargetLoweringInfo(),
1732 DAG.getDataLayout(), InReg, Ty,
1733 std::nullopt); // This is not an ABI copy.
1734 SDValue Chain = DAG.getEntryNode();
1735 Result = RFV.getCopyFromRegs(DAG, FuncInfo, dl: getCurSDLoc(), Chain, Glue: nullptr,
1736 V);
1737 resolveDanglingDebugInfo(V, Val: Result);
1738 }
1739
1740 return Result;
1741}
1742
1743/// getValue - Return an SDValue for the given Value.
1744SDValue SelectionDAGBuilder::getValue(const Value *V) {
1745 // If we already have an SDValue for this value, use it. It's important
1746 // to do this first, so that we don't create a CopyFromReg if we already
1747 // have a regular SDValue.
1748 SDValue &N = NodeMap[V];
1749 if (N.getNode()) return N;
1750
1751 // If there's a virtual register allocated and initialized for this
1752 // value, use it.
1753 if (SDValue copyFromReg = getCopyFromRegs(V, Ty: V->getType()))
1754 return copyFromReg;
1755
1756 // Otherwise create a new SDValue and remember it.
1757 SDValue Val = getValueImpl(V);
1758 NodeMap[V] = Val;
1759 resolveDanglingDebugInfo(V, Val);
1760 return Val;
1761}
1762
1763/// getNonRegisterValue - Return an SDValue for the given Value, but
1764/// don't look in FuncInfo.ValueMap for a virtual register.
1765SDValue SelectionDAGBuilder::getNonRegisterValue(const Value *V) {
1766 // If we already have an SDValue for this value, use it.
1767 SDValue &N = NodeMap[V];
1768 if (N.getNode()) {
1769 if (isIntOrFPConstant(V: N)) {
1770 // Remove the debug location from the node as the node is about to be used
1771 // in a location which may differ from the original debug location. This
1772 // is relevant to Constant and ConstantFP nodes because they can appear
1773 // as constant expressions inside PHI nodes.
1774 N->setDebugLoc(DebugLoc());
1775 }
1776 return N;
1777 }
1778
1779 // Otherwise create a new SDValue and remember it.
1780 SDValue Val = getValueImpl(V);
1781 NodeMap[V] = Val;
1782 resolveDanglingDebugInfo(V, Val);
1783 return Val;
1784}
1785
1786/// getValueImpl - Helper function for getValue and getNonRegisterValue.
1787/// Create an SDValue for the given value.
1788SDValue SelectionDAGBuilder::getValueImpl(const Value *V) {
1789 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1790
1791 if (const Constant *C = dyn_cast<Constant>(Val: V)) {
1792 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: V->getType(), AllowUnknown: true);
1793
1794 if (const ConstantInt *CI = dyn_cast<ConstantInt>(Val: C))
1795 return DAG.getConstant(Val: *CI, DL: getCurSDLoc(), VT);
1796
1797 if (const GlobalValue *GV = dyn_cast<GlobalValue>(Val: C))
1798 return DAG.getGlobalAddress(GV, DL: getCurSDLoc(), VT);
1799
1800 if (const ConstantPtrAuth *CPA = dyn_cast<ConstantPtrAuth>(Val: C)) {
1801 return DAG.getNode(Opcode: ISD::PtrAuthGlobalAddress, DL: getCurSDLoc(), VT,
1802 N1: getValue(V: CPA->getPointer()), N2: getValue(V: CPA->getKey()),
1803 N3: getValue(V: CPA->getAddrDiscriminator()),
1804 N4: getValue(V: CPA->getDiscriminator()));
1805 }
1806
1807 if (isa<ConstantPointerNull>(Val: C)) {
1808 unsigned AS = V->getType()->getPointerAddressSpace();
1809 return DAG.getConstant(Val: 0, DL: getCurSDLoc(),
1810 VT: TLI.getPointerTy(DL: DAG.getDataLayout(), AS));
1811 }
1812
1813 if (match(V: C, P: m_VScale()))
1814 return DAG.getVScale(DL: getCurSDLoc(), VT, MulImm: APInt(VT.getSizeInBits(), 1));
1815
1816 if (const ConstantFP *CFP = dyn_cast<ConstantFP>(Val: C))
1817 return DAG.getConstantFP(V: *CFP, DL: getCurSDLoc(), VT);
1818
1819 if (isa<UndefValue>(Val: C) && !V->getType()->isAggregateType())
1820 return isa<PoisonValue>(Val: C) ? DAG.getPOISON(VT) : DAG.getUNDEF(VT);
1821
1822 if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(Val: C)) {
1823 visit(Opcode: CE->getOpcode(), I: *CE);
1824 SDValue N1 = NodeMap[V];
1825 assert(N1.getNode() && "visit didn't populate the NodeMap!");
1826 return N1;
1827 }
1828
1829 if (isa<ConstantStruct>(Val: C) || isa<ConstantArray>(Val: C)) {
1830 SmallVector<SDValue, 4> Constants;
1831 for (const Use &U : C->operands()) {
1832 SDNode *Val = getValue(V: U).getNode();
1833 // If the operand is an empty aggregate, there are no values.
1834 if (!Val) continue;
1835 // Add each leaf value from the operand to the Constants list
1836 // to form a flattened list of all the values.
1837 for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
1838 Constants.push_back(Elt: SDValue(Val, i));
1839 }
1840
1841 return DAG.getMergeValues(Ops: Constants, dl: getCurSDLoc());
1842 }
1843
1844 if (const ConstantDataSequential *CDS =
1845 dyn_cast<ConstantDataSequential>(Val: C)) {
1846 SmallVector<SDValue, 4> Ops;
1847 for (uint64_t i = 0, e = CDS->getNumElements(); i != e; ++i) {
1848 SDNode *Val = getValue(V: CDS->getElementAsConstant(i)).getNode();
1849 // Add each leaf value from the operand to the Constants list
1850 // to form a flattened list of all the values.
1851 for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
1852 Ops.push_back(Elt: SDValue(Val, i));
1853 }
1854
1855 if (isa<ArrayType>(Val: CDS->getType()))
1856 return DAG.getMergeValues(Ops, dl: getCurSDLoc());
1857 return DAG.getBuildVector(VT, DL: getCurSDLoc(), Ops);
1858 }
1859
1860 if (C->getType()->isStructTy() || C->getType()->isArrayTy()) {
1861 assert((isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) &&
1862 "Unknown struct or array constant!");
1863
1864 SmallVector<EVT, 4> ValueVTs;
1865 ComputeValueVTs(TLI, DL: DAG.getDataLayout(), Ty: C->getType(), ValueVTs);
1866 unsigned NumElts = ValueVTs.size();
1867 if (NumElts == 0)
1868 return SDValue(); // empty struct
1869 SmallVector<SDValue, 4> Constants(NumElts);
1870 for (unsigned i = 0; i != NumElts; ++i) {
1871 EVT EltVT = ValueVTs[i];
1872 if (isa<UndefValue>(Val: C))
1873 Constants[i] = DAG.getUNDEF(VT: EltVT);
1874 else if (EltVT.isFloatingPoint())
1875 Constants[i] = DAG.getConstantFP(Val: 0, DL: getCurSDLoc(), VT: EltVT);
1876 else
1877 Constants[i] = DAG.getConstant(Val: 0, DL: getCurSDLoc(), VT: EltVT);
1878 }
1879
1880 return DAG.getMergeValues(Ops: Constants, dl: getCurSDLoc());
1881 }
1882
1883 if (const BlockAddress *BA = dyn_cast<BlockAddress>(Val: C))
1884 return DAG.getBlockAddress(BA, VT);
1885
1886 if (const auto *Equiv = dyn_cast<DSOLocalEquivalent>(Val: C))
1887 return getValue(V: Equiv->getGlobalValue());
1888
1889 if (const auto *NC = dyn_cast<NoCFIValue>(Val: C))
1890 return getValue(V: NC->getGlobalValue());
1891
1892 if (VT == MVT::aarch64svcount) {
1893 assert(C->isNullValue() && "Can only zero this target type!");
1894 return DAG.getNode(ISD::BITCAST, getCurSDLoc(), VT,
1895 DAG.getConstant(0, getCurSDLoc(), MVT::nxv16i1));
1896 }
1897
1898 if (VT.isRISCVVectorTuple()) {
1899 assert(C->isNullValue() && "Can only zero this target type!");
1900 return DAG.getNode(
1901 ISD::BITCAST, getCurSDLoc(), VT,
1902 DAG.getNode(
1903 ISD::SPLAT_VECTOR, getCurSDLoc(),
1904 EVT::getVectorVT(*DAG.getContext(), MVT::i8,
1905 VT.getSizeInBits().getKnownMinValue() / 8, true),
1906 DAG.getConstant(Val: 0, DL: getCurSDLoc(), VT: MVT::getIntegerVT(BitWidth: 8))));
1907 }
1908
1909 VectorType *VecTy = cast<VectorType>(Val: V->getType());
1910
1911 // Now that we know the number and type of the elements, get that number of
1912 // elements into the Ops array based on what kind of constant it is.
1913 if (const ConstantVector *CV = dyn_cast<ConstantVector>(Val: C)) {
1914 SmallVector<SDValue, 16> Ops;
1915 unsigned NumElements = cast<FixedVectorType>(Val: VecTy)->getNumElements();
1916 for (unsigned i = 0; i != NumElements; ++i)
1917 Ops.push_back(Elt: getValue(V: CV->getOperand(i_nocapture: i)));
1918
1919 return DAG.getBuildVector(VT, DL: getCurSDLoc(), Ops);
1920 }
1921
1922 if (isa<ConstantAggregateZero>(Val: C)) {
1923 EVT EltVT =
1924 TLI.getValueType(DL: DAG.getDataLayout(), Ty: VecTy->getElementType());
1925
1926 SDValue Op;
1927 if (EltVT.isFloatingPoint())
1928 Op = DAG.getConstantFP(Val: 0, DL: getCurSDLoc(), VT: EltVT);
1929 else
1930 Op = DAG.getConstant(Val: 0, DL: getCurSDLoc(), VT: EltVT);
1931
1932 return DAG.getSplat(VT, DL: getCurSDLoc(), Op);
1933 }
1934
1935 llvm_unreachable("Unknown vector constant");
1936 }
1937
1938 // If this is a static alloca, generate it as the frameindex instead of
1939 // computation.
1940 if (const AllocaInst *AI = dyn_cast<AllocaInst>(Val: V)) {
1941 DenseMap<const AllocaInst*, int>::iterator SI =
1942 FuncInfo.StaticAllocaMap.find(Val: AI);
1943 if (SI != FuncInfo.StaticAllocaMap.end())
1944 return DAG.getFrameIndex(
1945 FI: SI->second, VT: TLI.getValueType(DL: DAG.getDataLayout(), Ty: AI->getType()));
1946 }
1947
1948 // If this is an instruction which fast-isel has deferred, select it now.
1949 if (const Instruction *Inst = dyn_cast<Instruction>(Val: V)) {
1950 Register InReg = FuncInfo.InitializeRegForValue(V: Inst);
1951
1952 RegsForValue RFV(*DAG.getContext(), TLI, DAG.getDataLayout(), InReg,
1953 Inst->getType(), std::nullopt);
1954 SDValue Chain = DAG.getEntryNode();
1955 return RFV.getCopyFromRegs(DAG, FuncInfo, dl: getCurSDLoc(), Chain, Glue: nullptr, V);
1956 }
1957
1958 if (const MetadataAsValue *MD = dyn_cast<MetadataAsValue>(Val: V))
1959 return DAG.getMDNode(MD: cast<MDNode>(Val: MD->getMetadata()));
1960
1961 if (const auto *BB = dyn_cast<BasicBlock>(Val: V))
1962 return DAG.getBasicBlock(MBB: FuncInfo.getMBB(BB));
1963
1964 llvm_unreachable("Can't get register for value!");
1965}
1966
1967void SelectionDAGBuilder::visitCatchPad(const CatchPadInst &I) {
1968 auto Pers = classifyEHPersonality(Pers: FuncInfo.Fn->getPersonalityFn());
1969 bool IsMSVCCXX = Pers == EHPersonality::MSVC_CXX;
1970 bool IsCoreCLR = Pers == EHPersonality::CoreCLR;
1971 bool IsSEH = isAsynchronousEHPersonality(Pers);
1972 MachineBasicBlock *CatchPadMBB = FuncInfo.MBB;
1973 if (IsSEH) {
1974 // For SEH, EHCont Guard needs to know that this catchpad is a target.
1975 CatchPadMBB->setIsEHContTarget(true);
1976 DAG.getMachineFunction().setHasEHContTarget(true);
1977 } else
1978 CatchPadMBB->setIsEHScopeEntry();
1979 // In MSVC C++ and CoreCLR, catchblocks are funclets and need prologues.
1980 if (IsMSVCCXX || IsCoreCLR)
1981 CatchPadMBB->setIsEHFuncletEntry();
1982}
1983
1984void SelectionDAGBuilder::visitCatchRet(const CatchReturnInst &I) {
1985 // Update machine-CFG edge.
1986 MachineBasicBlock *TargetMBB = FuncInfo.getMBB(BB: I.getSuccessor());
1987 FuncInfo.MBB->addSuccessor(Succ: TargetMBB);
1988
1989 auto Pers = classifyEHPersonality(Pers: FuncInfo.Fn->getPersonalityFn());
1990 bool IsSEH = isAsynchronousEHPersonality(Pers);
1991 if (IsSEH) {
1992 // If this is not a fall-through branch or optimizations are switched off,
1993 // emit the branch.
1994 if (TargetMBB != NextBlock(MBB: FuncInfo.MBB) ||
1995 TM.getOptLevel() == CodeGenOptLevel::None)
1996 DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other,
1997 getControlRoot(), DAG.getBasicBlock(MBB: TargetMBB)));
1998 return;
1999 }
2000
2001 // For non-SEH, EHCont Guard needs to know that this catchret is a target.
2002 TargetMBB->setIsEHContTarget(true);
2003 DAG.getMachineFunction().setHasEHContTarget(true);
2004
2005 // Figure out the funclet membership for the catchret's successor.
2006 // This will be used by the FuncletLayout pass to determine how to order the
2007 // BB's.
2008 // A 'catchret' returns to the outer scope's color.
2009 Value *ParentPad = I.getCatchSwitchParentPad();
2010 const BasicBlock *SuccessorColor;
2011 if (isa<ConstantTokenNone>(Val: ParentPad))
2012 SuccessorColor = &FuncInfo.Fn->getEntryBlock();
2013 else
2014 SuccessorColor = cast<Instruction>(Val: ParentPad)->getParent();
2015 assert(SuccessorColor && "No parent funclet for catchret!");
2016 MachineBasicBlock *SuccessorColorMBB = FuncInfo.getMBB(BB: SuccessorColor);
2017 assert(SuccessorColorMBB && "No MBB for SuccessorColor!");
2018
2019 // Create the terminator node.
2020 SDValue Ret = DAG.getNode(ISD::CATCHRET, getCurSDLoc(), MVT::Other,
2021 getControlRoot(), DAG.getBasicBlock(MBB: TargetMBB),
2022 DAG.getBasicBlock(MBB: SuccessorColorMBB));
2023 DAG.setRoot(Ret);
2024}
2025
2026void SelectionDAGBuilder::visitCleanupPad(const CleanupPadInst &CPI) {
2027 // Don't emit any special code for the cleanuppad instruction. It just marks
2028 // the start of an EH scope/funclet.
2029 FuncInfo.MBB->setIsEHScopeEntry();
2030 auto Pers = classifyEHPersonality(Pers: FuncInfo.Fn->getPersonalityFn());
2031 if (Pers != EHPersonality::Wasm_CXX) {
2032 FuncInfo.MBB->setIsEHFuncletEntry();
2033 FuncInfo.MBB->setIsCleanupFuncletEntry();
2034 }
2035}
2036
2037/// When an invoke or a cleanupret unwinds to the next EH pad, there are
2038/// many places it could ultimately go. In the IR, we have a single unwind
2039/// destination, but in the machine CFG, we enumerate all the possible blocks.
2040/// This function skips over imaginary basic blocks that hold catchswitch
2041/// instructions, and finds all the "real" machine
2042/// basic block destinations. As those destinations may not be successors of
2043/// EHPadBB, here we also calculate the edge probability to those destinations.
2044/// The passed-in Prob is the edge probability to EHPadBB.
2045static void findUnwindDestinations(
2046 FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB,
2047 BranchProbability Prob,
2048 SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
2049 &UnwindDests) {
2050 EHPersonality Personality =
2051 classifyEHPersonality(Pers: FuncInfo.Fn->getPersonalityFn());
2052 bool IsMSVCCXX = Personality == EHPersonality::MSVC_CXX;
2053 bool IsCoreCLR = Personality == EHPersonality::CoreCLR;
2054 bool IsWasmCXX = Personality == EHPersonality::Wasm_CXX;
2055 bool IsSEH = isAsynchronousEHPersonality(Pers: Personality);
2056
2057 while (EHPadBB) {
2058 BasicBlock::const_iterator Pad = EHPadBB->getFirstNonPHIIt();
2059 BasicBlock *NewEHPadBB = nullptr;
2060 if (isa<LandingPadInst>(Val: Pad)) {
2061 // Stop on landingpads. They are not funclets.
2062 UnwindDests.emplace_back(Args: FuncInfo.getMBB(BB: EHPadBB), Args&: Prob);
2063 break;
2064 } else if (isa<CleanupPadInst>(Val: Pad)) {
2065 // Stop on cleanup pads. Cleanups are always funclet entries for all known
2066 // personalities except Wasm. And in Wasm this becomes a catch_all(_ref),
2067 // which always catches an exception.
2068 UnwindDests.emplace_back(Args: FuncInfo.getMBB(BB: EHPadBB), Args&: Prob);
2069 UnwindDests.back().first->setIsEHScopeEntry();
2070 // In Wasm, EH scopes are not funclets
2071 if (!IsWasmCXX)
2072 UnwindDests.back().first->setIsEHFuncletEntry();
2073 break;
2074 } else if (const auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Val&: Pad)) {
2075 // Add the catchpad handlers to the possible destinations.
2076 for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
2077 UnwindDests.emplace_back(Args: FuncInfo.getMBB(BB: CatchPadBB), Args&: Prob);
2078 // For MSVC++ and the CLR, catchblocks are funclets and need prologues.
2079 if (IsMSVCCXX || IsCoreCLR)
2080 UnwindDests.back().first->setIsEHFuncletEntry();
2081 if (!IsSEH)
2082 UnwindDests.back().first->setIsEHScopeEntry();
2083 }
2084 NewEHPadBB = CatchSwitch->getUnwindDest();
2085 } else {
2086 continue;
2087 }
2088
2089 BranchProbabilityInfo *BPI = FuncInfo.BPI;
2090 if (BPI && NewEHPadBB)
2091 Prob *= BPI->getEdgeProbability(Src: EHPadBB, Dst: NewEHPadBB);
2092 EHPadBB = NewEHPadBB;
2093 }
2094}
2095
2096void SelectionDAGBuilder::visitCleanupRet(const CleanupReturnInst &I) {
2097 // Update successor info.
2098 SmallVector<std::pair<MachineBasicBlock *, BranchProbability>, 1> UnwindDests;
2099 auto UnwindDest = I.getUnwindDest();
2100 BranchProbabilityInfo *BPI = FuncInfo.BPI;
2101 BranchProbability UnwindDestProb =
2102 (BPI && UnwindDest)
2103 ? BPI->getEdgeProbability(Src: FuncInfo.MBB->getBasicBlock(), Dst: UnwindDest)
2104 : BranchProbability::getZero();
2105 findUnwindDestinations(FuncInfo, EHPadBB: UnwindDest, Prob: UnwindDestProb, UnwindDests);
2106 for (auto &UnwindDest : UnwindDests) {
2107 UnwindDest.first->setIsEHPad();
2108 addSuccessorWithProb(Src: FuncInfo.MBB, Dst: UnwindDest.first, Prob: UnwindDest.second);
2109 }
2110 FuncInfo.MBB->normalizeSuccProbs();
2111
2112 // Create the terminator node.
2113 MachineBasicBlock *CleanupPadMBB =
2114 FuncInfo.getMBB(BB: I.getCleanupPad()->getParent());
2115 SDValue Ret = DAG.getNode(ISD::CLEANUPRET, getCurSDLoc(), MVT::Other,
2116 getControlRoot(), DAG.getBasicBlock(MBB: CleanupPadMBB));
2117 DAG.setRoot(Ret);
2118}
2119
2120void SelectionDAGBuilder::visitCatchSwitch(const CatchSwitchInst &CSI) {
2121 report_fatal_error(reason: "visitCatchSwitch not yet implemented!");
2122}
2123
2124void SelectionDAGBuilder::visitRet(const ReturnInst &I) {
2125 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2126 auto &DL = DAG.getDataLayout();
2127 SDValue Chain = getControlRoot();
2128 SmallVector<ISD::OutputArg, 8> Outs;
2129 SmallVector<SDValue, 8> OutVals;
2130
2131 // Calls to @llvm.experimental.deoptimize don't generate a return value, so
2132 // lower
2133 //
2134 // %val = call <ty> @llvm.experimental.deoptimize()
2135 // ret <ty> %val
2136 //
2137 // differently.
2138 if (I.getParent()->getTerminatingDeoptimizeCall()) {
2139 LowerDeoptimizingReturn();
2140 return;
2141 }
2142
2143 if (!FuncInfo.CanLowerReturn) {
2144 Register DemoteReg = FuncInfo.DemoteRegister;
2145
2146 // Emit a store of the return value through the virtual register.
2147 // Leave Outs empty so that LowerReturn won't try to load return
2148 // registers the usual way.
2149 MVT PtrValueVT = TLI.getPointerTy(DL, AS: DL.getAllocaAddrSpace());
2150 SDValue RetPtr =
2151 DAG.getCopyFromReg(Chain, dl: getCurSDLoc(), Reg: DemoteReg, VT: PtrValueVT);
2152 SDValue RetOp = getValue(V: I.getOperand(i_nocapture: 0));
2153
2154 SmallVector<EVT, 4> ValueVTs, MemVTs;
2155 SmallVector<uint64_t, 4> Offsets;
2156 ComputeValueVTs(TLI, DL, Ty: I.getOperand(i_nocapture: 0)->getType(), ValueVTs, MemVTs: &MemVTs,
2157 FixedOffsets: &Offsets, StartingOffset: 0);
2158 unsigned NumValues = ValueVTs.size();
2159
2160 SmallVector<SDValue, 4> Chains(NumValues);
2161 Align BaseAlign = DL.getPrefTypeAlign(Ty: I.getOperand(i_nocapture: 0)->getType());
2162 for (unsigned i = 0; i != NumValues; ++i) {
2163 // An aggregate return value cannot wrap around the address space, so
2164 // offsets to its parts don't wrap either.
2165 SDValue Ptr = DAG.getObjectPtrOffset(SL: getCurSDLoc(), Ptr: RetPtr,
2166 Offset: TypeSize::getFixed(ExactSize: Offsets[i]));
2167
2168 SDValue Val = RetOp.getValue(R: RetOp.getResNo() + i);
2169 if (MemVTs[i] != ValueVTs[i])
2170 Val = DAG.getPtrExtOrTrunc(Op: Val, DL: getCurSDLoc(), VT: MemVTs[i]);
2171 Chains[i] = DAG.getStore(
2172 Chain, dl: getCurSDLoc(), Val,
2173 // FIXME: better loc info would be nice.
2174 Ptr, PtrInfo: MachinePointerInfo::getUnknownStack(MF&: DAG.getMachineFunction()),
2175 Alignment: commonAlignment(A: BaseAlign, Offset: Offsets[i]));
2176 }
2177
2178 Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(),
2179 MVT::Other, Chains);
2180 } else if (I.getNumOperands() != 0) {
2181 SmallVector<EVT, 4> ValueVTs;
2182 ComputeValueVTs(TLI, DL, Ty: I.getOperand(i_nocapture: 0)->getType(), ValueVTs);
2183 unsigned NumValues = ValueVTs.size();
2184 if (NumValues) {
2185 SDValue RetOp = getValue(V: I.getOperand(i_nocapture: 0));
2186
2187 const Function *F = I.getParent()->getParent();
2188
2189 bool NeedsRegBlock = TLI.functionArgumentNeedsConsecutiveRegisters(
2190 Ty: I.getOperand(i_nocapture: 0)->getType(), CallConv: F->getCallingConv(),
2191 /*IsVarArg*/ isVarArg: false, DL);
2192
2193 ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
2194 if (F->getAttributes().hasRetAttr(Attribute::SExt))
2195 ExtendKind = ISD::SIGN_EXTEND;
2196 else if (F->getAttributes().hasRetAttr(Attribute::ZExt))
2197 ExtendKind = ISD::ZERO_EXTEND;
2198
2199 LLVMContext &Context = F->getContext();
2200 bool RetInReg = F->getAttributes().hasRetAttr(Attribute::InReg);
2201
2202 for (unsigned j = 0; j != NumValues; ++j) {
2203 EVT VT = ValueVTs[j];
2204
2205 if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger())
2206 VT = TLI.getTypeForExtReturn(Context, VT, ExtendKind);
2207
2208 CallingConv::ID CC = F->getCallingConv();
2209
2210 unsigned NumParts = TLI.getNumRegistersForCallingConv(Context, CC, VT);
2211 MVT PartVT = TLI.getRegisterTypeForCallingConv(Context, CC, VT);
2212 SmallVector<SDValue, 4> Parts(NumParts);
2213 getCopyToParts(DAG, DL: getCurSDLoc(),
2214 Val: SDValue(RetOp.getNode(), RetOp.getResNo() + j),
2215 Parts: &Parts[0], NumParts, PartVT, V: &I, CallConv: CC, ExtendKind);
2216
2217 // 'inreg' on function refers to return value
2218 ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
2219 if (RetInReg)
2220 Flags.setInReg();
2221
2222 if (I.getOperand(i_nocapture: 0)->getType()->isPointerTy()) {
2223 Flags.setPointer();
2224 Flags.setPointerAddrSpace(
2225 cast<PointerType>(Val: I.getOperand(i_nocapture: 0)->getType())->getAddressSpace());
2226 }
2227
2228 if (NeedsRegBlock) {
2229 Flags.setInConsecutiveRegs();
2230 if (j == NumValues - 1)
2231 Flags.setInConsecutiveRegsLast();
2232 }
2233
2234 // Propagate extension type if any
2235 if (ExtendKind == ISD::SIGN_EXTEND)
2236 Flags.setSExt();
2237 else if (ExtendKind == ISD::ZERO_EXTEND)
2238 Flags.setZExt();
2239 else if (F->getAttributes().hasRetAttr(Attribute::NoExt))
2240 Flags.setNoExt();
2241
2242 for (unsigned i = 0; i < NumParts; ++i) {
2243 Outs.push_back(Elt: ISD::OutputArg(Flags,
2244 Parts[i].getValueType().getSimpleVT(),
2245 VT, /*isfixed=*/true, 0, 0));
2246 OutVals.push_back(Elt: Parts[i]);
2247 }
2248 }
2249 }
2250 }
2251
2252 // Push in swifterror virtual register as the last element of Outs. This makes
2253 // sure swifterror virtual register will be returned in the swifterror
2254 // physical register.
2255 const Function *F = I.getParent()->getParent();
2256 if (TLI.supportSwiftError() &&
2257 F->getAttributes().hasAttrSomewhere(Attribute::Kind: SwiftError)) {
2258 assert(SwiftError.getFunctionArg() && "Need a swift error argument");
2259 ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
2260 Flags.setSwiftError();
2261 Outs.push_back(Elt: ISD::OutputArg(
2262 Flags, /*vt=*/TLI.getPointerTy(DL), /*argvt=*/EVT(TLI.getPointerTy(DL)),
2263 /*isfixed=*/true, /*origidx=*/1, /*partOffs=*/0));
2264 // Create SDNode for the swifterror virtual register.
2265 OutVals.push_back(
2266 Elt: DAG.getRegister(Reg: SwiftError.getOrCreateVRegUseAt(
2267 &I, FuncInfo.MBB, SwiftError.getFunctionArg()),
2268 VT: EVT(TLI.getPointerTy(DL))));
2269 }
2270
2271 bool isVarArg = DAG.getMachineFunction().getFunction().isVarArg();
2272 CallingConv::ID CallConv =
2273 DAG.getMachineFunction().getFunction().getCallingConv();
2274 Chain = DAG.getTargetLoweringInfo().LowerReturn(
2275 Chain, CallConv, isVarArg, Outs, OutVals, getCurSDLoc(), DAG);
2276
2277 // Verify that the target's LowerReturn behaved as expected.
2278 assert(Chain.getNode() && Chain.getValueType() == MVT::Other &&
2279 "LowerReturn didn't return a valid chain!");
2280
2281 // Update the DAG with the new chain value resulting from return lowering.
2282 DAG.setRoot(Chain);
2283}
2284
2285/// CopyToExportRegsIfNeeded - If the given value has virtual registers
2286/// created for it, emit nodes to copy the value into the virtual
2287/// registers.
2288void SelectionDAGBuilder::CopyToExportRegsIfNeeded(const Value *V) {
2289 // Skip empty types
2290 if (V->getType()->isEmptyTy())
2291 return;
2292
2293 DenseMap<const Value *, Register>::iterator VMI = FuncInfo.ValueMap.find(Val: V);
2294 if (VMI != FuncInfo.ValueMap.end()) {
2295 assert((!V->use_empty() || isa<CallBrInst>(V)) &&
2296 "Unused value assigned virtual registers!");
2297 CopyValueToVirtualRegister(V, Reg: VMI->second);
2298 }
2299}
2300
2301/// ExportFromCurrentBlock - If this condition isn't known to be exported from
2302/// the current basic block, add it to ValueMap now so that we'll get a
2303/// CopyTo/FromReg.
2304void SelectionDAGBuilder::ExportFromCurrentBlock(const Value *V) {
2305 // No need to export constants.
2306 if (!isa<Instruction>(Val: V) && !isa<Argument>(Val: V)) return;
2307
2308 // Already exported?
2309 if (FuncInfo.isExportedInst(V)) return;
2310
2311 Register Reg = FuncInfo.InitializeRegForValue(V);
2312 CopyValueToVirtualRegister(V, Reg);
2313}
2314
2315bool SelectionDAGBuilder::isExportableFromCurrentBlock(const Value *V,
2316 const BasicBlock *FromBB) {
2317 // The operands of the setcc have to be in this block. We don't know
2318 // how to export them from some other block.
2319 if (const Instruction *VI = dyn_cast<Instruction>(Val: V)) {
2320 // Can export from current BB.
2321 if (VI->getParent() == FromBB)
2322 return true;
2323
2324 // Is already exported, noop.
2325 return FuncInfo.isExportedInst(V);
2326 }
2327
2328 // If this is an argument, we can export it if the BB is the entry block or
2329 // if it is already exported.
2330 if (isa<Argument>(Val: V)) {
2331 if (FromBB->isEntryBlock())
2332 return true;
2333
2334 // Otherwise, can only export this if it is already exported.
2335 return FuncInfo.isExportedInst(V);
2336 }
2337
2338 // Otherwise, constants can always be exported.
2339 return true;
2340}
2341
2342/// Return branch probability calculated by BranchProbabilityInfo for IR blocks.
2343BranchProbability
2344SelectionDAGBuilder::getEdgeProbability(const MachineBasicBlock *Src,
2345 const MachineBasicBlock *Dst) const {
2346 BranchProbabilityInfo *BPI = FuncInfo.BPI;
2347 const BasicBlock *SrcBB = Src->getBasicBlock();
2348 const BasicBlock *DstBB = Dst->getBasicBlock();
2349 if (!BPI) {
2350 // If BPI is not available, set the default probability as 1 / N, where N is
2351 // the number of successors.
2352 auto SuccSize = std::max<uint32_t>(a: succ_size(BB: SrcBB), b: 1);
2353 return BranchProbability(1, SuccSize);
2354 }
2355 return BPI->getEdgeProbability(Src: SrcBB, Dst: DstBB);
2356}
2357
2358void SelectionDAGBuilder::addSuccessorWithProb(MachineBasicBlock *Src,
2359 MachineBasicBlock *Dst,
2360 BranchProbability Prob) {
2361 if (!FuncInfo.BPI)
2362 Src->addSuccessorWithoutProb(Succ: Dst);
2363 else {
2364 if (Prob.isUnknown())
2365 Prob = getEdgeProbability(Src, Dst);
2366 Src->addSuccessor(Succ: Dst, Prob);
2367 }
2368}
2369
2370static bool InBlock(const Value *V, const BasicBlock *BB) {
2371 if (const Instruction *I = dyn_cast<Instruction>(Val: V))
2372 return I->getParent() == BB;
2373 return true;
2374}
2375
2376/// EmitBranchForMergedCondition - Helper method for FindMergedConditions.
2377/// This function emits a branch and is used at the leaves of an OR or an
2378/// AND operator tree.
2379void
2380SelectionDAGBuilder::EmitBranchForMergedCondition(const Value *Cond,
2381 MachineBasicBlock *TBB,
2382 MachineBasicBlock *FBB,
2383 MachineBasicBlock *CurBB,
2384 MachineBasicBlock *SwitchBB,
2385 BranchProbability TProb,
2386 BranchProbability FProb,
2387 bool InvertCond) {
2388 const BasicBlock *BB = CurBB->getBasicBlock();
2389
2390 // If the leaf of the tree is a comparison, merge the condition into
2391 // the caseblock.
2392 if (const CmpInst *BOp = dyn_cast<CmpInst>(Val: Cond)) {
2393 // The operands of the cmp have to be in this block. We don't know
2394 // how to export them from some other block. If this is the first block
2395 // of the sequence, no exporting is needed.
2396 if (CurBB == SwitchBB ||
2397 (isExportableFromCurrentBlock(V: BOp->getOperand(i_nocapture: 0), FromBB: BB) &&
2398 isExportableFromCurrentBlock(V: BOp->getOperand(i_nocapture: 1), FromBB: BB))) {
2399 ISD::CondCode Condition;
2400 if (const ICmpInst *IC = dyn_cast<ICmpInst>(Val: Cond)) {
2401 ICmpInst::Predicate Pred =
2402 InvertCond ? IC->getInversePredicate() : IC->getPredicate();
2403 Condition = getICmpCondCode(Pred);
2404 } else {
2405 const FCmpInst *FC = cast<FCmpInst>(Val: Cond);
2406 FCmpInst::Predicate Pred =
2407 InvertCond ? FC->getInversePredicate() : FC->getPredicate();
2408 Condition = getFCmpCondCode(Pred);
2409 if (TM.Options.NoNaNsFPMath)
2410 Condition = getFCmpCodeWithoutNaN(CC: Condition);
2411 }
2412
2413 CaseBlock CB(Condition, BOp->getOperand(i_nocapture: 0), BOp->getOperand(i_nocapture: 1), nullptr,
2414 TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);
2415 SL->SwitchCases.push_back(x: CB);
2416 return;
2417 }
2418 }
2419
2420 // Create a CaseBlock record representing this branch.
2421 ISD::CondCode Opc = InvertCond ? ISD::SETNE : ISD::SETEQ;
2422 CaseBlock CB(Opc, Cond, ConstantInt::getTrue(Context&: *DAG.getContext()),
2423 nullptr, TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);
2424 SL->SwitchCases.push_back(x: CB);
2425}
2426
2427// Collect dependencies on V recursively. This is used for the cost analysis in
2428// `shouldKeepJumpConditionsTogether`.
2429static bool collectInstructionDeps(
2430 SmallMapVector<const Instruction *, bool, 8> *Deps, const Value *V,
2431 SmallMapVector<const Instruction *, bool, 8> *Necessary = nullptr,
2432 unsigned Depth = 0) {
2433 // Return false if we have an incomplete count.
2434 if (Depth >= SelectionDAG::MaxRecursionDepth)
2435 return false;
2436
2437 auto *I = dyn_cast<Instruction>(Val: V);
2438 if (I == nullptr)
2439 return true;
2440
2441 if (Necessary != nullptr) {
2442 // This instruction is necessary for the other side of the condition so
2443 // don't count it.
2444 if (Necessary->contains(Key: I))
2445 return true;
2446 }
2447
2448 // Already added this dep.
2449 if (!Deps->try_emplace(Key: I, Args: false).second)
2450 return true;
2451
2452 for (unsigned OpIdx = 0, E = I->getNumOperands(); OpIdx < E; ++OpIdx)
2453 if (!collectInstructionDeps(Deps, V: I->getOperand(i: OpIdx), Necessary,
2454 Depth: Depth + 1))
2455 return false;
2456 return true;
2457}
2458
2459bool SelectionDAGBuilder::shouldKeepJumpConditionsTogether(
2460 const FunctionLoweringInfo &FuncInfo, const BranchInst &I,
2461 Instruction::BinaryOps Opc, const Value *Lhs, const Value *Rhs,
2462 TargetLoweringBase::CondMergingParams Params) const {
2463 if (I.getNumSuccessors() != 2)
2464 return false;
2465
2466 if (!I.isConditional())
2467 return false;
2468
2469 if (Params.BaseCost < 0)
2470 return false;
2471
2472 // Baseline cost.
2473 InstructionCost CostThresh = Params.BaseCost;
2474
2475 BranchProbabilityInfo *BPI = nullptr;
2476 if (Params.LikelyBias || Params.UnlikelyBias)
2477 BPI = FuncInfo.BPI;
2478 if (BPI != nullptr) {
2479 // See if we are either likely to get an early out or compute both lhs/rhs
2480 // of the condition.
2481 BasicBlock *IfFalse = I.getSuccessor(i: 0);
2482 BasicBlock *IfTrue = I.getSuccessor(i: 1);
2483
2484 std::optional<bool> Likely;
2485 if (BPI->isEdgeHot(Src: I.getParent(), Dst: IfTrue))
2486 Likely = true;
2487 else if (BPI->isEdgeHot(Src: I.getParent(), Dst: IfFalse))
2488 Likely = false;
2489
2490 if (Likely) {
2491 if (Opc == (*Likely ? Instruction::And : Instruction::Or))
2492 // Its likely we will have to compute both lhs and rhs of condition
2493 CostThresh += Params.LikelyBias;
2494 else {
2495 if (Params.UnlikelyBias < 0)
2496 return false;
2497 // Its likely we will get an early out.
2498 CostThresh -= Params.UnlikelyBias;
2499 }
2500 }
2501 }
2502
2503 if (CostThresh <= 0)
2504 return false;
2505
2506 // Collect "all" instructions that lhs condition is dependent on.
2507 // Use map for stable iteration (to avoid non-determanism of iteration of
2508 // SmallPtrSet). The `bool` value is just a dummy.
2509 SmallMapVector<const Instruction *, bool, 8> LhsDeps, RhsDeps;
2510 collectInstructionDeps(Deps: &LhsDeps, V: Lhs);
2511 // Collect "all" instructions that rhs condition is dependent on AND are
2512 // dependencies of lhs. This gives us an estimate on which instructions we
2513 // stand to save by splitting the condition.
2514 if (!collectInstructionDeps(Deps: &RhsDeps, V: Rhs, Necessary: &LhsDeps))
2515 return false;
2516 // Add the compare instruction itself unless its a dependency on the LHS.
2517 if (const auto *RhsI = dyn_cast<Instruction>(Val: Rhs))
2518 if (!LhsDeps.contains(Key: RhsI))
2519 RhsDeps.try_emplace(Key: RhsI, Args: false);
2520
2521 const auto &TLI = DAG.getTargetLoweringInfo();
2522 const auto &TTI =
2523 TLI.getTargetMachine().getTargetTransformInfo(F: *I.getFunction());
2524
2525 InstructionCost CostOfIncluding = 0;
2526 // See if this instruction will need to computed independently of whether RHS
2527 // is.
2528 Value *BrCond = I.getCondition();
2529 auto ShouldCountInsn = [&RhsDeps, &BrCond](const Instruction *Ins) {
2530 for (const auto *U : Ins->users()) {
2531 // If user is independent of RHS calculation we don't need to count it.
2532 if (auto *UIns = dyn_cast<Instruction>(Val: U))
2533 if (UIns != BrCond && !RhsDeps.contains(Key: UIns))
2534 return false;
2535 }
2536 return true;
2537 };
2538
2539 // Prune instructions from RHS Deps that are dependencies of unrelated
2540 // instructions. The value (SelectionDAG::MaxRecursionDepth) is fairly
2541 // arbitrary and just meant to cap the how much time we spend in the pruning
2542 // loop. Its highly unlikely to come into affect.
2543 const unsigned MaxPruneIters = SelectionDAG::MaxRecursionDepth;
2544 // Stop after a certain point. No incorrectness from including too many
2545 // instructions.
2546 for (unsigned PruneIters = 0; PruneIters < MaxPruneIters; ++PruneIters) {
2547 const Instruction *ToDrop = nullptr;
2548 for (const auto &InsPair : RhsDeps) {
2549 if (!ShouldCountInsn(InsPair.first)) {
2550 ToDrop = InsPair.first;
2551 break;
2552 }
2553 }
2554 if (ToDrop == nullptr)
2555 break;
2556 RhsDeps.erase(Key: ToDrop);
2557 }
2558
2559 for (const auto &InsPair : RhsDeps) {
2560 // Finally accumulate latency that we can only attribute to computing the
2561 // RHS condition. Use latency because we are essentially trying to calculate
2562 // the cost of the dependency chain.
2563 // Possible TODO: We could try to estimate ILP and make this more precise.
2564 CostOfIncluding +=
2565 TTI.getInstructionCost(U: InsPair.first, CostKind: TargetTransformInfo::TCK_Latency);
2566
2567 if (CostOfIncluding > CostThresh)
2568 return false;
2569 }
2570 return true;
2571}
2572
2573void SelectionDAGBuilder::FindMergedConditions(const Value *Cond,
2574 MachineBasicBlock *TBB,
2575 MachineBasicBlock *FBB,
2576 MachineBasicBlock *CurBB,
2577 MachineBasicBlock *SwitchBB,
2578 Instruction::BinaryOps Opc,
2579 BranchProbability TProb,
2580 BranchProbability FProb,
2581 bool InvertCond) {
2582 // Skip over not part of the tree and remember to invert op and operands at
2583 // next level.
2584 Value *NotCond;
2585 if (match(V: Cond, P: m_OneUse(SubPattern: m_Not(V: m_Value(V&: NotCond)))) &&
2586 InBlock(V: NotCond, BB: CurBB->getBasicBlock())) {
2587 FindMergedConditions(Cond: NotCond, TBB, FBB, CurBB, SwitchBB, Opc, TProb, FProb,
2588 InvertCond: !InvertCond);
2589 return;
2590 }
2591
2592 const Instruction *BOp = dyn_cast<Instruction>(Val: Cond);
2593 const Value *BOpOp0, *BOpOp1;
2594 // Compute the effective opcode for Cond, taking into account whether it needs
2595 // to be inverted, e.g.
2596 // and (not (or A, B)), C
2597 // gets lowered as
2598 // and (and (not A, not B), C)
2599 Instruction::BinaryOps BOpc = (Instruction::BinaryOps)0;
2600 if (BOp) {
2601 BOpc = match(V: BOp, P: m_LogicalAnd(L: m_Value(V&: BOpOp0), R: m_Value(V&: BOpOp1)))
2602 ? Instruction::And
2603 : (match(V: BOp, P: m_LogicalOr(L: m_Value(V&: BOpOp0), R: m_Value(V&: BOpOp1)))
2604 ? Instruction::Or
2605 : (Instruction::BinaryOps)0);
2606 if (InvertCond) {
2607 if (BOpc == Instruction::And)
2608 BOpc = Instruction::Or;
2609 else if (BOpc == Instruction::Or)
2610 BOpc = Instruction::And;
2611 }
2612 }
2613
2614 // If this node is not part of the or/and tree, emit it as a branch.
2615 // Note that all nodes in the tree should have same opcode.
2616 bool BOpIsInOrAndTree = BOpc && BOpc == Opc && BOp->hasOneUse();
2617 if (!BOpIsInOrAndTree || BOp->getParent() != CurBB->getBasicBlock() ||
2618 !InBlock(V: BOpOp0, BB: CurBB->getBasicBlock()) ||
2619 !InBlock(V: BOpOp1, BB: CurBB->getBasicBlock())) {
2620 EmitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB,
2621 TProb, FProb, InvertCond);
2622 return;
2623 }
2624
2625 // Create TmpBB after CurBB.
2626 MachineFunction::iterator BBI(CurBB);
2627 MachineFunction &MF = DAG.getMachineFunction();
2628 MachineBasicBlock *TmpBB = MF.CreateMachineBasicBlock(BB: CurBB->getBasicBlock());
2629 CurBB->getParent()->insert(MBBI: ++BBI, MBB: TmpBB);
2630
2631 if (Opc == Instruction::Or) {
2632 // Codegen X | Y as:
2633 // BB1:
2634 // jmp_if_X TBB
2635 // jmp TmpBB
2636 // TmpBB:
2637 // jmp_if_Y TBB
2638 // jmp FBB
2639 //
2640
2641 // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
2642 // The requirement is that
2643 // TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB)
2644 // = TrueProb for original BB.
2645 // Assuming the original probabilities are A and B, one choice is to set
2646 // BB1's probabilities to A/2 and A/2+B, and set TmpBB's probabilities to
2647 // A/(1+B) and 2B/(1+B). This choice assumes that
2648 // TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB.
2649 // Another choice is to assume TrueProb for BB1 equals to TrueProb for
2650 // TmpBB, but the math is more complicated.
2651
2652 auto NewTrueProb = TProb / 2;
2653 auto NewFalseProb = TProb / 2 + FProb;
2654 // Emit the LHS condition.
2655 FindMergedConditions(Cond: BOpOp0, TBB, FBB: TmpBB, CurBB, SwitchBB, Opc, TProb: NewTrueProb,
2656 FProb: NewFalseProb, InvertCond);
2657
2658 // Normalize A/2 and B to get A/(1+B) and 2B/(1+B).
2659 SmallVector<BranchProbability, 2> Probs{TProb / 2, FProb};
2660 BranchProbability::normalizeProbabilities(Begin: Probs.begin(), End: Probs.end());
2661 // Emit the RHS condition into TmpBB.
2662 FindMergedConditions(Cond: BOpOp1, TBB, FBB, CurBB: TmpBB, SwitchBB, Opc, TProb: Probs[0],
2663 FProb: Probs[1], InvertCond);
2664 } else {
2665 assert(Opc == Instruction::And && "Unknown merge op!");
2666 // Codegen X & Y as:
2667 // BB1:
2668 // jmp_if_X TmpBB
2669 // jmp FBB
2670 // TmpBB:
2671 // jmp_if_Y TBB
2672 // jmp FBB
2673 //
2674 // This requires creation of TmpBB after CurBB.
2675
2676 // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
2677 // The requirement is that
2678 // FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB)
2679 // = FalseProb for original BB.
2680 // Assuming the original probabilities are A and B, one choice is to set
2681 // BB1's probabilities to A+B/2 and B/2, and set TmpBB's probabilities to
2682 // 2A/(1+A) and B/(1+A). This choice assumes that FalseProb for BB1 ==
2683 // TrueProb for BB1 * FalseProb for TmpBB.
2684
2685 auto NewTrueProb = TProb + FProb / 2;
2686 auto NewFalseProb = FProb / 2;
2687 // Emit the LHS condition.
2688 FindMergedConditions(Cond: BOpOp0, TBB: TmpBB, FBB, CurBB, SwitchBB, Opc, TProb: NewTrueProb,
2689 FProb: NewFalseProb, InvertCond);
2690
2691 // Normalize A and B/2 to get 2A/(1+A) and B/(1+A).
2692 SmallVector<BranchProbability, 2> Probs{TProb, FProb / 2};
2693 BranchProbability::normalizeProbabilities(Begin: Probs.begin(), End: Probs.end());
2694 // Emit the RHS condition into TmpBB.
2695 FindMergedConditions(Cond: BOpOp1, TBB, FBB, CurBB: TmpBB, SwitchBB, Opc, TProb: Probs[0],
2696 FProb: Probs[1], InvertCond);
2697 }
2698}
2699
2700/// If the set of cases should be emitted as a series of branches, return true.
2701/// If we should emit this as a bunch of and/or'd together conditions, return
2702/// false.
2703bool
2704SelectionDAGBuilder::ShouldEmitAsBranches(const std::vector<CaseBlock> &Cases) {
2705 if (Cases.size() != 2) return true;
2706
2707 // If this is two comparisons of the same values or'd or and'd together, they
2708 // will get folded into a single comparison, so don't emit two blocks.
2709 if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&
2710 Cases[0].CmpRHS == Cases[1].CmpRHS) ||
2711 (Cases[0].CmpRHS == Cases[1].CmpLHS &&
2712 Cases[0].CmpLHS == Cases[1].CmpRHS)) {
2713 return false;
2714 }
2715
2716 // Handle: (X != null) | (Y != null) --> (X|Y) != 0
2717 // Handle: (X == null) & (Y == null) --> (X|Y) == 0
2718 if (Cases[0].CmpRHS == Cases[1].CmpRHS &&
2719 Cases[0].CC == Cases[1].CC &&
2720 isa<Constant>(Val: Cases[0].CmpRHS) &&
2721 cast<Constant>(Val: Cases[0].CmpRHS)->isNullValue()) {
2722 if (Cases[0].CC == ISD::SETEQ && Cases[0].TrueBB == Cases[1].ThisBB)
2723 return false;
2724 if (Cases[0].CC == ISD::SETNE && Cases[0].FalseBB == Cases[1].ThisBB)
2725 return false;
2726 }
2727
2728 return true;
2729}
2730
2731void SelectionDAGBuilder::visitBr(const BranchInst &I) {
2732 MachineBasicBlock *BrMBB = FuncInfo.MBB;
2733
2734 // Update machine-CFG edges.
2735 MachineBasicBlock *Succ0MBB = FuncInfo.getMBB(BB: I.getSuccessor(i: 0));
2736
2737 if (I.isUnconditional()) {
2738 // Update machine-CFG edges.
2739 BrMBB->addSuccessor(Succ: Succ0MBB);
2740
2741 // If this is not a fall-through branch or optimizations are switched off,
2742 // emit the branch.
2743 if (Succ0MBB != NextBlock(MBB: BrMBB) ||
2744 TM.getOptLevel() == CodeGenOptLevel::None) {
2745 auto Br = DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other,
2746 getControlRoot(), DAG.getBasicBlock(Succ0MBB));
2747 setValue(V: &I, NewN: Br);
2748 DAG.setRoot(Br);
2749 }
2750
2751 return;
2752 }
2753
2754 // If this condition is one of the special cases we handle, do special stuff
2755 // now.
2756 const Value *CondVal = I.getCondition();
2757 MachineBasicBlock *Succ1MBB = FuncInfo.getMBB(BB: I.getSuccessor(i: 1));
2758
2759 // If this is a series of conditions that are or'd or and'd together, emit
2760 // this as a sequence of branches instead of setcc's with and/or operations.
2761 // As long as jumps are not expensive (exceptions for multi-use logic ops,
2762 // unpredictable branches, and vector extracts because those jumps are likely
2763 // expensive for any target), this should improve performance.
2764 // For example, instead of something like:
2765 // cmp A, B
2766 // C = seteq
2767 // cmp D, E
2768 // F = setle
2769 // or C, F
2770 // jnz foo
2771 // Emit:
2772 // cmp A, B
2773 // je foo
2774 // cmp D, E
2775 // jle foo
2776 bool IsUnpredictable = I.hasMetadata(KindID: LLVMContext::MD_unpredictable);
2777 const Instruction *BOp = dyn_cast<Instruction>(Val: CondVal);
2778 if (!DAG.getTargetLoweringInfo().isJumpExpensive() && BOp &&
2779 BOp->hasOneUse() && !IsUnpredictable) {
2780 Value *Vec;
2781 const Value *BOp0, *BOp1;
2782 Instruction::BinaryOps Opcode = (Instruction::BinaryOps)0;
2783 if (match(V: BOp, P: m_LogicalAnd(L: m_Value(V&: BOp0), R: m_Value(V&: BOp1))))
2784 Opcode = Instruction::And;
2785 else if (match(V: BOp, P: m_LogicalOr(L: m_Value(V&: BOp0), R: m_Value(V&: BOp1))))
2786 Opcode = Instruction::Or;
2787
2788 if (Opcode &&
2789 !(match(V: BOp0, P: m_ExtractElt(Val: m_Value(V&: Vec), Idx: m_Value())) &&
2790 match(V: BOp1, P: m_ExtractElt(Val: m_Specific(V: Vec), Idx: m_Value()))) &&
2791 !shouldKeepJumpConditionsTogether(
2792 FuncInfo, I, Opc: Opcode, Lhs: BOp0, Rhs: BOp1,
2793 Params: DAG.getTargetLoweringInfo().getJumpConditionMergingParams(
2794 Opcode, BOp0, BOp1))) {
2795 FindMergedConditions(Cond: BOp, TBB: Succ0MBB, FBB: Succ1MBB, CurBB: BrMBB, SwitchBB: BrMBB, Opc: Opcode,
2796 TProb: getEdgeProbability(Src: BrMBB, Dst: Succ0MBB),
2797 FProb: getEdgeProbability(Src: BrMBB, Dst: Succ1MBB),
2798 /*InvertCond=*/false);
2799 // If the compares in later blocks need to use values not currently
2800 // exported from this block, export them now. This block should always
2801 // be the first entry.
2802 assert(SL->SwitchCases[0].ThisBB == BrMBB && "Unexpected lowering!");
2803
2804 // Allow some cases to be rejected.
2805 if (ShouldEmitAsBranches(Cases: SL->SwitchCases)) {
2806 for (unsigned i = 1, e = SL->SwitchCases.size(); i != e; ++i) {
2807 ExportFromCurrentBlock(V: SL->SwitchCases[i].CmpLHS);
2808 ExportFromCurrentBlock(V: SL->SwitchCases[i].CmpRHS);
2809 }
2810
2811 // Emit the branch for this block.
2812 visitSwitchCase(CB&: SL->SwitchCases[0], SwitchBB: BrMBB);
2813 SL->SwitchCases.erase(position: SL->SwitchCases.begin());
2814 return;
2815 }
2816
2817 // Okay, we decided not to do this, remove any inserted MBB's and clear
2818 // SwitchCases.
2819 for (unsigned i = 1, e = SL->SwitchCases.size(); i != e; ++i)
2820 FuncInfo.MF->erase(MBBI: SL->SwitchCases[i].ThisBB);
2821
2822 SL->SwitchCases.clear();
2823 }
2824 }
2825
2826 // Create a CaseBlock record representing this branch.
2827 CaseBlock CB(ISD::SETEQ, CondVal, ConstantInt::getTrue(Context&: *DAG.getContext()),
2828 nullptr, Succ0MBB, Succ1MBB, BrMBB, getCurSDLoc(),
2829 BranchProbability::getUnknown(), BranchProbability::getUnknown(),
2830 IsUnpredictable);
2831
2832 // Use visitSwitchCase to actually insert the fast branch sequence for this
2833 // cond branch.
2834 visitSwitchCase(CB, SwitchBB: BrMBB);
2835}
2836
2837/// visitSwitchCase - Emits the necessary code to represent a single node in
2838/// the binary search tree resulting from lowering a switch instruction.
2839void SelectionDAGBuilder::visitSwitchCase(CaseBlock &CB,
2840 MachineBasicBlock *SwitchBB) {
2841 SDValue Cond;
2842 SDValue CondLHS = getValue(V: CB.CmpLHS);
2843 SDLoc dl = CB.DL;
2844
2845 if (CB.CC == ISD::SETTRUE) {
2846 // Branch or fall through to TrueBB.
2847 addSuccessorWithProb(Src: SwitchBB, Dst: CB.TrueBB, Prob: CB.TrueProb);
2848 SwitchBB->normalizeSuccProbs();
2849 if (CB.TrueBB != NextBlock(MBB: SwitchBB)) {
2850 DAG.setRoot(DAG.getNode(ISD::BR, dl, MVT::Other, getControlRoot(),
2851 DAG.getBasicBlock(MBB: CB.TrueBB)));
2852 }
2853 return;
2854 }
2855
2856 auto &TLI = DAG.getTargetLoweringInfo();
2857 EVT MemVT = TLI.getMemValueType(DL: DAG.getDataLayout(), Ty: CB.CmpLHS->getType());
2858
2859 // Build the setcc now.
2860 if (!CB.CmpMHS) {
2861 // Fold "(X == true)" to X and "(X == false)" to !X to
2862 // handle common cases produced by branch lowering.
2863 if (CB.CmpRHS == ConstantInt::getTrue(Context&: *DAG.getContext()) &&
2864 CB.CC == ISD::SETEQ)
2865 Cond = CondLHS;
2866 else if (CB.CmpRHS == ConstantInt::getFalse(Context&: *DAG.getContext()) &&
2867 CB.CC == ISD::SETEQ) {
2868 SDValue True = DAG.getConstant(Val: 1, DL: dl, VT: CondLHS.getValueType());
2869 Cond = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: CondLHS.getValueType(), N1: CondLHS, N2: True);
2870 } else {
2871 SDValue CondRHS = getValue(V: CB.CmpRHS);
2872
2873 // If a pointer's DAG type is larger than its memory type then the DAG
2874 // values are zero-extended. This breaks signed comparisons so truncate
2875 // back to the underlying type before doing the compare.
2876 if (CondLHS.getValueType() != MemVT) {
2877 CondLHS = DAG.getPtrExtOrTrunc(Op: CondLHS, DL: getCurSDLoc(), VT: MemVT);
2878 CondRHS = DAG.getPtrExtOrTrunc(Op: CondRHS, DL: getCurSDLoc(), VT: MemVT);
2879 }
2880 Cond = DAG.getSetCC(DL: dl, MVT::VT: i1, LHS: CondLHS, RHS: CondRHS, Cond: CB.CC);
2881 }
2882 } else {
2883 assert(CB.CC == ISD::SETLE && "Can handle only LE ranges now");
2884
2885 const APInt& Low = cast<ConstantInt>(Val: CB.CmpLHS)->getValue();
2886 const APInt& High = cast<ConstantInt>(Val: CB.CmpRHS)->getValue();
2887
2888 SDValue CmpOp = getValue(V: CB.CmpMHS);
2889 EVT VT = CmpOp.getValueType();
2890
2891 if (cast<ConstantInt>(Val: CB.CmpLHS)->isMinValue(IsSigned: true)) {
2892 Cond = DAG.getSetCC(DL: dl, MVT::VT: i1, LHS: CmpOp, RHS: DAG.getConstant(Val: High, DL: dl, VT),
2893 Cond: ISD::SETLE);
2894 } else {
2895 SDValue SUB = DAG.getNode(Opcode: ISD::SUB, DL: dl,
2896 VT, N1: CmpOp, N2: DAG.getConstant(Val: Low, DL: dl, VT));
2897 Cond = DAG.getSetCC(DL: dl, MVT::VT: i1, LHS: SUB,
2898 RHS: DAG.getConstant(Val: High-Low, DL: dl, VT), Cond: ISD::SETULE);
2899 }
2900 }
2901
2902 // Update successor info
2903 addSuccessorWithProb(Src: SwitchBB, Dst: CB.TrueBB, Prob: CB.TrueProb);
2904 // TrueBB and FalseBB are always different unless the incoming IR is
2905 // degenerate. This only happens when running llc on weird IR.
2906 if (CB.TrueBB != CB.FalseBB)
2907 addSuccessorWithProb(Src: SwitchBB, Dst: CB.FalseBB, Prob: CB.FalseProb);
2908 SwitchBB->normalizeSuccProbs();
2909
2910 // If the lhs block is the next block, invert the condition so that we can
2911 // fall through to the lhs instead of the rhs block.
2912 if (CB.TrueBB == NextBlock(MBB: SwitchBB)) {
2913 std::swap(a&: CB.TrueBB, b&: CB.FalseBB);
2914 SDValue True = DAG.getConstant(Val: 1, DL: dl, VT: Cond.getValueType());
2915 Cond = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: Cond.getValueType(), N1: Cond, N2: True);
2916 }
2917
2918 SDNodeFlags Flags;
2919 Flags.setUnpredictable(CB.IsUnpredictable);
2920 SDValue BrCond = DAG.getNode(ISD::BRCOND, dl, MVT::Other, getControlRoot(),
2921 Cond, DAG.getBasicBlock(MBB: CB.TrueBB), Flags);
2922
2923 setValue(V: CurInst, NewN: BrCond);
2924
2925 // Insert the false branch. Do this even if it's a fall through branch,
2926 // this makes it easier to do DAG optimizations which require inverting
2927 // the branch condition.
2928 BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
2929 DAG.getBasicBlock(MBB: CB.FalseBB));
2930
2931 DAG.setRoot(BrCond);
2932}
2933
2934/// visitJumpTable - Emit JumpTable node in the current MBB
2935void SelectionDAGBuilder::visitJumpTable(SwitchCG::JumpTable &JT) {
2936 // Emit the code for the jump table
2937 assert(JT.SL && "Should set SDLoc for SelectionDAG!");
2938 assert(JT.Reg && "Should lower JT Header first!");
2939 EVT PTy = DAG.getTargetLoweringInfo().getJumpTableRegTy(DL: DAG.getDataLayout());
2940 SDValue Index = DAG.getCopyFromReg(Chain: getControlRoot(), dl: *JT.SL, Reg: JT.Reg, VT: PTy);
2941 SDValue Table = DAG.getJumpTable(JTI: JT.JTI, VT: PTy);
2942 SDValue BrJumpTable = DAG.getNode(ISD::BR_JT, *JT.SL, MVT::Other,
2943 Index.getValue(R: 1), Table, Index);
2944 DAG.setRoot(BrJumpTable);
2945}
2946
2947/// visitJumpTableHeader - This function emits necessary code to produce index
2948/// in the JumpTable from switch case.
2949void SelectionDAGBuilder::visitJumpTableHeader(SwitchCG::JumpTable &JT,
2950 JumpTableHeader &JTH,
2951 MachineBasicBlock *SwitchBB) {
2952 assert(JT.SL && "Should set SDLoc for SelectionDAG!");
2953 const SDLoc &dl = *JT.SL;
2954
2955 // Subtract the lowest switch case value from the value being switched on.
2956 SDValue SwitchOp = getValue(V: JTH.SValue);
2957 EVT VT = SwitchOp.getValueType();
2958 SDValue Sub = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: SwitchOp,
2959 N2: DAG.getConstant(Val: JTH.First, DL: dl, VT));
2960
2961 // The SDNode we just created, which holds the value being switched on minus
2962 // the smallest case value, needs to be copied to a virtual register so it
2963 // can be used as an index into the jump table in a subsequent basic block.
2964 // This value may be smaller or larger than the target's pointer type, and
2965 // therefore require extension or truncating.
2966 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2967 SwitchOp =
2968 DAG.getZExtOrTrunc(Op: Sub, DL: dl, VT: TLI.getJumpTableRegTy(DL: DAG.getDataLayout()));
2969
2970 Register JumpTableReg =
2971 FuncInfo.CreateReg(VT: TLI.getJumpTableRegTy(DL: DAG.getDataLayout()));
2972 SDValue CopyTo =
2973 DAG.getCopyToReg(Chain: getControlRoot(), dl, Reg: JumpTableReg, N: SwitchOp);
2974 JT.Reg = JumpTableReg;
2975
2976 if (!JTH.FallthroughUnreachable) {
2977 // Emit the range check for the jump table, and branch to the default block
2978 // for the switch statement if the value being switched on exceeds the
2979 // largest case in the switch.
2980 SDValue CMP = DAG.getSetCC(
2981 DL: dl, VT: TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(),
2982 VT: Sub.getValueType()),
2983 LHS: Sub, RHS: DAG.getConstant(Val: JTH.Last - JTH.First, DL: dl, VT), Cond: ISD::SETUGT);
2984
2985 SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
2986 MVT::Other, CopyTo, CMP,
2987 DAG.getBasicBlock(MBB: JT.Default));
2988
2989 // Avoid emitting unnecessary branches to the next block.
2990 if (JT.MBB != NextBlock(MBB: SwitchBB))
2991 BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
2992 DAG.getBasicBlock(MBB: JT.MBB));
2993
2994 DAG.setRoot(BrCond);
2995 } else {
2996 // Avoid emitting unnecessary branches to the next block.
2997 if (JT.MBB != NextBlock(MBB: SwitchBB))
2998 DAG.setRoot(DAG.getNode(ISD::BR, dl, MVT::Other, CopyTo,
2999 DAG.getBasicBlock(MBB: JT.MBB)));
3000 else
3001 DAG.setRoot(CopyTo);
3002 }
3003}
3004
3005/// Create a LOAD_STACK_GUARD node, and let it carry the target specific global
3006/// variable if there exists one.
3007static SDValue getLoadStackGuard(SelectionDAG &DAG, const SDLoc &DL,
3008 SDValue &Chain) {
3009 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3010 EVT PtrTy = TLI.getPointerTy(DL: DAG.getDataLayout());
3011 EVT PtrMemTy = TLI.getPointerMemTy(DL: DAG.getDataLayout());
3012 MachineFunction &MF = DAG.getMachineFunction();
3013 Value *Global = TLI.getSDagStackGuard(M: *MF.getFunction().getParent());
3014 MachineSDNode *Node =
3015 DAG.getMachineNode(Opcode: TargetOpcode::LOAD_STACK_GUARD, dl: DL, VT: PtrTy, Op1: Chain);
3016 if (Global) {
3017 MachinePointerInfo MPInfo(Global);
3018 auto Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant |
3019 MachineMemOperand::MODereferenceable;
3020 MachineMemOperand *MemRef = MF.getMachineMemOperand(
3021 PtrInfo: MPInfo, F: Flags, Size: PtrTy.getSizeInBits() / 8, BaseAlignment: DAG.getEVTAlign(MemoryVT: PtrTy));
3022 DAG.setNodeMemRefs(N: Node, NewMemRefs: {MemRef});
3023 }
3024 if (PtrTy != PtrMemTy)
3025 return DAG.getPtrExtOrTrunc(Op: SDValue(Node, 0), DL, VT: PtrMemTy);
3026 return SDValue(Node, 0);
3027}
3028
3029/// Codegen a new tail for a stack protector check ParentMBB which has had its
3030/// tail spliced into a stack protector check success bb.
3031///
3032/// For a high level explanation of how this fits into the stack protector
3033/// generation see the comment on the declaration of class
3034/// StackProtectorDescriptor.
3035void SelectionDAGBuilder::visitSPDescriptorParent(StackProtectorDescriptor &SPD,
3036 MachineBasicBlock *ParentBB) {
3037
3038 // First create the loads to the guard/stack slot for the comparison.
3039 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3040 EVT PtrTy = TLI.getPointerTy(DL: DAG.getDataLayout());
3041 EVT PtrMemTy = TLI.getPointerMemTy(DL: DAG.getDataLayout());
3042
3043 MachineFrameInfo &MFI = ParentBB->getParent()->getFrameInfo();
3044 int FI = MFI.getStackProtectorIndex();
3045
3046 SDValue Guard;
3047 SDLoc dl = getCurSDLoc();
3048 SDValue StackSlotPtr = DAG.getFrameIndex(FI, VT: PtrTy);
3049 const Module &M = *ParentBB->getParent()->getFunction().getParent();
3050 Align Align =
3051 DAG.getDataLayout().getPrefTypeAlign(Ty: PointerType::get(C&: M.getContext(), AddressSpace: 0));
3052
3053 // Generate code to load the content of the guard slot.
3054 SDValue GuardVal = DAG.getLoad(
3055 VT: PtrMemTy, dl, Chain: DAG.getEntryNode(), Ptr: StackSlotPtr,
3056 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI), Alignment: Align,
3057 MMOFlags: MachineMemOperand::MOVolatile);
3058
3059 if (TLI.useStackGuardXorFP())
3060 GuardVal = TLI.emitStackGuardXorFP(DAG, Val: GuardVal, DL: dl);
3061
3062 // Retrieve guard check function, nullptr if instrumentation is inlined.
3063 if (const Function *GuardCheckFn = TLI.getSSPStackGuardCheck(M)) {
3064 // The target provides a guard check function to validate the guard value.
3065 // Generate a call to that function with the content of the guard slot as
3066 // argument.
3067 FunctionType *FnTy = GuardCheckFn->getFunctionType();
3068 assert(FnTy->getNumParams() == 1 && "Invalid function signature");
3069
3070 TargetLowering::ArgListTy Args;
3071 TargetLowering::ArgListEntry Entry;
3072 Entry.Node = GuardVal;
3073 Entry.Ty = FnTy->getParamType(i: 0);
3074 if (GuardCheckFn->hasParamAttribute(0, Attribute::AttrKind::InReg))
3075 Entry.IsInReg = true;
3076 Args.push_back(x: Entry);
3077
3078 TargetLowering::CallLoweringInfo CLI(DAG);
3079 CLI.setDebugLoc(getCurSDLoc())
3080 .setChain(DAG.getEntryNode())
3081 .setCallee(CC: GuardCheckFn->getCallingConv(), ResultType: FnTy->getReturnType(),
3082 Target: getValue(V: GuardCheckFn), ArgsList: std::move(Args));
3083
3084 std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
3085 DAG.setRoot(Result.second);
3086 return;
3087 }
3088
3089 // If useLoadStackGuardNode returns true, generate LOAD_STACK_GUARD.
3090 // Otherwise, emit a volatile load to retrieve the stack guard value.
3091 SDValue Chain = DAG.getEntryNode();
3092 if (TLI.useLoadStackGuardNode(M)) {
3093 Guard = getLoadStackGuard(DAG, DL: dl, Chain);
3094 } else {
3095 const Value *IRGuard = TLI.getSDagStackGuard(M);
3096 SDValue GuardPtr = getValue(V: IRGuard);
3097
3098 Guard = DAG.getLoad(VT: PtrMemTy, dl, Chain, Ptr: GuardPtr,
3099 PtrInfo: MachinePointerInfo(IRGuard, 0), Alignment: Align,
3100 MMOFlags: MachineMemOperand::MOVolatile);
3101 }
3102
3103 // Perform the comparison via a getsetcc.
3104 SDValue Cmp = DAG.getSetCC(DL: dl, VT: TLI.getSetCCResultType(DL: DAG.getDataLayout(),
3105 Context&: *DAG.getContext(),
3106 VT: Guard.getValueType()),
3107 LHS: Guard, RHS: GuardVal, Cond: ISD::SETNE);
3108
3109 // If the guard/stackslot do not equal, branch to failure MBB.
3110 SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
3111 MVT::Other, GuardVal.getOperand(0),
3112 Cmp, DAG.getBasicBlock(SPD.getFailureMBB()));
3113 // Otherwise branch to success MBB.
3114 SDValue Br = DAG.getNode(ISD::BR, dl,
3115 MVT::Other, BrCond,
3116 DAG.getBasicBlock(SPD.getSuccessMBB()));
3117
3118 DAG.setRoot(Br);
3119}
3120
3121/// Codegen the failure basic block for a stack protector check.
3122///
3123/// A failure stack protector machine basic block consists simply of a call to
3124/// __stack_chk_fail().
3125///
3126/// For a high level explanation of how this fits into the stack protector
3127/// generation see the comment on the declaration of class
3128/// StackProtectorDescriptor.
3129void
3130SelectionDAGBuilder::visitSPDescriptorFailure(StackProtectorDescriptor &SPD) {
3131 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3132 TargetLowering::MakeLibCallOptions CallOptions;
3133 CallOptions.setDiscardResult(true);
3134 SDValue Chain = TLI.makeLibCall(DAG, RTLIB::STACKPROTECTOR_CHECK_FAIL,
3135 MVT::isVoid, {}, CallOptions, getCurSDLoc())
3136 .second;
3137
3138 // Emit a trap instruction if we are required to do so.
3139 const TargetOptions &TargetOpts = DAG.getTarget().Options;
3140 if (TargetOpts.TrapUnreachable && !TargetOpts.NoTrapAfterNoreturn)
3141 Chain = DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, Chain);
3142
3143 DAG.setRoot(Chain);
3144}
3145
3146/// visitBitTestHeader - This function emits necessary code to produce value
3147/// suitable for "bit tests"
3148void SelectionDAGBuilder::visitBitTestHeader(BitTestBlock &B,
3149 MachineBasicBlock *SwitchBB) {
3150 SDLoc dl = getCurSDLoc();
3151
3152 // Subtract the minimum value.
3153 SDValue SwitchOp = getValue(V: B.SValue);
3154 EVT VT = SwitchOp.getValueType();
3155 SDValue RangeSub =
3156 DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: SwitchOp, N2: DAG.getConstant(Val: B.First, DL: dl, VT));
3157
3158 // Determine the type of the test operands.
3159 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3160 bool UsePtrType = false;
3161 if (!TLI.isTypeLegal(VT)) {
3162 UsePtrType = true;
3163 } else {
3164 for (const BitTestCase &Case : B.Cases)
3165 if (!isUIntN(N: VT.getSizeInBits(), x: Case.Mask)) {
3166 // Switch table case range are encoded into series of masks.
3167 // Just use pointer type, it's guaranteed to fit.
3168 UsePtrType = true;
3169 break;
3170 }
3171 }
3172 SDValue Sub = RangeSub;
3173 if (UsePtrType) {
3174 VT = TLI.getPointerTy(DL: DAG.getDataLayout());
3175 Sub = DAG.getZExtOrTrunc(Op: Sub, DL: dl, VT);
3176 }
3177
3178 B.RegVT = VT.getSimpleVT();
3179 B.Reg = FuncInfo.CreateReg(VT: B.RegVT);
3180 SDValue CopyTo = DAG.getCopyToReg(Chain: getControlRoot(), dl, Reg: B.Reg, N: Sub);
3181
3182 MachineBasicBlock* MBB = B.Cases[0].ThisBB;
3183
3184 if (!B.FallthroughUnreachable)
3185 addSuccessorWithProb(Src: SwitchBB, Dst: B.Default, Prob: B.DefaultProb);
3186 addSuccessorWithProb(Src: SwitchBB, Dst: MBB, Prob: B.Prob);
3187 SwitchBB->normalizeSuccProbs();
3188
3189 SDValue Root = CopyTo;
3190 if (!B.FallthroughUnreachable) {
3191 // Conditional branch to the default block.
3192 SDValue RangeCmp = DAG.getSetCC(DL: dl,
3193 VT: TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(),
3194 VT: RangeSub.getValueType()),
3195 LHS: RangeSub, RHS: DAG.getConstant(Val: B.Range, DL: dl, VT: RangeSub.getValueType()),
3196 Cond: ISD::SETUGT);
3197
3198 Root = DAG.getNode(ISD::BRCOND, dl, MVT::Other, Root, RangeCmp,
3199 DAG.getBasicBlock(B.Default));
3200 }
3201
3202 // Avoid emitting unnecessary branches to the next block.
3203 if (MBB != NextBlock(SwitchBB))
3204 Root = DAG.getNode(ISD::BR, dl, MVT::Other, Root, DAG.getBasicBlock(MBB));
3205
3206 DAG.setRoot(Root);
3207}
3208
3209/// visitBitTestCase - this function produces one "bit test"
3210void SelectionDAGBuilder::visitBitTestCase(BitTestBlock &BB,
3211 MachineBasicBlock *NextMBB,
3212 BranchProbability BranchProbToNext,
3213 Register Reg, BitTestCase &B,
3214 MachineBasicBlock *SwitchBB) {
3215 SDLoc dl = getCurSDLoc();
3216 MVT VT = BB.RegVT;
3217 SDValue ShiftOp = DAG.getCopyFromReg(Chain: getControlRoot(), dl, Reg, VT);
3218 SDValue Cmp;
3219 unsigned PopCount = llvm::popcount(Value: B.Mask);
3220 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3221 if (PopCount == 1) {
3222 // Testing for a single bit; just compare the shift count with what it
3223 // would need to be to shift a 1 bit in that position.
3224 Cmp = DAG.getSetCC(
3225 DL: dl, VT: TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT),
3226 LHS: ShiftOp, RHS: DAG.getConstant(Val: llvm::countr_zero(Val: B.Mask), DL: dl, VT),
3227 Cond: ISD::SETEQ);
3228 } else if (PopCount == BB.Range) {
3229 // There is only one zero bit in the range, test for it directly.
3230 Cmp = DAG.getSetCC(
3231 DL: dl, VT: TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT),
3232 LHS: ShiftOp, RHS: DAG.getConstant(Val: llvm::countr_one(Value: B.Mask), DL: dl, VT), Cond: ISD::SETNE);
3233 } else {
3234 // Make desired shift
3235 SDValue SwitchVal = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT,
3236 N1: DAG.getConstant(Val: 1, DL: dl, VT), N2: ShiftOp);
3237
3238 // Emit bit tests and jumps
3239 SDValue AndOp = DAG.getNode(Opcode: ISD::AND, DL: dl,
3240 VT, N1: SwitchVal, N2: DAG.getConstant(Val: B.Mask, DL: dl, VT));
3241 Cmp = DAG.getSetCC(
3242 DL: dl, VT: TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT),
3243 LHS: AndOp, RHS: DAG.getConstant(Val: 0, DL: dl, VT), Cond: ISD::SETNE);
3244 }
3245
3246 // The branch probability from SwitchBB to B.TargetBB is B.ExtraProb.
3247 addSuccessorWithProb(Src: SwitchBB, Dst: B.TargetBB, Prob: B.ExtraProb);
3248 // The branch probability from SwitchBB to NextMBB is BranchProbToNext.
3249 addSuccessorWithProb(Src: SwitchBB, Dst: NextMBB, Prob: BranchProbToNext);
3250 // It is not guaranteed that the sum of B.ExtraProb and BranchProbToNext is
3251 // one as they are relative probabilities (and thus work more like weights),
3252 // and hence we need to normalize them to let the sum of them become one.
3253 SwitchBB->normalizeSuccProbs();
3254
3255 SDValue BrAnd = DAG.getNode(ISD::BRCOND, dl,
3256 MVT::Other, getControlRoot(),
3257 Cmp, DAG.getBasicBlock(B.TargetBB));
3258
3259 // Avoid emitting unnecessary branches to the next block.
3260 if (NextMBB != NextBlock(SwitchBB))
3261 BrAnd = DAG.getNode(ISD::BR, dl, MVT::Other, BrAnd,
3262 DAG.getBasicBlock(NextMBB));
3263
3264 DAG.setRoot(BrAnd);
3265}
3266
3267void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) {
3268 MachineBasicBlock *InvokeMBB = FuncInfo.MBB;
3269
3270 // Retrieve successors. Look through artificial IR level blocks like
3271 // catchswitch for successors.
3272 MachineBasicBlock *Return = FuncInfo.getMBB(BB: I.getSuccessor(i: 0));
3273 const BasicBlock *EHPadBB = I.getSuccessor(i: 1);
3274 MachineBasicBlock *EHPadMBB = FuncInfo.getMBB(BB: EHPadBB);
3275
3276 // Deopt and ptrauth bundles are lowered in helper functions, and we don't
3277 // have to do anything here to lower funclet bundles.
3278 if (I.hasOperandBundlesOtherThan(
3279 IDs: {LLVMContext::OB_deopt, LLVMContext::OB_gc_transition,
3280 LLVMContext::OB_gc_live, LLVMContext::OB_funclet,
3281 LLVMContext::OB_cfguardtarget, LLVMContext::OB_ptrauth,
3282 LLVMContext::OB_clang_arc_attachedcall}))
3283 reportFatalUsageError(
3284 reason: "cannot lower invokes with arbitrary operand bundles!");
3285
3286 const Value *Callee(I.getCalledOperand());
3287 const Function *Fn = dyn_cast<Function>(Val: Callee);
3288 if (isa<InlineAsm>(Val: Callee))
3289 visitInlineAsm(Call: I, EHPadBB);
3290 else if (Fn && Fn->isIntrinsic()) {
3291 switch (Fn->getIntrinsicID()) {
3292 default:
3293 llvm_unreachable("Cannot invoke this intrinsic");
3294 case Intrinsic::donothing:
3295 // Ignore invokes to @llvm.donothing: jump directly to the next BB.
3296 case Intrinsic::seh_try_begin:
3297 case Intrinsic::seh_scope_begin:
3298 case Intrinsic::seh_try_end:
3299 case Intrinsic::seh_scope_end:
3300 if (EHPadMBB)
3301 // a block referenced by EH table
3302 // so dtor-funclet not removed by opts
3303 EHPadMBB->setMachineBlockAddressTaken();
3304 break;
3305 case Intrinsic::experimental_patchpoint_void:
3306 case Intrinsic::experimental_patchpoint:
3307 visitPatchpoint(CB: I, EHPadBB);
3308 break;
3309 case Intrinsic::experimental_gc_statepoint:
3310 LowerStatepoint(I: cast<GCStatepointInst>(Val: I), EHPadBB);
3311 break;
3312 // wasm_throw, wasm_rethrow: This is usually done in visitTargetIntrinsic,
3313 // but these intrinsics are special because they can be invoked, so we
3314 // manually lower it to a DAG node here.
3315 case Intrinsic::wasm_throw: {
3316 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3317 std::array<SDValue, 4> Ops = {
3318 getControlRoot(), // inchain for the terminator node
3319 DAG.getTargetConstant(Intrinsic::wasm_throw, getCurSDLoc(),
3320 TLI.getPointerTy(DAG.getDataLayout())),
3321 getValue(I.getArgOperand(0)), // tag
3322 getValue(I.getArgOperand(1)) // thrown value
3323 };
3324 SDVTList VTs = DAG.getVTList(ArrayRef<EVT>({MVT::Other})); // outchain
3325 DAG.setRoot(DAG.getNode(Opcode: ISD::INTRINSIC_VOID, DL: getCurSDLoc(), VTList: VTs, Ops));
3326 break;
3327 }
3328 case Intrinsic::wasm_rethrow: {
3329 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3330 std::array<SDValue, 2> Ops = {
3331 getControlRoot(), // inchain for the terminator node
3332 DAG.getTargetConstant(Intrinsic::wasm_rethrow, getCurSDLoc(),
3333 TLI.getPointerTy(DAG.getDataLayout()))};
3334 SDVTList VTs = DAG.getVTList(ArrayRef<EVT>({MVT::Other})); // outchain
3335 DAG.setRoot(DAG.getNode(Opcode: ISD::INTRINSIC_VOID, DL: getCurSDLoc(), VTList: VTs, Ops));
3336 break;
3337 }
3338 }
3339 } else if (I.hasDeoptState()) {
3340 // Currently we do not lower any intrinsic calls with deopt operand bundles.
3341 // Eventually we will support lowering the @llvm.experimental.deoptimize
3342 // intrinsic, and right now there are no plans to support other intrinsics
3343 // with deopt state.
3344 LowerCallSiteWithDeoptBundle(Call: &I, Callee: getValue(V: Callee), EHPadBB);
3345 } else if (I.countOperandBundlesOfType(ID: LLVMContext::OB_ptrauth)) {
3346 LowerCallSiteWithPtrAuthBundle(CB: cast<CallBase>(Val: I), EHPadBB);
3347 } else {
3348 LowerCallTo(CB: I, Callee: getValue(V: Callee), IsTailCall: false, IsMustTailCall: false, EHPadBB);
3349 }
3350
3351 // If the value of the invoke is used outside of its defining block, make it
3352 // available as a virtual register.
3353 // We already took care of the exported value for the statepoint instruction
3354 // during call to the LowerStatepoint.
3355 if (!isa<GCStatepointInst>(Val: I)) {
3356 CopyToExportRegsIfNeeded(V: &I);
3357 }
3358
3359 SmallVector<std::pair<MachineBasicBlock *, BranchProbability>, 1> UnwindDests;
3360 BranchProbabilityInfo *BPI = FuncInfo.BPI;
3361 BranchProbability EHPadBBProb =
3362 BPI ? BPI->getEdgeProbability(Src: InvokeMBB->getBasicBlock(), Dst: EHPadBB)
3363 : BranchProbability::getZero();
3364 findUnwindDestinations(FuncInfo, EHPadBB, Prob: EHPadBBProb, UnwindDests);
3365
3366 // Update successor info.
3367 addSuccessorWithProb(Src: InvokeMBB, Dst: Return);
3368 for (auto &UnwindDest : UnwindDests) {
3369 UnwindDest.first->setIsEHPad();
3370 addSuccessorWithProb(Src: InvokeMBB, Dst: UnwindDest.first, Prob: UnwindDest.second);
3371 }
3372 InvokeMBB->normalizeSuccProbs();
3373
3374 // Drop into normal successor.
3375 DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, getControlRoot(),
3376 DAG.getBasicBlock(Return)));
3377}
3378
3379void SelectionDAGBuilder::visitCallBr(const CallBrInst &I) {
3380 MachineBasicBlock *CallBrMBB = FuncInfo.MBB;
3381
3382 // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
3383 // have to do anything here to lower funclet bundles.
3384 if (I.hasOperandBundlesOtherThan(
3385 IDs: {LLVMContext::OB_deopt, LLVMContext::OB_funclet}))
3386 reportFatalUsageError(
3387 reason: "cannot lower callbrs with arbitrary operand bundles!");
3388
3389 assert(I.isInlineAsm() && "Only know how to handle inlineasm callbr");
3390 visitInlineAsm(Call: I);
3391 CopyToExportRegsIfNeeded(V: &I);
3392
3393 // Retrieve successors.
3394 SmallPtrSet<BasicBlock *, 8> Dests;
3395 Dests.insert(Ptr: I.getDefaultDest());
3396 MachineBasicBlock *Return = FuncInfo.getMBB(BB: I.getDefaultDest());
3397
3398 // Update successor info.
3399 addSuccessorWithProb(Src: CallBrMBB, Dst: Return, Prob: BranchProbability::getOne());
3400 for (unsigned i = 0, e = I.getNumIndirectDests(); i < e; ++i) {
3401 BasicBlock *Dest = I.getIndirectDest(i);
3402 MachineBasicBlock *Target = FuncInfo.getMBB(BB: Dest);
3403 Target->setIsInlineAsmBrIndirectTarget();
3404 // If we introduce a type of asm goto statement that is permitted to use an
3405 // indirect call instruction to jump to its labels, then we should add a
3406 // call to Target->setMachineBlockAddressTaken() here, to mark the target
3407 // block as requiring a BTI.
3408
3409 Target->setLabelMustBeEmitted();
3410 // Don't add duplicate machine successors.
3411 if (Dests.insert(Ptr: Dest).second)
3412 addSuccessorWithProb(Src: CallBrMBB, Dst: Target, Prob: BranchProbability::getZero());
3413 }
3414 CallBrMBB->normalizeSuccProbs();
3415
3416 // Drop into default successor.
3417 DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),
3418 MVT::Other, getControlRoot(),
3419 DAG.getBasicBlock(Return)));
3420}
3421
3422void SelectionDAGBuilder::visitResume(const ResumeInst &RI) {
3423 llvm_unreachable("SelectionDAGBuilder shouldn't visit resume instructions!");
3424}
3425
3426void SelectionDAGBuilder::visitLandingPad(const LandingPadInst &LP) {
3427 assert(FuncInfo.MBB->isEHPad() &&
3428 "Call to landingpad not in landing pad!");
3429
3430 // If there aren't registers to copy the values into (e.g., during SjLj
3431 // exceptions), then don't bother to create these DAG nodes.
3432 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3433 const Constant *PersonalityFn = FuncInfo.Fn->getPersonalityFn();
3434 if (TLI.getExceptionPointerRegister(PersonalityFn) == 0 &&
3435 TLI.getExceptionSelectorRegister(PersonalityFn) == 0)
3436 return;
3437
3438 // If landingpad's return type is token type, we don't create DAG nodes
3439 // for its exception pointer and selector value. The extraction of exception
3440 // pointer or selector value from token type landingpads is not currently
3441 // supported.
3442 if (LP.getType()->isTokenTy())
3443 return;
3444
3445 SmallVector<EVT, 2> ValueVTs;
3446 SDLoc dl = getCurSDLoc();
3447 ComputeValueVTs(TLI, DL: DAG.getDataLayout(), Ty: LP.getType(), ValueVTs);
3448 assert(ValueVTs.size() == 2 && "Only two-valued landingpads are supported");
3449
3450 // Get the two live-in registers as SDValues. The physregs have already been
3451 // copied into virtual registers.
3452 SDValue Ops[2];
3453 if (FuncInfo.ExceptionPointerVirtReg) {
3454 Ops[0] = DAG.getZExtOrTrunc(
3455 Op: DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl,
3456 Reg: FuncInfo.ExceptionPointerVirtReg,
3457 VT: TLI.getPointerTy(DL: DAG.getDataLayout())),
3458 DL: dl, VT: ValueVTs[0]);
3459 } else {
3460 Ops[0] = DAG.getConstant(Val: 0, DL: dl, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
3461 }
3462 Ops[1] = DAG.getZExtOrTrunc(
3463 Op: DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl,
3464 Reg: FuncInfo.ExceptionSelectorVirtReg,
3465 VT: TLI.getPointerTy(DL: DAG.getDataLayout())),
3466 DL: dl, VT: ValueVTs[1]);
3467
3468 // Merge into one.
3469 SDValue Res = DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: dl,
3470 VTList: DAG.getVTList(VTs: ValueVTs), Ops);
3471 setValue(V: &LP, NewN: Res);
3472}
3473
3474void SelectionDAGBuilder::UpdateSplitBlock(MachineBasicBlock *First,
3475 MachineBasicBlock *Last) {
3476 // Update JTCases.
3477 for (JumpTableBlock &JTB : SL->JTCases)
3478 if (JTB.first.HeaderBB == First)
3479 JTB.first.HeaderBB = Last;
3480
3481 // Update BitTestCases.
3482 for (BitTestBlock &BTB : SL->BitTestCases)
3483 if (BTB.Parent == First)
3484 BTB.Parent = Last;
3485}
3486
3487void SelectionDAGBuilder::visitIndirectBr(const IndirectBrInst &I) {
3488 MachineBasicBlock *IndirectBrMBB = FuncInfo.MBB;
3489
3490 // Update machine-CFG edges with unique successors.
3491 SmallSet<BasicBlock*, 32> Done;
3492 for (unsigned i = 0, e = I.getNumSuccessors(); i != e; ++i) {
3493 BasicBlock *BB = I.getSuccessor(i);
3494 bool Inserted = Done.insert(Ptr: BB).second;
3495 if (!Inserted)
3496 continue;
3497
3498 MachineBasicBlock *Succ = FuncInfo.getMBB(BB);
3499 addSuccessorWithProb(Src: IndirectBrMBB, Dst: Succ);
3500 }
3501 IndirectBrMBB->normalizeSuccProbs();
3502
3503 DAG.setRoot(DAG.getNode(ISD::BRIND, getCurSDLoc(),
3504 MVT::Other, getControlRoot(),
3505 getValue(I.getAddress())));
3506}
3507
3508void SelectionDAGBuilder::visitUnreachable(const UnreachableInst &I) {
3509 if (!I.shouldLowerToTrap(TrapUnreachable: DAG.getTarget().Options.TrapUnreachable,
3510 NoTrapAfterNoreturn: DAG.getTarget().Options.NoTrapAfterNoreturn))
3511 return;
3512
3513 DAG.setRoot(DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, DAG.getRoot()));
3514}
3515
3516void SelectionDAGBuilder::visitUnary(const User &I, unsigned Opcode) {
3517 SDNodeFlags Flags;
3518 if (auto *FPOp = dyn_cast<FPMathOperator>(Val: &I))
3519 Flags.copyFMF(FPMO: *FPOp);
3520
3521 SDValue Op = getValue(V: I.getOperand(i: 0));
3522 SDValue UnNodeValue = DAG.getNode(Opcode, DL: getCurSDLoc(), VT: Op.getValueType(),
3523 Operand: Op, Flags);
3524 setValue(V: &I, NewN: UnNodeValue);
3525}
3526
3527void SelectionDAGBuilder::visitBinary(const User &I, unsigned Opcode) {
3528 SDNodeFlags Flags;
3529 if (auto *OFBinOp = dyn_cast<OverflowingBinaryOperator>(Val: &I)) {
3530 Flags.setNoSignedWrap(OFBinOp->hasNoSignedWrap());
3531 Flags.setNoUnsignedWrap(OFBinOp->hasNoUnsignedWrap());
3532 }
3533 if (auto *ExactOp = dyn_cast<PossiblyExactOperator>(Val: &I))
3534 Flags.setExact(ExactOp->isExact());
3535 if (auto *DisjointOp = dyn_cast<PossiblyDisjointInst>(Val: &I))
3536 Flags.setDisjoint(DisjointOp->isDisjoint());
3537 if (auto *FPOp = dyn_cast<FPMathOperator>(Val: &I))
3538 Flags.copyFMF(FPMO: *FPOp);
3539
3540 SDValue Op1 = getValue(V: I.getOperand(i: 0));
3541 SDValue Op2 = getValue(V: I.getOperand(i: 1));
3542 SDValue BinNodeValue = DAG.getNode(Opcode, DL: getCurSDLoc(), VT: Op1.getValueType(),
3543 N1: Op1, N2: Op2, Flags);
3544 setValue(V: &I, NewN: BinNodeValue);
3545}
3546
3547void SelectionDAGBuilder::visitShift(const User &I, unsigned Opcode) {
3548 SDValue Op1 = getValue(V: I.getOperand(i: 0));
3549 SDValue Op2 = getValue(V: I.getOperand(i: 1));
3550
3551 EVT ShiftTy = DAG.getTargetLoweringInfo().getShiftAmountTy(
3552 LHSTy: Op1.getValueType(), DL: DAG.getDataLayout());
3553
3554 // Coerce the shift amount to the right type if we can. This exposes the
3555 // truncate or zext to optimization early.
3556 if (!I.getType()->isVectorTy() && Op2.getValueType() != ShiftTy) {
3557 assert(ShiftTy.getSizeInBits() >= Log2_32_Ceil(Op1.getValueSizeInBits()) &&
3558 "Unexpected shift type");
3559 Op2 = DAG.getZExtOrTrunc(Op: Op2, DL: getCurSDLoc(), VT: ShiftTy);
3560 }
3561
3562 bool nuw = false;
3563 bool nsw = false;
3564 bool exact = false;
3565
3566 if (Opcode == ISD::SRL || Opcode == ISD::SRA || Opcode == ISD::SHL) {
3567
3568 if (const OverflowingBinaryOperator *OFBinOp =
3569 dyn_cast<const OverflowingBinaryOperator>(Val: &I)) {
3570 nuw = OFBinOp->hasNoUnsignedWrap();
3571 nsw = OFBinOp->hasNoSignedWrap();
3572 }
3573 if (const PossiblyExactOperator *ExactOp =
3574 dyn_cast<const PossiblyExactOperator>(Val: &I))
3575 exact = ExactOp->isExact();
3576 }
3577 SDNodeFlags Flags;
3578 Flags.setExact(exact);
3579 Flags.setNoSignedWrap(nsw);
3580 Flags.setNoUnsignedWrap(nuw);
3581 SDValue Res = DAG.getNode(Opcode, DL: getCurSDLoc(), VT: Op1.getValueType(), N1: Op1, N2: Op2,
3582 Flags);
3583 setValue(V: &I, NewN: Res);
3584}
3585
3586void SelectionDAGBuilder::visitSDiv(const User &I) {
3587 SDValue Op1 = getValue(V: I.getOperand(i: 0));
3588 SDValue Op2 = getValue(V: I.getOperand(i: 1));
3589
3590 SDNodeFlags Flags;
3591 Flags.setExact(isa<PossiblyExactOperator>(Val: &I) &&
3592 cast<PossiblyExactOperator>(Val: &I)->isExact());
3593 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::SDIV, DL: getCurSDLoc(), VT: Op1.getValueType(), N1: Op1,
3594 N2: Op2, Flags));
3595}
3596
3597void SelectionDAGBuilder::visitICmp(const ICmpInst &I) {
3598 ICmpInst::Predicate predicate = I.getPredicate();
3599 SDValue Op1 = getValue(V: I.getOperand(i_nocapture: 0));
3600 SDValue Op2 = getValue(V: I.getOperand(i_nocapture: 1));
3601 ISD::CondCode Opcode = getICmpCondCode(Pred: predicate);
3602
3603 auto &TLI = DAG.getTargetLoweringInfo();
3604 EVT MemVT =
3605 TLI.getMemValueType(DL: DAG.getDataLayout(), Ty: I.getOperand(i_nocapture: 0)->getType());
3606
3607 // If a pointer's DAG type is larger than its memory type then the DAG values
3608 // are zero-extended. This breaks signed comparisons so truncate back to the
3609 // underlying type before doing the compare.
3610 if (Op1.getValueType() != MemVT) {
3611 Op1 = DAG.getPtrExtOrTrunc(Op: Op1, DL: getCurSDLoc(), VT: MemVT);
3612 Op2 = DAG.getPtrExtOrTrunc(Op: Op2, DL: getCurSDLoc(), VT: MemVT);
3613 }
3614
3615 SDNodeFlags Flags;
3616 Flags.setSameSign(I.hasSameSign());
3617 SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);
3618
3619 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DL: DAG.getDataLayout(),
3620 Ty: I.getType());
3621 setValue(V: &I, NewN: DAG.getSetCC(DL: getCurSDLoc(), VT: DestVT, LHS: Op1, RHS: Op2, Cond: Opcode));
3622}
3623
3624void SelectionDAGBuilder::visitFCmp(const FCmpInst &I) {
3625 FCmpInst::Predicate predicate = I.getPredicate();
3626 SDValue Op1 = getValue(V: I.getOperand(i_nocapture: 0));
3627 SDValue Op2 = getValue(V: I.getOperand(i_nocapture: 1));
3628
3629 ISD::CondCode Condition = getFCmpCondCode(Pred: predicate);
3630 auto *FPMO = cast<FPMathOperator>(Val: &I);
3631 if (FPMO->hasNoNaNs() || TM.Options.NoNaNsFPMath)
3632 Condition = getFCmpCodeWithoutNaN(CC: Condition);
3633
3634 SDNodeFlags Flags;
3635 Flags.copyFMF(FPMO: *FPMO);
3636 SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);
3637
3638 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DL: DAG.getDataLayout(),
3639 Ty: I.getType());
3640 setValue(V: &I, NewN: DAG.getSetCC(DL: getCurSDLoc(), VT: DestVT, LHS: Op1, RHS: Op2, Cond: Condition));
3641}
3642
3643// Check if the condition of the select has one use or two users that are both
3644// selects with the same condition.
3645static bool hasOnlySelectUsers(const Value *Cond) {
3646 return llvm::all_of(Range: Cond->users(), P: [](const Value *V) {
3647 return isa<SelectInst>(Val: V);
3648 });
3649}
3650
3651void SelectionDAGBuilder::visitSelect(const User &I) {
3652 SmallVector<EVT, 4> ValueVTs;
3653 ComputeValueVTs(TLI: DAG.getTargetLoweringInfo(), DL: DAG.getDataLayout(), Ty: I.getType(),
3654 ValueVTs);
3655 unsigned NumValues = ValueVTs.size();
3656 if (NumValues == 0) return;
3657
3658 SmallVector<SDValue, 4> Values(NumValues);
3659 SDValue Cond = getValue(V: I.getOperand(i: 0));
3660 SDValue LHSVal = getValue(V: I.getOperand(i: 1));
3661 SDValue RHSVal = getValue(V: I.getOperand(i: 2));
3662 SmallVector<SDValue, 1> BaseOps(1, Cond);
3663 ISD::NodeType OpCode =
3664 Cond.getValueType().isVector() ? ISD::VSELECT : ISD::SELECT;
3665
3666 bool IsUnaryAbs = false;
3667 bool Negate = false;
3668
3669 SDNodeFlags Flags;
3670 if (auto *FPOp = dyn_cast<FPMathOperator>(Val: &I))
3671 Flags.copyFMF(FPMO: *FPOp);
3672
3673 Flags.setUnpredictable(
3674 cast<SelectInst>(Val: I).getMetadata(KindID: LLVMContext::MD_unpredictable));
3675
3676 // Min/max matching is only viable if all output VTs are the same.
3677 if (all_equal(Range&: ValueVTs)) {
3678 EVT VT = ValueVTs[0];
3679 LLVMContext &Ctx = *DAG.getContext();
3680 auto &TLI = DAG.getTargetLoweringInfo();
3681
3682 // We care about the legality of the operation after it has been type
3683 // legalized.
3684 while (TLI.getTypeAction(Context&: Ctx, VT) != TargetLoweringBase::TypeLegal)
3685 VT = TLI.getTypeToTransformTo(Context&: Ctx, VT);
3686
3687 // If the vselect is legal, assume we want to leave this as a vector setcc +
3688 // vselect. Otherwise, if this is going to be scalarized, we want to see if
3689 // min/max is legal on the scalar type.
3690 bool UseScalarMinMax = VT.isVector() &&
3691 !TLI.isOperationLegalOrCustom(Op: ISD::VSELECT, VT);
3692
3693 // ValueTracking's select pattern matching does not account for -0.0,
3694 // so we can't lower to FMINIMUM/FMAXIMUM because those nodes specify that
3695 // -0.0 is less than +0.0.
3696 const Value *LHS, *RHS;
3697 auto SPR = matchSelectPattern(V: &I, LHS, RHS);
3698 ISD::NodeType Opc = ISD::DELETED_NODE;
3699 switch (SPR.Flavor) {
3700 case SPF_UMAX: Opc = ISD::UMAX; break;
3701 case SPF_UMIN: Opc = ISD::UMIN; break;
3702 case SPF_SMAX: Opc = ISD::SMAX; break;
3703 case SPF_SMIN: Opc = ISD::SMIN; break;
3704 case SPF_FMINNUM:
3705 switch (SPR.NaNBehavior) {
3706 case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?");
3707 case SPNB_RETURNS_NAN: break;
3708 case SPNB_RETURNS_OTHER: Opc = ISD::FMINNUM; break;
3709 case SPNB_RETURNS_ANY:
3710 if (TLI.isOperationLegalOrCustom(Op: ISD::FMINNUM, VT) ||
3711 (UseScalarMinMax &&
3712 TLI.isOperationLegalOrCustom(Op: ISD::FMINNUM, VT: VT.getScalarType())))
3713 Opc = ISD::FMINNUM;
3714 break;
3715 }
3716 break;
3717 case SPF_FMAXNUM:
3718 switch (SPR.NaNBehavior) {
3719 case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?");
3720 case SPNB_RETURNS_NAN: break;
3721 case SPNB_RETURNS_OTHER: Opc = ISD::FMAXNUM; break;
3722 case SPNB_RETURNS_ANY:
3723 if (TLI.isOperationLegalOrCustom(Op: ISD::FMAXNUM, VT) ||
3724 (UseScalarMinMax &&
3725 TLI.isOperationLegalOrCustom(Op: ISD::FMAXNUM, VT: VT.getScalarType())))
3726 Opc = ISD::FMAXNUM;
3727 break;
3728 }
3729 break;
3730 case SPF_NABS:
3731 Negate = true;
3732 [[fallthrough]];
3733 case SPF_ABS:
3734 IsUnaryAbs = true;
3735 Opc = ISD::ABS;
3736 break;
3737 default: break;
3738 }
3739
3740 if (!IsUnaryAbs && Opc != ISD::DELETED_NODE &&
3741 (TLI.isOperationLegalOrCustom(Op: Opc, VT) ||
3742 (UseScalarMinMax &&
3743 TLI.isOperationLegalOrCustom(Op: Opc, VT: VT.getScalarType()))) &&
3744 // If the underlying comparison instruction is used by any other
3745 // instruction, the consumed instructions won't be destroyed, so it is
3746 // not profitable to convert to a min/max.
3747 hasOnlySelectUsers(Cond: cast<SelectInst>(Val: I).getCondition())) {
3748 OpCode = Opc;
3749 LHSVal = getValue(V: LHS);
3750 RHSVal = getValue(V: RHS);
3751 BaseOps.clear();
3752 }
3753
3754 if (IsUnaryAbs) {
3755 OpCode = Opc;
3756 LHSVal = getValue(V: LHS);
3757 BaseOps.clear();
3758 }
3759 }
3760
3761 if (IsUnaryAbs) {
3762 for (unsigned i = 0; i != NumValues; ++i) {
3763 SDLoc dl = getCurSDLoc();
3764 EVT VT = LHSVal.getNode()->getValueType(ResNo: LHSVal.getResNo() + i);
3765 Values[i] =
3766 DAG.getNode(Opcode: OpCode, DL: dl, VT, Operand: LHSVal.getValue(R: LHSVal.getResNo() + i));
3767 if (Negate)
3768 Values[i] = DAG.getNegative(Val: Values[i], DL: dl, VT);
3769 }
3770 } else {
3771 for (unsigned i = 0; i != NumValues; ++i) {
3772 SmallVector<SDValue, 3> Ops(BaseOps.begin(), BaseOps.end());
3773 Ops.push_back(Elt: SDValue(LHSVal.getNode(), LHSVal.getResNo() + i));
3774 Ops.push_back(Elt: SDValue(RHSVal.getNode(), RHSVal.getResNo() + i));
3775 Values[i] = DAG.getNode(
3776 Opcode: OpCode, DL: getCurSDLoc(),
3777 VT: LHSVal.getNode()->getValueType(ResNo: LHSVal.getResNo() + i), Ops, Flags);
3778 }
3779 }
3780
3781 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: getCurSDLoc(),
3782 VTList: DAG.getVTList(VTs: ValueVTs), Ops: Values));
3783}
3784
3785void SelectionDAGBuilder::visitTrunc(const User &I) {
3786 // TruncInst cannot be a no-op cast because sizeof(src) > sizeof(dest).
3787 SDValue N = getValue(V: I.getOperand(i: 0));
3788 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DL: DAG.getDataLayout(),
3789 Ty: I.getType());
3790 SDNodeFlags Flags;
3791 if (auto *Trunc = dyn_cast<TruncInst>(Val: &I)) {
3792 Flags.setNoSignedWrap(Trunc->hasNoSignedWrap());
3793 Flags.setNoUnsignedWrap(Trunc->hasNoUnsignedWrap());
3794 }
3795
3796 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::TRUNCATE, DL: getCurSDLoc(), VT: DestVT, Operand: N, Flags));
3797}
3798
3799void SelectionDAGBuilder::visitZExt(const User &I) {
3800 // ZExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
3801 // ZExt also can't be a cast to bool for same reason. So, nothing much to do
3802 SDValue N = getValue(V: I.getOperand(i: 0));
3803 auto &TLI = DAG.getTargetLoweringInfo();
3804 EVT DestVT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
3805
3806 SDNodeFlags Flags;
3807 if (auto *PNI = dyn_cast<PossiblyNonNegInst>(Val: &I))
3808 Flags.setNonNeg(PNI->hasNonNeg());
3809
3810 // Eagerly use nonneg information to canonicalize towards sign_extend if
3811 // that is the target's preference.
3812 // TODO: Let the target do this later.
3813 if (Flags.hasNonNeg() &&
3814 TLI.isSExtCheaperThanZExt(FromTy: N.getValueType(), ToTy: DestVT)) {
3815 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: getCurSDLoc(), VT: DestVT, Operand: N));
3816 return;
3817 }
3818
3819 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: getCurSDLoc(), VT: DestVT, Operand: N, Flags));
3820}
3821
3822void SelectionDAGBuilder::visitSExt(const User &I) {
3823 // SExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
3824 // SExt also can't be a cast to bool for same reason. So, nothing much to do
3825 SDValue N = getValue(V: I.getOperand(i: 0));
3826 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DL: DAG.getDataLayout(),
3827 Ty: I.getType());
3828 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: getCurSDLoc(), VT: DestVT, Operand: N));
3829}
3830
3831void SelectionDAGBuilder::visitFPTrunc(const User &I) {
3832 // FPTrunc is never a no-op cast, no need to check
3833 SDValue N = getValue(V: I.getOperand(i: 0));
3834 SDLoc dl = getCurSDLoc();
3835 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3836 EVT DestVT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
3837 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: DestVT, N1: N,
3838 N2: DAG.getTargetConstant(
3839 Val: 0, DL: dl, VT: TLI.getPointerTy(DL: DAG.getDataLayout()))));
3840}
3841
3842void SelectionDAGBuilder::visitFPExt(const User &I) {
3843 // FPExt is never a no-op cast, no need to check
3844 SDValue N = getValue(V: I.getOperand(i: 0));
3845 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DL: DAG.getDataLayout(),
3846 Ty: I.getType());
3847 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FP_EXTEND, DL: getCurSDLoc(), VT: DestVT, Operand: N));
3848}
3849
3850void SelectionDAGBuilder::visitFPToUI(const User &I) {
3851 // FPToUI is never a no-op cast, no need to check
3852 SDValue N = getValue(V: I.getOperand(i: 0));
3853 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DL: DAG.getDataLayout(),
3854 Ty: I.getType());
3855 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FP_TO_UINT, DL: getCurSDLoc(), VT: DestVT, Operand: N));
3856}
3857
3858void SelectionDAGBuilder::visitFPToSI(const User &I) {
3859 // FPToSI is never a no-op cast, no need to check
3860 SDValue N = getValue(V: I.getOperand(i: 0));
3861 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DL: DAG.getDataLayout(),
3862 Ty: I.getType());
3863 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FP_TO_SINT, DL: getCurSDLoc(), VT: DestVT, Operand: N));
3864}
3865
3866void SelectionDAGBuilder::visitUIToFP(const User &I) {
3867 // UIToFP is never a no-op cast, no need to check
3868 SDValue N = getValue(V: I.getOperand(i: 0));
3869 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DL: DAG.getDataLayout(),
3870 Ty: I.getType());
3871 SDNodeFlags Flags;
3872 if (auto *PNI = dyn_cast<PossiblyNonNegInst>(Val: &I))
3873 Flags.setNonNeg(PNI->hasNonNeg());
3874
3875 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::UINT_TO_FP, DL: getCurSDLoc(), VT: DestVT, Operand: N, Flags));
3876}
3877
3878void SelectionDAGBuilder::visitSIToFP(const User &I) {
3879 // SIToFP is never a no-op cast, no need to check
3880 SDValue N = getValue(V: I.getOperand(i: 0));
3881 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DL: DAG.getDataLayout(),
3882 Ty: I.getType());
3883 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: getCurSDLoc(), VT: DestVT, Operand: N));
3884}
3885
3886void SelectionDAGBuilder::visitPtrToInt(const User &I) {
3887 // What to do depends on the size of the integer and the size of the pointer.
3888 // We can either truncate, zero extend, or no-op, accordingly.
3889 SDValue N = getValue(V: I.getOperand(i: 0));
3890 auto &TLI = DAG.getTargetLoweringInfo();
3891 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DL: DAG.getDataLayout(),
3892 Ty: I.getType());
3893 EVT PtrMemVT =
3894 TLI.getMemValueType(DL: DAG.getDataLayout(), Ty: I.getOperand(i: 0)->getType());
3895 N = DAG.getPtrExtOrTrunc(Op: N, DL: getCurSDLoc(), VT: PtrMemVT);
3896 N = DAG.getZExtOrTrunc(Op: N, DL: getCurSDLoc(), VT: DestVT);
3897 setValue(V: &I, NewN: N);
3898}
3899
3900void SelectionDAGBuilder::visitIntToPtr(const User &I) {
3901 // What to do depends on the size of the integer and the size of the pointer.
3902 // We can either truncate, zero extend, or no-op, accordingly.
3903 SDValue N = getValue(V: I.getOperand(i: 0));
3904 auto &TLI = DAG.getTargetLoweringInfo();
3905 EVT DestVT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
3906 EVT PtrMemVT = TLI.getMemValueType(DL: DAG.getDataLayout(), Ty: I.getType());
3907 N = DAG.getZExtOrTrunc(Op: N, DL: getCurSDLoc(), VT: PtrMemVT);
3908 N = DAG.getPtrExtOrTrunc(Op: N, DL: getCurSDLoc(), VT: DestVT);
3909 setValue(V: &I, NewN: N);
3910}
3911
3912void SelectionDAGBuilder::visitBitCast(const User &I) {
3913 SDValue N = getValue(V: I.getOperand(i: 0));
3914 SDLoc dl = getCurSDLoc();
3915 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DL: DAG.getDataLayout(),
3916 Ty: I.getType());
3917
3918 // BitCast assures us that source and destination are the same size so this is
3919 // either a BITCAST or a no-op.
3920 if (DestVT != N.getValueType())
3921 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::BITCAST, DL: dl,
3922 VT: DestVT, Operand: N)); // convert types.
3923 // Check if the original LLVM IR Operand was a ConstantInt, because getValue()
3924 // might fold any kind of constant expression to an integer constant and that
3925 // is not what we are looking for. Only recognize a bitcast of a genuine
3926 // constant integer as an opaque constant.
3927 else if(ConstantInt *C = dyn_cast<ConstantInt>(Val: I.getOperand(i: 0)))
3928 setValue(V: &I, NewN: DAG.getConstant(Val: C->getValue(), DL: dl, VT: DestVT, /*isTarget=*/false,
3929 /*isOpaque*/true));
3930 else
3931 setValue(V: &I, NewN: N); // noop cast.
3932}
3933
3934void SelectionDAGBuilder::visitAddrSpaceCast(const User &I) {
3935 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3936 const Value *SV = I.getOperand(i: 0);
3937 SDValue N = getValue(V: SV);
3938 EVT DestVT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
3939
3940 unsigned SrcAS = SV->getType()->getPointerAddressSpace();
3941 unsigned DestAS = I.getType()->getPointerAddressSpace();
3942
3943 if (!TM.isNoopAddrSpaceCast(SrcAS, DestAS))
3944 N = DAG.getAddrSpaceCast(dl: getCurSDLoc(), VT: DestVT, Ptr: N, SrcAS, DestAS);
3945
3946 setValue(V: &I, NewN: N);
3947}
3948
3949void SelectionDAGBuilder::visitInsertElement(const User &I) {
3950 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3951 SDValue InVec = getValue(V: I.getOperand(i: 0));
3952 SDValue InVal = getValue(V: I.getOperand(i: 1));
3953 SDValue InIdx = DAG.getZExtOrTrunc(Op: getValue(V: I.getOperand(i: 2)), DL: getCurSDLoc(),
3954 VT: TLI.getVectorIdxTy(DL: DAG.getDataLayout()));
3955 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: getCurSDLoc(),
3956 VT: TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType()),
3957 N1: InVec, N2: InVal, N3: InIdx));
3958}
3959
3960void SelectionDAGBuilder::visitExtractElement(const User &I) {
3961 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3962 SDValue InVec = getValue(V: I.getOperand(i: 0));
3963 SDValue InIdx = DAG.getZExtOrTrunc(Op: getValue(V: I.getOperand(i: 1)), DL: getCurSDLoc(),
3964 VT: TLI.getVectorIdxTy(DL: DAG.getDataLayout()));
3965 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: getCurSDLoc(),
3966 VT: TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType()),
3967 N1: InVec, N2: InIdx));
3968}
3969
3970void SelectionDAGBuilder::visitShuffleVector(const User &I) {
3971 SDValue Src1 = getValue(V: I.getOperand(i: 0));
3972 SDValue Src2 = getValue(V: I.getOperand(i: 1));
3973 ArrayRef<int> Mask;
3974 if (auto *SVI = dyn_cast<ShuffleVectorInst>(Val: &I))
3975 Mask = SVI->getShuffleMask();
3976 else
3977 Mask = cast<ConstantExpr>(Val: I).getShuffleMask();
3978 SDLoc DL = getCurSDLoc();
3979 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3980 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
3981 EVT SrcVT = Src1.getValueType();
3982
3983 if (all_of(Range&: Mask, P: [](int Elem) { return Elem == 0; }) &&
3984 VT.isScalableVector()) {
3985 // Canonical splat form of first element of first input vector.
3986 SDValue FirstElt =
3987 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: SrcVT.getScalarType(), N1: Src1,
3988 N2: DAG.getVectorIdxConstant(Val: 0, DL));
3989 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL, VT, Operand: FirstElt));
3990 return;
3991 }
3992
3993 // For now, we only handle splats for scalable vectors.
3994 // The DAGCombiner will perform a BUILD_VECTOR -> SPLAT_VECTOR transformation
3995 // for targets that support a SPLAT_VECTOR for non-scalable vector types.
3996 assert(!VT.isScalableVector() && "Unsupported scalable vector shuffle");
3997
3998 unsigned SrcNumElts = SrcVT.getVectorNumElements();
3999 unsigned MaskNumElts = Mask.size();
4000
4001 if (SrcNumElts == MaskNumElts) {
4002 setValue(V: &I, NewN: DAG.getVectorShuffle(VT, dl: DL, N1: Src1, N2: Src2, Mask));
4003 return;
4004 }
4005
4006 // Normalize the shuffle vector since mask and vector length don't match.
4007 if (SrcNumElts < MaskNumElts) {
4008 // Mask is longer than the source vectors. We can use concatenate vector to
4009 // make the mask and vectors lengths match.
4010
4011 if (MaskNumElts % SrcNumElts == 0) {
4012 // Mask length is a multiple of the source vector length.
4013 // Check if the shuffle is some kind of concatenation of the input
4014 // vectors.
4015 unsigned NumConcat = MaskNumElts / SrcNumElts;
4016 bool IsConcat = true;
4017 SmallVector<int, 8> ConcatSrcs(NumConcat, -1);
4018 for (unsigned i = 0; i != MaskNumElts; ++i) {
4019 int Idx = Mask[i];
4020 if (Idx < 0)
4021 continue;
4022 // Ensure the indices in each SrcVT sized piece are sequential and that
4023 // the same source is used for the whole piece.
4024 if ((Idx % SrcNumElts != (i % SrcNumElts)) ||
4025 (ConcatSrcs[i / SrcNumElts] >= 0 &&
4026 ConcatSrcs[i / SrcNumElts] != (int)(Idx / SrcNumElts))) {
4027 IsConcat = false;
4028 break;
4029 }
4030 // Remember which source this index came from.
4031 ConcatSrcs[i / SrcNumElts] = Idx / SrcNumElts;
4032 }
4033
4034 // The shuffle is concatenating multiple vectors together. Just emit
4035 // a CONCAT_VECTORS operation.
4036 if (IsConcat) {
4037 SmallVector<SDValue, 8> ConcatOps;
4038 for (auto Src : ConcatSrcs) {
4039 if (Src < 0)
4040 ConcatOps.push_back(Elt: DAG.getUNDEF(VT: SrcVT));
4041 else if (Src == 0)
4042 ConcatOps.push_back(Elt: Src1);
4043 else
4044 ConcatOps.push_back(Elt: Src2);
4045 }
4046 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT, Ops: ConcatOps));
4047 return;
4048 }
4049 }
4050
4051 unsigned PaddedMaskNumElts = alignTo(Value: MaskNumElts, Align: SrcNumElts);
4052 unsigned NumConcat = PaddedMaskNumElts / SrcNumElts;
4053 EVT PaddedVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: VT.getScalarType(),
4054 NumElements: PaddedMaskNumElts);
4055
4056 // Pad both vectors with undefs to make them the same length as the mask.
4057 SDValue UndefVal = DAG.getUNDEF(VT: SrcVT);
4058
4059 SmallVector<SDValue, 8> MOps1(NumConcat, UndefVal);
4060 SmallVector<SDValue, 8> MOps2(NumConcat, UndefVal);
4061 MOps1[0] = Src1;
4062 MOps2[0] = Src2;
4063
4064 Src1 = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: PaddedVT, Ops: MOps1);
4065 Src2 = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: PaddedVT, Ops: MOps2);
4066
4067 // Readjust mask for new input vector length.
4068 SmallVector<int, 8> MappedOps(PaddedMaskNumElts, -1);
4069 for (unsigned i = 0; i != MaskNumElts; ++i) {
4070 int Idx = Mask[i];
4071 if (Idx >= (int)SrcNumElts)
4072 Idx -= SrcNumElts - PaddedMaskNumElts;
4073 MappedOps[i] = Idx;
4074 }
4075
4076 SDValue Result = DAG.getVectorShuffle(VT: PaddedVT, dl: DL, N1: Src1, N2: Src2, Mask: MappedOps);
4077
4078 // If the concatenated vector was padded, extract a subvector with the
4079 // correct number of elements.
4080 if (MaskNumElts != PaddedMaskNumElts)
4081 Result = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT, N1: Result,
4082 N2: DAG.getVectorIdxConstant(Val: 0, DL));
4083
4084 setValue(V: &I, NewN: Result);
4085 return;
4086 }
4087
4088 assert(SrcNumElts > MaskNumElts);
4089
4090 // Analyze the access pattern of the vector to see if we can extract
4091 // two subvectors and do the shuffle.
4092 int StartIdx[2] = {-1, -1}; // StartIdx to extract from
4093 bool CanExtract = true;
4094 for (int Idx : Mask) {
4095 unsigned Input = 0;
4096 if (Idx < 0)
4097 continue;
4098
4099 if (Idx >= (int)SrcNumElts) {
4100 Input = 1;
4101 Idx -= SrcNumElts;
4102 }
4103
4104 // If all the indices come from the same MaskNumElts sized portion of
4105 // the sources we can use extract. Also make sure the extract wouldn't
4106 // extract past the end of the source.
4107 int NewStartIdx = alignDown(Value: Idx, Align: MaskNumElts);
4108 if (NewStartIdx + MaskNumElts > SrcNumElts ||
4109 (StartIdx[Input] >= 0 && StartIdx[Input] != NewStartIdx))
4110 CanExtract = false;
4111 // Make sure we always update StartIdx as we use it to track if all
4112 // elements are undef.
4113 StartIdx[Input] = NewStartIdx;
4114 }
4115
4116 if (StartIdx[0] < 0 && StartIdx[1] < 0) {
4117 setValue(V: &I, NewN: DAG.getUNDEF(VT)); // Vectors are not used.
4118 return;
4119 }
4120 if (CanExtract) {
4121 // Extract appropriate subvector and generate a vector shuffle
4122 for (unsigned Input = 0; Input < 2; ++Input) {
4123 SDValue &Src = Input == 0 ? Src1 : Src2;
4124 if (StartIdx[Input] < 0)
4125 Src = DAG.getUNDEF(VT);
4126 else {
4127 Src = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT, N1: Src,
4128 N2: DAG.getVectorIdxConstant(Val: StartIdx[Input], DL));
4129 }
4130 }
4131
4132 // Calculate new mask.
4133 SmallVector<int, 8> MappedOps(Mask);
4134 for (int &Idx : MappedOps) {
4135 if (Idx >= (int)SrcNumElts)
4136 Idx -= SrcNumElts + StartIdx[1] - MaskNumElts;
4137 else if (Idx >= 0)
4138 Idx -= StartIdx[0];
4139 }
4140
4141 setValue(V: &I, NewN: DAG.getVectorShuffle(VT, dl: DL, N1: Src1, N2: Src2, Mask: MappedOps));
4142 return;
4143 }
4144
4145 // We can't use either concat vectors or extract subvectors so fall back to
4146 // replacing the shuffle with extract and build vector.
4147 // to insert and build vector.
4148 EVT EltVT = VT.getVectorElementType();
4149 SmallVector<SDValue,8> Ops;
4150 for (int Idx : Mask) {
4151 SDValue Res;
4152
4153 if (Idx < 0) {
4154 Res = DAG.getUNDEF(VT: EltVT);
4155 } else {
4156 SDValue &Src = Idx < (int)SrcNumElts ? Src1 : Src2;
4157 if (Idx >= (int)SrcNumElts) Idx -= SrcNumElts;
4158
4159 Res = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: EltVT, N1: Src,
4160 N2: DAG.getVectorIdxConstant(Val: Idx, DL));
4161 }
4162
4163 Ops.push_back(Elt: Res);
4164 }
4165
4166 setValue(V: &I, NewN: DAG.getBuildVector(VT, DL, Ops));
4167}
4168
4169void SelectionDAGBuilder::visitInsertValue(const InsertValueInst &I) {
4170 ArrayRef<unsigned> Indices = I.getIndices();
4171 const Value *Op0 = I.getOperand(i_nocapture: 0);
4172 const Value *Op1 = I.getOperand(i_nocapture: 1);
4173 Type *AggTy = I.getType();
4174 Type *ValTy = Op1->getType();
4175 bool IntoUndef = isa<UndefValue>(Val: Op0);
4176 bool FromUndef = isa<UndefValue>(Val: Op1);
4177
4178 unsigned LinearIndex = ComputeLinearIndex(Ty: AggTy, Indices);
4179
4180 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4181 SmallVector<EVT, 4> AggValueVTs;
4182 ComputeValueVTs(TLI, DL: DAG.getDataLayout(), Ty: AggTy, ValueVTs&: AggValueVTs);
4183 SmallVector<EVT, 4> ValValueVTs;
4184 ComputeValueVTs(TLI, DL: DAG.getDataLayout(), Ty: ValTy, ValueVTs&: ValValueVTs);
4185
4186 unsigned NumAggValues = AggValueVTs.size();
4187 unsigned NumValValues = ValValueVTs.size();
4188 SmallVector<SDValue, 4> Values(NumAggValues);
4189
4190 // Ignore an insertvalue that produces an empty object
4191 if (!NumAggValues) {
4192 setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));
4193 return;
4194 }
4195
4196 SDValue Agg = getValue(V: Op0);
4197 unsigned i = 0;
4198 // Copy the beginning value(s) from the original aggregate.
4199 for (; i != LinearIndex; ++i)
4200 Values[i] = IntoUndef ? DAG.getUNDEF(VT: AggValueVTs[i]) :
4201 SDValue(Agg.getNode(), Agg.getResNo() + i);
4202 // Copy values from the inserted value(s).
4203 if (NumValValues) {
4204 SDValue Val = getValue(V: Op1);
4205 for (; i != LinearIndex + NumValValues; ++i)
4206 Values[i] = FromUndef ? DAG.getUNDEF(VT: AggValueVTs[i]) :
4207 SDValue(Val.getNode(), Val.getResNo() + i - LinearIndex);
4208 }
4209 // Copy remaining value(s) from the original aggregate.
4210 for (; i != NumAggValues; ++i)
4211 Values[i] = IntoUndef ? DAG.getUNDEF(VT: AggValueVTs[i]) :
4212 SDValue(Agg.getNode(), Agg.getResNo() + i);
4213
4214 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: getCurSDLoc(),
4215 VTList: DAG.getVTList(VTs: AggValueVTs), Ops: Values));
4216}
4217
4218void SelectionDAGBuilder::visitExtractValue(const ExtractValueInst &I) {
4219 ArrayRef<unsigned> Indices = I.getIndices();
4220 const Value *Op0 = I.getOperand(i_nocapture: 0);
4221 Type *AggTy = Op0->getType();
4222 Type *ValTy = I.getType();
4223 bool OutOfUndef = isa<UndefValue>(Val: Op0);
4224
4225 unsigned LinearIndex = ComputeLinearIndex(Ty: AggTy, Indices);
4226
4227 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4228 SmallVector<EVT, 4> ValValueVTs;
4229 ComputeValueVTs(TLI, DL: DAG.getDataLayout(), Ty: ValTy, ValueVTs&: ValValueVTs);
4230
4231 unsigned NumValValues = ValValueVTs.size();
4232
4233 // Ignore a extractvalue that produces an empty object
4234 if (!NumValValues) {
4235 setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));
4236 return;
4237 }
4238
4239 SmallVector<SDValue, 4> Values(NumValValues);
4240
4241 SDValue Agg = getValue(V: Op0);
4242 // Copy out the selected value(s).
4243 for (unsigned i = LinearIndex; i != LinearIndex + NumValValues; ++i)
4244 Values[i - LinearIndex] =
4245 OutOfUndef ?
4246 DAG.getUNDEF(VT: Agg.getNode()->getValueType(ResNo: Agg.getResNo() + i)) :
4247 SDValue(Agg.getNode(), Agg.getResNo() + i);
4248
4249 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: getCurSDLoc(),
4250 VTList: DAG.getVTList(VTs: ValValueVTs), Ops: Values));
4251}
4252
4253void SelectionDAGBuilder::visitGetElementPtr(const User &I) {
4254 Value *Op0 = I.getOperand(i: 0);
4255 // Note that the pointer operand may be a vector of pointers. Take the scalar
4256 // element which holds a pointer.
4257 unsigned AS = Op0->getType()->getScalarType()->getPointerAddressSpace();
4258 SDValue N = getValue(V: Op0);
4259 SDLoc dl = getCurSDLoc();
4260 auto &TLI = DAG.getTargetLoweringInfo();
4261 GEPNoWrapFlags NW = cast<GEPOperator>(Val: I).getNoWrapFlags();
4262
4263 // Normalize Vector GEP - all scalar operands should be converted to the
4264 // splat vector.
4265 bool IsVectorGEP = I.getType()->isVectorTy();
4266 ElementCount VectorElementCount =
4267 IsVectorGEP ? cast<VectorType>(Val: I.getType())->getElementCount()
4268 : ElementCount::getFixed(MinVal: 0);
4269
4270 if (IsVectorGEP && !N.getValueType().isVector()) {
4271 LLVMContext &Context = *DAG.getContext();
4272 EVT VT = EVT::getVectorVT(Context, VT: N.getValueType(), EC: VectorElementCount);
4273 N = DAG.getSplat(VT, DL: dl, Op: N);
4274 }
4275
4276 for (gep_type_iterator GTI = gep_type_begin(GEP: &I), E = gep_type_end(GEP: &I);
4277 GTI != E; ++GTI) {
4278 const Value *Idx = GTI.getOperand();
4279 if (StructType *StTy = GTI.getStructTypeOrNull()) {
4280 unsigned Field = cast<Constant>(Val: Idx)->getUniqueInteger().getZExtValue();
4281 if (Field) {
4282 // N = N + Offset
4283 uint64_t Offset =
4284 DAG.getDataLayout().getStructLayout(Ty: StTy)->getElementOffset(Idx: Field);
4285
4286 // In an inbounds GEP with an offset that is nonnegative even when
4287 // interpreted as signed, assume there is no unsigned overflow.
4288 SDNodeFlags Flags;
4289 if (NW.hasNoUnsignedWrap() ||
4290 (int64_t(Offset) >= 0 && NW.hasNoUnsignedSignedWrap()))
4291 Flags |= SDNodeFlags::NoUnsignedWrap;
4292
4293 N = DAG.getMemBasePlusOffset(
4294 Base: N, Offset: DAG.getConstant(Val: Offset, DL: dl, VT: N.getValueType()), DL: dl, Flags);
4295 }
4296 } else {
4297 // IdxSize is the width of the arithmetic according to IR semantics.
4298 // In SelectionDAG, we may prefer to do arithmetic in a wider bitwidth
4299 // (and fix up the result later).
4300 unsigned IdxSize = DAG.getDataLayout().getIndexSizeInBits(AS);
4301 MVT IdxTy = MVT::getIntegerVT(BitWidth: IdxSize);
4302 TypeSize ElementSize =
4303 GTI.getSequentialElementStride(DL: DAG.getDataLayout());
4304 // We intentionally mask away the high bits here; ElementSize may not
4305 // fit in IdxTy.
4306 APInt ElementMul(IdxSize, ElementSize.getKnownMinValue(),
4307 /*isSigned=*/false, /*implicitTrunc=*/true);
4308 bool ElementScalable = ElementSize.isScalable();
4309
4310 // If this is a scalar constant or a splat vector of constants,
4311 // handle it quickly.
4312 const auto *C = dyn_cast<Constant>(Val: Idx);
4313 if (C && isa<VectorType>(Val: C->getType()))
4314 C = C->getSplatValue();
4315
4316 const auto *CI = dyn_cast_or_null<ConstantInt>(Val: C);
4317 if (CI && CI->isZero())
4318 continue;
4319 if (CI && !ElementScalable) {
4320 APInt Offs = ElementMul * CI->getValue().sextOrTrunc(width: IdxSize);
4321 LLVMContext &Context = *DAG.getContext();
4322 SDValue OffsVal;
4323 if (IsVectorGEP)
4324 OffsVal = DAG.getConstant(
4325 Val: Offs, DL: dl, VT: EVT::getVectorVT(Context, VT: IdxTy, EC: VectorElementCount));
4326 else
4327 OffsVal = DAG.getConstant(Val: Offs, DL: dl, VT: IdxTy);
4328
4329 // In an inbounds GEP with an offset that is nonnegative even when
4330 // interpreted as signed, assume there is no unsigned overflow.
4331 SDNodeFlags Flags;
4332 if (NW.hasNoUnsignedWrap() ||
4333 (Offs.isNonNegative() && NW.hasNoUnsignedSignedWrap()))
4334 Flags.setNoUnsignedWrap(true);
4335
4336 OffsVal = DAG.getSExtOrTrunc(Op: OffsVal, DL: dl, VT: N.getValueType());
4337
4338 N = DAG.getMemBasePlusOffset(Base: N, Offset: OffsVal, DL: dl, Flags);
4339 continue;
4340 }
4341
4342 // N = N + Idx * ElementMul;
4343 SDValue IdxN = getValue(V: Idx);
4344
4345 if (!IdxN.getValueType().isVector() && IsVectorGEP) {
4346 EVT VT = EVT::getVectorVT(Context&: *Context, VT: IdxN.getValueType(),
4347 EC: VectorElementCount);
4348 IdxN = DAG.getSplat(VT, DL: dl, Op: IdxN);
4349 }
4350
4351 // If the index is smaller or larger than intptr_t, truncate or extend
4352 // it.
4353 IdxN = DAG.getSExtOrTrunc(Op: IdxN, DL: dl, VT: N.getValueType());
4354
4355 SDNodeFlags ScaleFlags;
4356 // The multiplication of an index by the type size does not wrap the
4357 // pointer index type in a signed sense (mul nsw).
4358 ScaleFlags.setNoSignedWrap(NW.hasNoUnsignedSignedWrap());
4359
4360 // The multiplication of an index by the type size does not wrap the
4361 // pointer index type in an unsigned sense (mul nuw).
4362 ScaleFlags.setNoUnsignedWrap(NW.hasNoUnsignedWrap());
4363
4364 if (ElementScalable) {
4365 EVT VScaleTy = N.getValueType().getScalarType();
4366 SDValue VScale = DAG.getNode(
4367 Opcode: ISD::VSCALE, DL: dl, VT: VScaleTy,
4368 Operand: DAG.getConstant(Val: ElementMul.getZExtValue(), DL: dl, VT: VScaleTy));
4369 if (IsVectorGEP)
4370 VScale = DAG.getSplatVector(VT: N.getValueType(), DL: dl, Op: VScale);
4371 IdxN = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: N.getValueType(), N1: IdxN, N2: VScale,
4372 Flags: ScaleFlags);
4373 } else {
4374 // If this is a multiply by a power of two, turn it into a shl
4375 // immediately. This is a very common case.
4376 if (ElementMul != 1) {
4377 if (ElementMul.isPowerOf2()) {
4378 unsigned Amt = ElementMul.logBase2();
4379 IdxN = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: N.getValueType(), N1: IdxN,
4380 N2: DAG.getConstant(Val: Amt, DL: dl, VT: IdxN.getValueType()),
4381 Flags: ScaleFlags);
4382 } else {
4383 SDValue Scale = DAG.getConstant(Val: ElementMul.getZExtValue(), DL: dl,
4384 VT: IdxN.getValueType());
4385 IdxN = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: N.getValueType(), N1: IdxN, N2: Scale,
4386 Flags: ScaleFlags);
4387 }
4388 }
4389 }
4390
4391 // The successive addition of the current address, truncated to the
4392 // pointer index type and interpreted as an unsigned number, and each
4393 // offset, also interpreted as an unsigned number, does not wrap the
4394 // pointer index type (add nuw).
4395 SDNodeFlags AddFlags;
4396 AddFlags.setNoUnsignedWrap(NW.hasNoUnsignedWrap());
4397
4398 N = DAG.getMemBasePlusOffset(Base: N, Offset: IdxN, DL: dl, Flags: AddFlags);
4399 }
4400 }
4401
4402 MVT PtrTy = TLI.getPointerTy(DL: DAG.getDataLayout(), AS);
4403 MVT PtrMemTy = TLI.getPointerMemTy(DL: DAG.getDataLayout(), AS);
4404 if (IsVectorGEP) {
4405 PtrTy = MVT::getVectorVT(VT: PtrTy, EC: VectorElementCount);
4406 PtrMemTy = MVT::getVectorVT(VT: PtrMemTy, EC: VectorElementCount);
4407 }
4408
4409 if (PtrMemTy != PtrTy && !cast<GEPOperator>(Val: I).isInBounds())
4410 N = DAG.getPtrExtendInReg(Op: N, DL: dl, VT: PtrMemTy);
4411
4412 setValue(V: &I, NewN: N);
4413}
4414
4415void SelectionDAGBuilder::visitAlloca(const AllocaInst &I) {
4416 // If this is a fixed sized alloca in the entry block of the function,
4417 // allocate it statically on the stack.
4418 if (FuncInfo.StaticAllocaMap.count(Val: &I))
4419 return; // getValue will auto-populate this.
4420
4421 SDLoc dl = getCurSDLoc();
4422 Type *Ty = I.getAllocatedType();
4423 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4424 auto &DL = DAG.getDataLayout();
4425 TypeSize TySize = DL.getTypeAllocSize(Ty);
4426 MaybeAlign Alignment = std::max(a: DL.getPrefTypeAlign(Ty), b: I.getAlign());
4427
4428 SDValue AllocSize = getValue(V: I.getArraySize());
4429
4430 EVT IntPtr = TLI.getPointerTy(DL, AS: I.getAddressSpace());
4431 if (AllocSize.getValueType() != IntPtr)
4432 AllocSize = DAG.getZExtOrTrunc(Op: AllocSize, DL: dl, VT: IntPtr);
4433
4434 if (TySize.isScalable())
4435 AllocSize = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: IntPtr, N1: AllocSize,
4436 N2: DAG.getVScale(DL: dl, VT: IntPtr,
4437 MulImm: APInt(IntPtr.getScalarSizeInBits(),
4438 TySize.getKnownMinValue())));
4439 else {
4440 SDValue TySizeValue =
4441 DAG.getConstant(Val: TySize.getFixedValue(), DL: dl, VT: MVT::getIntegerVT(BitWidth: 64));
4442 AllocSize = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: IntPtr, N1: AllocSize,
4443 N2: DAG.getZExtOrTrunc(Op: TySizeValue, DL: dl, VT: IntPtr));
4444 }
4445
4446 // Handle alignment. If the requested alignment is less than or equal to
4447 // the stack alignment, ignore it. If the size is greater than or equal to
4448 // the stack alignment, we note this in the DYNAMIC_STACKALLOC node.
4449 Align StackAlign = DAG.getSubtarget().getFrameLowering()->getStackAlign();
4450 if (*Alignment <= StackAlign)
4451 Alignment = std::nullopt;
4452
4453 const uint64_t StackAlignMask = StackAlign.value() - 1U;
4454 // Round the size of the allocation up to the stack alignment size
4455 // by add SA-1 to the size. This doesn't overflow because we're computing
4456 // an address inside an alloca.
4457 AllocSize = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: AllocSize.getValueType(), N1: AllocSize,
4458 N2: DAG.getConstant(Val: StackAlignMask, DL: dl, VT: IntPtr),
4459 Flags: SDNodeFlags::NoUnsignedWrap);
4460
4461 // Mask out the low bits for alignment purposes.
4462 AllocSize = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: AllocSize.getValueType(), N1: AllocSize,
4463 N2: DAG.getSignedConstant(Val: ~StackAlignMask, DL: dl, VT: IntPtr));
4464
4465 SDValue Ops[] = {
4466 getRoot(), AllocSize,
4467 DAG.getConstant(Val: Alignment ? Alignment->value() : 0, DL: dl, VT: IntPtr)};
4468 SDVTList VTs = DAG.getVTList(AllocSize.getValueType(), MVT::Other);
4469 SDValue DSA = DAG.getNode(Opcode: ISD::DYNAMIC_STACKALLOC, DL: dl, VTList: VTs, Ops);
4470 setValue(V: &I, NewN: DSA);
4471 DAG.setRoot(DSA.getValue(R: 1));
4472
4473 assert(FuncInfo.MF->getFrameInfo().hasVarSizedObjects());
4474}
4475
4476static const MDNode *getRangeMetadata(const Instruction &I) {
4477 return I.getMetadata(KindID: LLVMContext::MD_range);
4478}
4479
4480static std::optional<ConstantRange> getRange(const Instruction &I) {
4481 if (const auto *CB = dyn_cast<CallBase>(Val: &I))
4482 if (std::optional<ConstantRange> CR = CB->getRange())
4483 return CR;
4484 if (const MDNode *Range = getRangeMetadata(I))
4485 return getConstantRangeFromMetadata(RangeMD: *Range);
4486 return std::nullopt;
4487}
4488
4489void SelectionDAGBuilder::visitLoad(const LoadInst &I) {
4490 if (I.isAtomic())
4491 return visitAtomicLoad(I);
4492
4493 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4494 const Value *SV = I.getOperand(i_nocapture: 0);
4495 if (TLI.supportSwiftError()) {
4496 // Swifterror values can come from either a function parameter with
4497 // swifterror attribute or an alloca with swifterror attribute.
4498 if (const Argument *Arg = dyn_cast<Argument>(Val: SV)) {
4499 if (Arg->hasSwiftErrorAttr())
4500 return visitLoadFromSwiftError(I);
4501 }
4502
4503 if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(Val: SV)) {
4504 if (Alloca->isSwiftError())
4505 return visitLoadFromSwiftError(I);
4506 }
4507 }
4508
4509 SDValue Ptr = getValue(V: SV);
4510
4511 Type *Ty = I.getType();
4512 SmallVector<EVT, 4> ValueVTs, MemVTs;
4513 SmallVector<TypeSize, 4> Offsets;
4514 ComputeValueVTs(TLI, DL: DAG.getDataLayout(), Ty, ValueVTs, MemVTs: &MemVTs, Offsets: &Offsets);
4515 unsigned NumValues = ValueVTs.size();
4516 if (NumValues == 0)
4517 return;
4518
4519 Align Alignment = I.getAlign();
4520 AAMDNodes AAInfo = I.getAAMetadata();
4521 const MDNode *Ranges = getRangeMetadata(I);
4522 bool isVolatile = I.isVolatile();
4523 MachineMemOperand::Flags MMOFlags =
4524 TLI.getLoadMemOperandFlags(LI: I, DL: DAG.getDataLayout(), AC, LibInfo);
4525
4526 SDValue Root;
4527 bool ConstantMemory = false;
4528 if (isVolatile)
4529 // Serialize volatile loads with other side effects.
4530 Root = getRoot();
4531 else if (NumValues > MaxParallelChains)
4532 Root = getMemoryRoot();
4533 else if (BatchAA &&
4534 BatchAA->pointsToConstantMemory(Loc: MemoryLocation(
4535 SV,
4536 LocationSize::precise(Value: DAG.getDataLayout().getTypeStoreSize(Ty)),
4537 AAInfo))) {
4538 // Do not serialize (non-volatile) loads of constant memory with anything.
4539 Root = DAG.getEntryNode();
4540 ConstantMemory = true;
4541 MMOFlags |= MachineMemOperand::MOInvariant;
4542 } else {
4543 // Do not serialize non-volatile loads against each other.
4544 Root = DAG.getRoot();
4545 }
4546
4547 SDLoc dl = getCurSDLoc();
4548
4549 if (isVolatile)
4550 Root = TLI.prepareVolatileOrAtomicLoad(Chain: Root, DL: dl, DAG);
4551
4552 SmallVector<SDValue, 4> Values(NumValues);
4553 SmallVector<SDValue, 4> Chains(std::min(a: MaxParallelChains, b: NumValues));
4554
4555 unsigned ChainI = 0;
4556 for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
4557 // Serializing loads here may result in excessive register pressure, and
4558 // TokenFactor places arbitrary choke points on the scheduler. SD scheduling
4559 // could recover a bit by hoisting nodes upward in the chain by recognizing
4560 // they are side-effect free or do not alias. The optimizer should really
4561 // avoid this case by converting large object/array copies to llvm.memcpy
4562 // (MaxParallelChains should always remain as failsafe).
4563 if (ChainI == MaxParallelChains) {
4564 assert(PendingLoads.empty() && "PendingLoads must be serialized first");
4565 SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
4566 ArrayRef(Chains.data(), ChainI));
4567 Root = Chain;
4568 ChainI = 0;
4569 }
4570
4571 // TODO: MachinePointerInfo only supports a fixed length offset.
4572 MachinePointerInfo PtrInfo =
4573 !Offsets[i].isScalable() || Offsets[i].isZero()
4574 ? MachinePointerInfo(SV, Offsets[i].getKnownMinValue())
4575 : MachinePointerInfo();
4576
4577 SDValue A = DAG.getObjectPtrOffset(SL: dl, Ptr, Offset: Offsets[i]);
4578 SDValue L = DAG.getLoad(VT: MemVTs[i], dl, Chain: Root, Ptr: A, PtrInfo, Alignment,
4579 MMOFlags, AAInfo, Ranges);
4580 Chains[ChainI] = L.getValue(R: 1);
4581
4582 if (MemVTs[i] != ValueVTs[i])
4583 L = DAG.getPtrExtOrTrunc(Op: L, DL: dl, VT: ValueVTs[i]);
4584
4585 Values[i] = L;
4586 }
4587
4588 if (!ConstantMemory) {
4589 SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
4590 ArrayRef(Chains.data(), ChainI));
4591 if (isVolatile)
4592 DAG.setRoot(Chain);
4593 else
4594 PendingLoads.push_back(Elt: Chain);
4595 }
4596
4597 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: dl,
4598 VTList: DAG.getVTList(VTs: ValueVTs), Ops: Values));
4599}
4600
4601void SelectionDAGBuilder::visitStoreToSwiftError(const StoreInst &I) {
4602 assert(DAG.getTargetLoweringInfo().supportSwiftError() &&
4603 "call visitStoreToSwiftError when backend supports swifterror");
4604
4605 SmallVector<EVT, 4> ValueVTs;
4606 SmallVector<uint64_t, 4> Offsets;
4607 const Value *SrcV = I.getOperand(i_nocapture: 0);
4608 ComputeValueVTs(TLI: DAG.getTargetLoweringInfo(), DL: DAG.getDataLayout(),
4609 Ty: SrcV->getType(), ValueVTs, FixedOffsets: &Offsets, StartingOffset: 0);
4610 assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&
4611 "expect a single EVT for swifterror");
4612
4613 SDValue Src = getValue(V: SrcV);
4614 // Create a virtual register, then update the virtual register.
4615 Register VReg =
4616 SwiftError.getOrCreateVRegDefAt(&I, FuncInfo.MBB, I.getPointerOperand());
4617 // Chain, DL, Reg, N or Chain, DL, Reg, N, Glue
4618 // Chain can be getRoot or getControlRoot.
4619 SDValue CopyNode = DAG.getCopyToReg(Chain: getRoot(), dl: getCurSDLoc(), Reg: VReg,
4620 N: SDValue(Src.getNode(), Src.getResNo()));
4621 DAG.setRoot(CopyNode);
4622}
4623
4624void SelectionDAGBuilder::visitLoadFromSwiftError(const LoadInst &I) {
4625 assert(DAG.getTargetLoweringInfo().supportSwiftError() &&
4626 "call visitLoadFromSwiftError when backend supports swifterror");
4627
4628 assert(!I.isVolatile() &&
4629 !I.hasMetadata(LLVMContext::MD_nontemporal) &&
4630 !I.hasMetadata(LLVMContext::MD_invariant_load) &&
4631 "Support volatile, non temporal, invariant for load_from_swift_error");
4632
4633 const Value *SV = I.getOperand(i_nocapture: 0);
4634 Type *Ty = I.getType();
4635 assert(
4636 (!BatchAA ||
4637 !BatchAA->pointsToConstantMemory(MemoryLocation(
4638 SV, LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)),
4639 I.getAAMetadata()))) &&
4640 "load_from_swift_error should not be constant memory");
4641
4642 SmallVector<EVT, 4> ValueVTs;
4643 SmallVector<uint64_t, 4> Offsets;
4644 ComputeValueVTs(TLI: DAG.getTargetLoweringInfo(), DL: DAG.getDataLayout(), Ty,
4645 ValueVTs, FixedOffsets: &Offsets, StartingOffset: 0);
4646 assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&
4647 "expect a single EVT for swifterror");
4648
4649 // Chain, DL, Reg, VT, Glue or Chain, DL, Reg, VT
4650 SDValue L = DAG.getCopyFromReg(
4651 Chain: getRoot(), dl: getCurSDLoc(),
4652 Reg: SwiftError.getOrCreateVRegUseAt(&I, FuncInfo.MBB, SV), VT: ValueVTs[0]);
4653
4654 setValue(V: &I, NewN: L);
4655}
4656
4657void SelectionDAGBuilder::visitStore(const StoreInst &I) {
4658 if (I.isAtomic())
4659 return visitAtomicStore(I);
4660
4661 const Value *SrcV = I.getOperand(i_nocapture: 0);
4662 const Value *PtrV = I.getOperand(i_nocapture: 1);
4663
4664 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4665 if (TLI.supportSwiftError()) {
4666 // Swifterror values can come from either a function parameter with
4667 // swifterror attribute or an alloca with swifterror attribute.
4668 if (const Argument *Arg = dyn_cast<Argument>(Val: PtrV)) {
4669 if (Arg->hasSwiftErrorAttr())
4670 return visitStoreToSwiftError(I);
4671 }
4672
4673 if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(Val: PtrV)) {
4674 if (Alloca->isSwiftError())
4675 return visitStoreToSwiftError(I);
4676 }
4677 }
4678
4679 SmallVector<EVT, 4> ValueVTs, MemVTs;
4680 SmallVector<TypeSize, 4> Offsets;
4681 ComputeValueVTs(TLI: DAG.getTargetLoweringInfo(), DL: DAG.getDataLayout(),
4682 Ty: SrcV->getType(), ValueVTs, MemVTs: &MemVTs, Offsets: &Offsets);
4683 unsigned NumValues = ValueVTs.size();
4684 if (NumValues == 0)
4685 return;
4686
4687 // Get the lowered operands. Note that we do this after
4688 // checking if NumResults is zero, because with zero results
4689 // the operands won't have values in the map.
4690 SDValue Src = getValue(V: SrcV);
4691 SDValue Ptr = getValue(V: PtrV);
4692
4693 SDValue Root = I.isVolatile() ? getRoot() : getMemoryRoot();
4694 SmallVector<SDValue, 4> Chains(std::min(a: MaxParallelChains, b: NumValues));
4695 SDLoc dl = getCurSDLoc();
4696 Align Alignment = I.getAlign();
4697 AAMDNodes AAInfo = I.getAAMetadata();
4698
4699 auto MMOFlags = TLI.getStoreMemOperandFlags(SI: I, DL: DAG.getDataLayout());
4700
4701 unsigned ChainI = 0;
4702 for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
4703 // See visitLoad comments.
4704 if (ChainI == MaxParallelChains) {
4705 SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
4706 ArrayRef(Chains.data(), ChainI));
4707 Root = Chain;
4708 ChainI = 0;
4709 }
4710
4711 // TODO: MachinePointerInfo only supports a fixed length offset.
4712 MachinePointerInfo PtrInfo =
4713 !Offsets[i].isScalable() || Offsets[i].isZero()
4714 ? MachinePointerInfo(PtrV, Offsets[i].getKnownMinValue())
4715 : MachinePointerInfo();
4716
4717 SDValue Add = DAG.getObjectPtrOffset(SL: dl, Ptr, Offset: Offsets[i]);
4718 SDValue Val = SDValue(Src.getNode(), Src.getResNo() + i);
4719 if (MemVTs[i] != ValueVTs[i])
4720 Val = DAG.getPtrExtOrTrunc(Op: Val, DL: dl, VT: MemVTs[i]);
4721 SDValue St =
4722 DAG.getStore(Chain: Root, dl, Val, Ptr: Add, PtrInfo, Alignment, MMOFlags, AAInfo);
4723 Chains[ChainI] = St;
4724 }
4725
4726 SDValue StoreNode = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
4727 ArrayRef(Chains.data(), ChainI));
4728 setValue(V: &I, NewN: StoreNode);
4729 DAG.setRoot(StoreNode);
4730}
4731
4732void SelectionDAGBuilder::visitMaskedStore(const CallInst &I,
4733 bool IsCompressing) {
4734 SDLoc sdl = getCurSDLoc();
4735
4736 auto getMaskedStoreOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0,
4737 Align &Alignment) {
4738 // llvm.masked.store.*(Src0, Ptr, alignment, Mask)
4739 Src0 = I.getArgOperand(i: 0);
4740 Ptr = I.getArgOperand(i: 1);
4741 Alignment = cast<ConstantInt>(Val: I.getArgOperand(i: 2))->getAlignValue();
4742 Mask = I.getArgOperand(i: 3);
4743 };
4744 auto getCompressingStoreOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0,
4745 Align &Alignment) {
4746 // llvm.masked.compressstore.*(Src0, Ptr, Mask)
4747 Src0 = I.getArgOperand(i: 0);
4748 Ptr = I.getArgOperand(i: 1);
4749 Mask = I.getArgOperand(i: 2);
4750 Alignment = I.getParamAlign(ArgNo: 1).valueOrOne();
4751 };
4752
4753 Value *PtrOperand, *MaskOperand, *Src0Operand;
4754 Align Alignment;
4755 if (IsCompressing)
4756 getCompressingStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4757 else
4758 getMaskedStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4759
4760 SDValue Ptr = getValue(V: PtrOperand);
4761 SDValue Src0 = getValue(V: Src0Operand);
4762 SDValue Mask = getValue(V: MaskOperand);
4763 SDValue Offset = DAG.getUNDEF(VT: Ptr.getValueType());
4764
4765 EVT VT = Src0.getValueType();
4766
4767 auto MMOFlags = MachineMemOperand::MOStore;
4768 if (I.hasMetadata(KindID: LLVMContext::MD_nontemporal))
4769 MMOFlags |= MachineMemOperand::MONonTemporal;
4770
4771 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
4772 PtrInfo: MachinePointerInfo(PtrOperand), F: MMOFlags,
4773 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment, AAInfo: I.getAAMetadata());
4774
4775 const auto &TLI = DAG.getTargetLoweringInfo();
4776 const auto &TTI =
4777 TLI.getTargetMachine().getTargetTransformInfo(F: *I.getFunction());
4778 SDValue StoreNode =
4779 !IsCompressing && TTI.hasConditionalLoadStoreForType(
4780 Ty: I.getArgOperand(i: 0)->getType(), /*IsStore=*/true)
4781 ? TLI.visitMaskedStore(DAG, DL: sdl, Chain: getMemoryRoot(), MMO, Ptr, Val: Src0,
4782 Mask)
4783 : DAG.getMaskedStore(Chain: getMemoryRoot(), dl: sdl, Val: Src0, Base: Ptr, Offset, Mask,
4784 MemVT: VT, MMO, AM: ISD::UNINDEXED, /*Truncating=*/IsTruncating: false,
4785 IsCompressing);
4786 DAG.setRoot(StoreNode);
4787 setValue(V: &I, NewN: StoreNode);
4788}
4789
4790// Get a uniform base for the Gather/Scatter intrinsic.
4791// The first argument of the Gather/Scatter intrinsic is a vector of pointers.
4792// We try to represent it as a base pointer + vector of indices.
4793// Usually, the vector of pointers comes from a 'getelementptr' instruction.
4794// The first operand of the GEP may be a single pointer or a vector of pointers
4795// Example:
4796// %gep.ptr = getelementptr i32, <8 x i32*> %vptr, <8 x i32> %ind
4797// or
4798// %gep.ptr = getelementptr i32, i32* %ptr, <8 x i32> %ind
4799// %res = call <8 x i32> @llvm.masked.gather.v8i32(<8 x i32*> %gep.ptr, ..
4800//
4801// When the first GEP operand is a single pointer - it is the uniform base we
4802// are looking for. If first operand of the GEP is a splat vector - we
4803// extract the splat value and use it as a uniform base.
4804// In all other cases the function returns 'false'.
4805static bool getUniformBase(const Value *Ptr, SDValue &Base, SDValue &Index,
4806 ISD::MemIndexType &IndexType, SDValue &Scale,
4807 SelectionDAGBuilder *SDB, const BasicBlock *CurBB,
4808 uint64_t ElemSize) {
4809 SelectionDAG& DAG = SDB->DAG;
4810 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4811 const DataLayout &DL = DAG.getDataLayout();
4812
4813 assert(Ptr->getType()->isVectorTy() && "Unexpected pointer type");
4814
4815 // Handle splat constant pointer.
4816 if (auto *C = dyn_cast<Constant>(Val: Ptr)) {
4817 C = C->getSplatValue();
4818 if (!C)
4819 return false;
4820
4821 Base = SDB->getValue(V: C);
4822
4823 ElementCount NumElts = cast<VectorType>(Val: Ptr->getType())->getElementCount();
4824 EVT VT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: TLI.getPointerTy(DL), EC: NumElts);
4825 Index = DAG.getConstant(Val: 0, DL: SDB->getCurSDLoc(), VT);
4826 IndexType = ISD::SIGNED_SCALED;
4827 Scale = DAG.getTargetConstant(Val: 1, DL: SDB->getCurSDLoc(), VT: TLI.getPointerTy(DL));
4828 return true;
4829 }
4830
4831 const GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Val: Ptr);
4832 if (!GEP || GEP->getParent() != CurBB)
4833 return false;
4834
4835 if (GEP->getNumOperands() != 2)
4836 return false;
4837
4838 const Value *BasePtr = GEP->getPointerOperand();
4839 const Value *IndexVal = GEP->getOperand(i_nocapture: GEP->getNumOperands() - 1);
4840
4841 // Make sure the base is scalar and the index is a vector.
4842 if (BasePtr->getType()->isVectorTy() || !IndexVal->getType()->isVectorTy())
4843 return false;
4844
4845 TypeSize ScaleVal = DL.getTypeAllocSize(Ty: GEP->getResultElementType());
4846 if (ScaleVal.isScalable())
4847 return false;
4848
4849 // Target may not support the required addressing mode.
4850 if (ScaleVal != 1 &&
4851 !TLI.isLegalScaleForGatherScatter(Scale: ScaleVal.getFixedValue(), ElemSize))
4852 return false;
4853
4854 Base = SDB->getValue(V: BasePtr);
4855 Index = SDB->getValue(V: IndexVal);
4856 IndexType = ISD::SIGNED_SCALED;
4857
4858 Scale =
4859 DAG.getTargetConstant(Val: ScaleVal, DL: SDB->getCurSDLoc(), VT: TLI.getPointerTy(DL));
4860 return true;
4861}
4862
4863void SelectionDAGBuilder::visitMaskedScatter(const CallInst &I) {
4864 SDLoc sdl = getCurSDLoc();
4865
4866 // llvm.masked.scatter.*(Src0, Ptrs, alignment, Mask)
4867 const Value *Ptr = I.getArgOperand(i: 1);
4868 SDValue Src0 = getValue(V: I.getArgOperand(i: 0));
4869 SDValue Mask = getValue(V: I.getArgOperand(i: 3));
4870 EVT VT = Src0.getValueType();
4871 Align Alignment = cast<ConstantInt>(Val: I.getArgOperand(i: 2))
4872 ->getMaybeAlignValue()
4873 .value_or(u: DAG.getEVTAlign(MemoryVT: VT.getScalarType()));
4874 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4875
4876 SDValue Base;
4877 SDValue Index;
4878 ISD::MemIndexType IndexType;
4879 SDValue Scale;
4880 bool UniformBase = getUniformBase(Ptr, Base, Index, IndexType, Scale, SDB: this,
4881 CurBB: I.getParent(), ElemSize: VT.getScalarStoreSize());
4882
4883 unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace();
4884 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
4885 PtrInfo: MachinePointerInfo(AS), F: MachineMemOperand::MOStore,
4886 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment, AAInfo: I.getAAMetadata());
4887 if (!UniformBase) {
4888 Base = DAG.getConstant(Val: 0, DL: sdl, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
4889 Index = getValue(V: Ptr);
4890 IndexType = ISD::SIGNED_SCALED;
4891 Scale = DAG.getTargetConstant(Val: 1, DL: sdl, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
4892 }
4893
4894 EVT IdxVT = Index.getValueType();
4895 EVT EltTy = IdxVT.getVectorElementType();
4896 if (TLI.shouldExtendGSIndex(VT: IdxVT, EltTy)) {
4897 EVT NewIdxVT = IdxVT.changeVectorElementType(EltVT: EltTy);
4898 Index = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: sdl, VT: NewIdxVT, Operand: Index);
4899 }
4900
4901 SDValue Ops[] = { getMemoryRoot(), Src0, Mask, Base, Index, Scale };
4902 SDValue Scatter = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), VT, sdl,
4903 Ops, MMO, IndexType, false);
4904 DAG.setRoot(Scatter);
4905 setValue(V: &I, NewN: Scatter);
4906}
4907
4908void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I, bool IsExpanding) {
4909 SDLoc sdl = getCurSDLoc();
4910
4911 auto getMaskedLoadOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0,
4912 Align &Alignment) {
4913 // @llvm.masked.load.*(Ptr, alignment, Mask, Src0)
4914 Ptr = I.getArgOperand(i: 0);
4915 Alignment = cast<ConstantInt>(Val: I.getArgOperand(i: 1))->getAlignValue();
4916 Mask = I.getArgOperand(i: 2);
4917 Src0 = I.getArgOperand(i: 3);
4918 };
4919 auto getExpandingLoadOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0,
4920 Align &Alignment) {
4921 // @llvm.masked.expandload.*(Ptr, Mask, Src0)
4922 Ptr = I.getArgOperand(i: 0);
4923 Alignment = I.getParamAlign(ArgNo: 0).valueOrOne();
4924 Mask = I.getArgOperand(i: 1);
4925 Src0 = I.getArgOperand(i: 2);
4926 };
4927
4928 Value *PtrOperand, *MaskOperand, *Src0Operand;
4929 Align Alignment;
4930 if (IsExpanding)
4931 getExpandingLoadOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4932 else
4933 getMaskedLoadOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4934
4935 SDValue Ptr = getValue(V: PtrOperand);
4936 SDValue Src0 = getValue(V: Src0Operand);
4937 SDValue Mask = getValue(V: MaskOperand);
4938 SDValue Offset = DAG.getUNDEF(VT: Ptr.getValueType());
4939
4940 EVT VT = Src0.getValueType();
4941 AAMDNodes AAInfo = I.getAAMetadata();
4942 const MDNode *Ranges = getRangeMetadata(I);
4943
4944 // Do not serialize masked loads of constant memory with anything.
4945 MemoryLocation ML = MemoryLocation::getAfter(Ptr: PtrOperand, AATags: AAInfo);
4946 bool AddToChain = !BatchAA || !BatchAA->pointsToConstantMemory(Loc: ML);
4947
4948 SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
4949
4950 auto MMOFlags = MachineMemOperand::MOLoad;
4951 if (I.hasMetadata(KindID: LLVMContext::MD_nontemporal))
4952 MMOFlags |= MachineMemOperand::MONonTemporal;
4953
4954 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
4955 PtrInfo: MachinePointerInfo(PtrOperand), F: MMOFlags,
4956 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment, AAInfo, Ranges);
4957
4958 const auto &TLI = DAG.getTargetLoweringInfo();
4959 const auto &TTI =
4960 TLI.getTargetMachine().getTargetTransformInfo(F: *I.getFunction());
4961 // The Load/Res may point to different values and both of them are output
4962 // variables.
4963 SDValue Load;
4964 SDValue Res;
4965 if (!IsExpanding && TTI.hasConditionalLoadStoreForType(Ty: Src0Operand->getType(),
4966 /*IsStore=*/false))
4967 Res = TLI.visitMaskedLoad(DAG, DL: sdl, Chain: InChain, MMO, NewLoad&: Load, Ptr, PassThru: Src0, Mask);
4968 else
4969 Res = Load =
4970 DAG.getMaskedLoad(VT, dl: sdl, Chain: InChain, Base: Ptr, Offset, Mask, Src0, MemVT: VT, MMO,
4971 AM: ISD::UNINDEXED, ISD::NON_EXTLOAD, IsExpanding);
4972 if (AddToChain)
4973 PendingLoads.push_back(Elt: Load.getValue(R: 1));
4974 setValue(V: &I, NewN: Res);
4975}
4976
4977void SelectionDAGBuilder::visitMaskedGather(const CallInst &I) {
4978 SDLoc sdl = getCurSDLoc();
4979
4980 // @llvm.masked.gather.*(Ptrs, alignment, Mask, Src0)
4981 const Value *Ptr = I.getArgOperand(i: 0);
4982 SDValue Src0 = getValue(V: I.getArgOperand(i: 3));
4983 SDValue Mask = getValue(V: I.getArgOperand(i: 2));
4984
4985 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4986 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
4987 Align Alignment = cast<ConstantInt>(Val: I.getArgOperand(i: 1))
4988 ->getMaybeAlignValue()
4989 .value_or(u: DAG.getEVTAlign(MemoryVT: VT.getScalarType()));
4990
4991 const MDNode *Ranges = getRangeMetadata(I);
4992
4993 SDValue Root = DAG.getRoot();
4994 SDValue Base;
4995 SDValue Index;
4996 ISD::MemIndexType IndexType;
4997 SDValue Scale;
4998 bool UniformBase = getUniformBase(Ptr, Base, Index, IndexType, Scale, SDB: this,
4999 CurBB: I.getParent(), ElemSize: VT.getScalarStoreSize());
5000 unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace();
5001 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
5002 PtrInfo: MachinePointerInfo(AS), F: MachineMemOperand::MOLoad,
5003 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment, AAInfo: I.getAAMetadata(),
5004 Ranges);
5005
5006 if (!UniformBase) {
5007 Base = DAG.getConstant(Val: 0, DL: sdl, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
5008 Index = getValue(V: Ptr);
5009 IndexType = ISD::SIGNED_SCALED;
5010 Scale = DAG.getTargetConstant(Val: 1, DL: sdl, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
5011 }
5012
5013 EVT IdxVT = Index.getValueType();
5014 EVT EltTy = IdxVT.getVectorElementType();
5015 if (TLI.shouldExtendGSIndex(VT: IdxVT, EltTy)) {
5016 EVT NewIdxVT = IdxVT.changeVectorElementType(EltVT: EltTy);
5017 Index = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: sdl, VT: NewIdxVT, Operand: Index);
5018 }
5019
5020 SDValue Ops[] = { Root, Src0, Mask, Base, Index, Scale };
5021 SDValue Gather = DAG.getMaskedGather(DAG.getVTList(VT, MVT::Other), VT, sdl,
5022 Ops, MMO, IndexType, ISD::NON_EXTLOAD);
5023
5024 PendingLoads.push_back(Elt: Gather.getValue(R: 1));
5025 setValue(V: &I, NewN: Gather);
5026}
5027
5028void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) {
5029 SDLoc dl = getCurSDLoc();
5030 AtomicOrdering SuccessOrdering = I.getSuccessOrdering();
5031 AtomicOrdering FailureOrdering = I.getFailureOrdering();
5032 SyncScope::ID SSID = I.getSyncScopeID();
5033
5034 SDValue InChain = getRoot();
5035
5036 MVT MemVT = getValue(V: I.getCompareOperand()).getSimpleValueType();
5037 SDVTList VTs = DAG.getVTList(MemVT, MVT::i1, MVT::Other);
5038
5039 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5040 auto Flags = TLI.getAtomicMemOperandFlags(AI: I, DL: DAG.getDataLayout());
5041
5042 MachineFunction &MF = DAG.getMachineFunction();
5043 MachineMemOperand *MMO = MF.getMachineMemOperand(
5044 PtrInfo: MachinePointerInfo(I.getPointerOperand()), F: Flags, Size: MemVT.getStoreSize(),
5045 BaseAlignment: DAG.getEVTAlign(MemoryVT: MemVT), AAInfo: AAMDNodes(), Ranges: nullptr, SSID, Ordering: SuccessOrdering,
5046 FailureOrdering);
5047
5048 SDValue L = DAG.getAtomicCmpSwap(Opcode: ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS,
5049 dl, MemVT, VTs, Chain: InChain,
5050 Ptr: getValue(V: I.getPointerOperand()),
5051 Cmp: getValue(V: I.getCompareOperand()),
5052 Swp: getValue(V: I.getNewValOperand()), MMO);
5053
5054 SDValue OutChain = L.getValue(R: 2);
5055
5056 setValue(V: &I, NewN: L);
5057 DAG.setRoot(OutChain);
5058}
5059
5060void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) {
5061 SDLoc dl = getCurSDLoc();
5062 ISD::NodeType NT;
5063 switch (I.getOperation()) {
5064 default: llvm_unreachable("Unknown atomicrmw operation");
5065 case AtomicRMWInst::Xchg: NT = ISD::ATOMIC_SWAP; break;
5066 case AtomicRMWInst::Add: NT = ISD::ATOMIC_LOAD_ADD; break;
5067 case AtomicRMWInst::Sub: NT = ISD::ATOMIC_LOAD_SUB; break;
5068 case AtomicRMWInst::And: NT = ISD::ATOMIC_LOAD_AND; break;
5069 case AtomicRMWInst::Nand: NT = ISD::ATOMIC_LOAD_NAND; break;
5070 case AtomicRMWInst::Or: NT = ISD::ATOMIC_LOAD_OR; break;
5071 case AtomicRMWInst::Xor: NT = ISD::ATOMIC_LOAD_XOR; break;
5072 case AtomicRMWInst::Max: NT = ISD::ATOMIC_LOAD_MAX; break;
5073 case AtomicRMWInst::Min: NT = ISD::ATOMIC_LOAD_MIN; break;
5074 case AtomicRMWInst::UMax: NT = ISD::ATOMIC_LOAD_UMAX; break;
5075 case AtomicRMWInst::UMin: NT = ISD::ATOMIC_LOAD_UMIN; break;
5076 case AtomicRMWInst::FAdd: NT = ISD::ATOMIC_LOAD_FADD; break;
5077 case AtomicRMWInst::FSub: NT = ISD::ATOMIC_LOAD_FSUB; break;
5078 case AtomicRMWInst::FMax: NT = ISD::ATOMIC_LOAD_FMAX; break;
5079 case AtomicRMWInst::FMin: NT = ISD::ATOMIC_LOAD_FMIN; break;
5080 case AtomicRMWInst::FMaximum:
5081 NT = ISD::ATOMIC_LOAD_FMAXIMUM;
5082 break;
5083 case AtomicRMWInst::FMinimum:
5084 NT = ISD::ATOMIC_LOAD_FMINIMUM;
5085 break;
5086 case AtomicRMWInst::UIncWrap:
5087 NT = ISD::ATOMIC_LOAD_UINC_WRAP;
5088 break;
5089 case AtomicRMWInst::UDecWrap:
5090 NT = ISD::ATOMIC_LOAD_UDEC_WRAP;
5091 break;
5092 case AtomicRMWInst::USubCond:
5093 NT = ISD::ATOMIC_LOAD_USUB_COND;
5094 break;
5095 case AtomicRMWInst::USubSat:
5096 NT = ISD::ATOMIC_LOAD_USUB_SAT;
5097 break;
5098 }
5099 AtomicOrdering Ordering = I.getOrdering();
5100 SyncScope::ID SSID = I.getSyncScopeID();
5101
5102 SDValue InChain = getRoot();
5103
5104 auto MemVT = getValue(V: I.getValOperand()).getSimpleValueType();
5105 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5106 auto Flags = TLI.getAtomicMemOperandFlags(AI: I, DL: DAG.getDataLayout());
5107
5108 MachineFunction &MF = DAG.getMachineFunction();
5109 MachineMemOperand *MMO = MF.getMachineMemOperand(
5110 PtrInfo: MachinePointerInfo(I.getPointerOperand()), F: Flags, Size: MemVT.getStoreSize(),
5111 BaseAlignment: DAG.getEVTAlign(MemoryVT: MemVT), AAInfo: AAMDNodes(), Ranges: nullptr, SSID, Ordering);
5112
5113 SDValue L =
5114 DAG.getAtomic(Opcode: NT, dl, MemVT, Chain: InChain,
5115 Ptr: getValue(V: I.getPointerOperand()), Val: getValue(V: I.getValOperand()),
5116 MMO);
5117
5118 SDValue OutChain = L.getValue(R: 1);
5119
5120 setValue(V: &I, NewN: L);
5121 DAG.setRoot(OutChain);
5122}
5123
5124void SelectionDAGBuilder::visitFence(const FenceInst &I) {
5125 SDLoc dl = getCurSDLoc();
5126 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5127 SDValue Ops[3];
5128 Ops[0] = getRoot();
5129 Ops[1] = DAG.getTargetConstant(Val: (unsigned)I.getOrdering(), DL: dl,
5130 VT: TLI.getFenceOperandTy(DL: DAG.getDataLayout()));
5131 Ops[2] = DAG.getTargetConstant(Val: I.getSyncScopeID(), DL: dl,
5132 VT: TLI.getFenceOperandTy(DL: DAG.getDataLayout()));
5133 SDValue N = DAG.getNode(ISD::ATOMIC_FENCE, dl, MVT::Other, Ops);
5134 setValue(V: &I, NewN: N);
5135 DAG.setRoot(N);
5136}
5137
5138void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) {
5139 SDLoc dl = getCurSDLoc();
5140 AtomicOrdering Order = I.getOrdering();
5141 SyncScope::ID SSID = I.getSyncScopeID();
5142
5143 SDValue InChain = getRoot();
5144
5145 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5146 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
5147 EVT MemVT = TLI.getMemValueType(DL: DAG.getDataLayout(), Ty: I.getType());
5148
5149 if (!TLI.supportsUnalignedAtomics() &&
5150 I.getAlign().value() < MemVT.getSizeInBits() / 8)
5151 report_fatal_error(reason: "Cannot generate unaligned atomic load");
5152
5153 auto Flags = TLI.getLoadMemOperandFlags(LI: I, DL: DAG.getDataLayout(), AC, LibInfo);
5154
5155 const MDNode *Ranges = getRangeMetadata(I);
5156 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
5157 PtrInfo: MachinePointerInfo(I.getPointerOperand()), F: Flags, Size: MemVT.getStoreSize(),
5158 BaseAlignment: I.getAlign(), AAInfo: AAMDNodes(), Ranges, SSID, Ordering: Order);
5159
5160 InChain = TLI.prepareVolatileOrAtomicLoad(Chain: InChain, DL: dl, DAG);
5161
5162 SDValue Ptr = getValue(V: I.getPointerOperand());
5163 SDValue L =
5164 DAG.getAtomicLoad(ExtType: ISD::NON_EXTLOAD, dl, MemVT, VT: MemVT, Chain: InChain, Ptr, MMO);
5165
5166 SDValue OutChain = L.getValue(R: 1);
5167 if (MemVT != VT)
5168 L = DAG.getPtrExtOrTrunc(Op: L, DL: dl, VT);
5169
5170 setValue(V: &I, NewN: L);
5171 DAG.setRoot(OutChain);
5172}
5173
5174void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) {
5175 SDLoc dl = getCurSDLoc();
5176
5177 AtomicOrdering Ordering = I.getOrdering();
5178 SyncScope::ID SSID = I.getSyncScopeID();
5179
5180 SDValue InChain = getRoot();
5181
5182 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5183 EVT MemVT =
5184 TLI.getMemValueType(DL: DAG.getDataLayout(), Ty: I.getValueOperand()->getType());
5185
5186 if (!TLI.supportsUnalignedAtomics() &&
5187 I.getAlign().value() < MemVT.getSizeInBits() / 8)
5188 report_fatal_error(reason: "Cannot generate unaligned atomic store");
5189
5190 auto Flags = TLI.getStoreMemOperandFlags(SI: I, DL: DAG.getDataLayout());
5191
5192 MachineFunction &MF = DAG.getMachineFunction();
5193 MachineMemOperand *MMO = MF.getMachineMemOperand(
5194 PtrInfo: MachinePointerInfo(I.getPointerOperand()), F: Flags, Size: MemVT.getStoreSize(),
5195 BaseAlignment: I.getAlign(), AAInfo: AAMDNodes(), Ranges: nullptr, SSID, Ordering);
5196
5197 SDValue Val = getValue(V: I.getValueOperand());
5198 if (Val.getValueType() != MemVT)
5199 Val = DAG.getPtrExtOrTrunc(Op: Val, DL: dl, VT: MemVT);
5200 SDValue Ptr = getValue(V: I.getPointerOperand());
5201
5202 SDValue OutChain =
5203 DAG.getAtomic(Opcode: ISD::ATOMIC_STORE, dl, MemVT, Chain: InChain, Ptr: Val, Val: Ptr, MMO);
5204
5205 setValue(V: &I, NewN: OutChain);
5206 DAG.setRoot(OutChain);
5207}
5208
5209/// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC
5210/// node.
5211void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I,
5212 unsigned Intrinsic) {
5213 // Ignore the callsite's attributes. A specific call site may be marked with
5214 // readnone, but the lowering code will expect the chain based on the
5215 // definition.
5216 const Function *F = I.getCalledFunction();
5217 bool HasChain = !F->doesNotAccessMemory();
5218 bool OnlyLoad =
5219 HasChain && F->onlyReadsMemory() && F->willReturn() && F->doesNotThrow();
5220
5221 // Build the operand list.
5222 SmallVector<SDValue, 8> Ops;
5223 if (HasChain) { // If this intrinsic has side-effects, chainify it.
5224 if (OnlyLoad) {
5225 // We don't need to serialize loads against other loads.
5226 Ops.push_back(Elt: DAG.getRoot());
5227 } else {
5228 Ops.push_back(Elt: getRoot());
5229 }
5230 }
5231
5232 // Info is set by getTgtMemIntrinsic
5233 TargetLowering::IntrinsicInfo Info;
5234 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5235 bool IsTgtIntrinsic = TLI.getTgtMemIntrinsic(Info, I,
5236 DAG.getMachineFunction(),
5237 Intrinsic);
5238
5239 // Add the intrinsic ID as an integer operand if it's not a target intrinsic.
5240 if (!IsTgtIntrinsic || Info.opc == ISD::INTRINSIC_VOID ||
5241 Info.opc == ISD::INTRINSIC_W_CHAIN)
5242 Ops.push_back(Elt: DAG.getTargetConstant(Val: Intrinsic, DL: getCurSDLoc(),
5243 VT: TLI.getPointerTy(DL: DAG.getDataLayout())));
5244
5245 // Add all operands of the call to the operand list.
5246 for (unsigned i = 0, e = I.arg_size(); i != e; ++i) {
5247 const Value *Arg = I.getArgOperand(i);
5248 if (!I.paramHasAttr(i, Attribute::ImmArg)) {
5249 Ops.push_back(Elt: getValue(V: Arg));
5250 continue;
5251 }
5252
5253 // Use TargetConstant instead of a regular constant for immarg.
5254 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: Arg->getType(), AllowUnknown: true);
5255 if (const ConstantInt *CI = dyn_cast<ConstantInt>(Val: Arg)) {
5256 assert(CI->getBitWidth() <= 64 &&
5257 "large intrinsic immediates not handled");
5258 Ops.push_back(Elt: DAG.getTargetConstant(Val: *CI, DL: SDLoc(), VT));
5259 } else {
5260 Ops.push_back(
5261 Elt: DAG.getTargetConstantFP(Val: *cast<ConstantFP>(Val: Arg), DL: SDLoc(), VT));
5262 }
5263 }
5264
5265 SmallVector<EVT, 4> ValueVTs;
5266 ComputeValueVTs(TLI, DL: DAG.getDataLayout(), Ty: I.getType(), ValueVTs);
5267
5268 if (HasChain)
5269 ValueVTs.push_back(MVT::Other);
5270
5271 SDVTList VTs = DAG.getVTList(VTs: ValueVTs);
5272
5273 // Propagate fast-math-flags from IR to node(s).
5274 SDNodeFlags Flags;
5275 if (auto *FPMO = dyn_cast<FPMathOperator>(Val: &I))
5276 Flags.copyFMF(FPMO: *FPMO);
5277 SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);
5278
5279 // Create the node.
5280 SDValue Result;
5281
5282 if (auto Bundle = I.getOperandBundle(ID: LLVMContext::OB_convergencectrl)) {
5283 auto *Token = Bundle->Inputs[0].get();
5284 SDValue ConvControlToken = getValue(V: Token);
5285 assert(Ops.back().getValueType() != MVT::Glue &&
5286 "Did not expected another glue node here.");
5287 ConvControlToken =
5288 DAG.getNode(ISD::CONVERGENCECTRL_GLUE, {}, MVT::Glue, ConvControlToken);
5289 Ops.push_back(Elt: ConvControlToken);
5290 }
5291
5292 // In some cases, custom collection of operands from CallInst I may be needed.
5293 TLI.CollectTargetIntrinsicOperands(I, Ops, DAG);
5294 if (IsTgtIntrinsic) {
5295 // This is target intrinsic that touches memory
5296 //
5297 // TODO: We currently just fallback to address space 0 if getTgtMemIntrinsic
5298 // didn't yield anything useful.
5299 MachinePointerInfo MPI;
5300 if (Info.ptrVal)
5301 MPI = MachinePointerInfo(Info.ptrVal, Info.offset);
5302 else if (Info.fallbackAddressSpace)
5303 MPI = MachinePointerInfo(*Info.fallbackAddressSpace);
5304 EVT MemVT = Info.memVT;
5305 LocationSize Size = LocationSize::precise(Value: Info.size);
5306 if (Size.hasValue() && !Size.getValue())
5307 Size = LocationSize::precise(Value: MemVT.getStoreSize());
5308 Align Alignment = Info.align.value_or(u: DAG.getEVTAlign(MemoryVT: MemVT));
5309 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
5310 PtrInfo: MPI, F: Info.flags, Size, BaseAlignment: Alignment, AAInfo: I.getAAMetadata(), /*Ranges=*/nullptr,
5311 SSID: Info.ssid, Ordering: Info.order, FailureOrdering: Info.failureOrder);
5312 Result =
5313 DAG.getMemIntrinsicNode(Opcode: Info.opc, dl: getCurSDLoc(), VTList: VTs, Ops, MemVT, MMO);
5314 } else if (!HasChain) {
5315 Result = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: getCurSDLoc(), VTList: VTs, Ops);
5316 } else if (!I.getType()->isVoidTy()) {
5317 Result = DAG.getNode(Opcode: ISD::INTRINSIC_W_CHAIN, DL: getCurSDLoc(), VTList: VTs, Ops);
5318 } else {
5319 Result = DAG.getNode(Opcode: ISD::INTRINSIC_VOID, DL: getCurSDLoc(), VTList: VTs, Ops);
5320 }
5321
5322 if (HasChain) {
5323 SDValue Chain = Result.getValue(R: Result.getNode()->getNumValues()-1);
5324 if (OnlyLoad)
5325 PendingLoads.push_back(Elt: Chain);
5326 else
5327 DAG.setRoot(Chain);
5328 }
5329
5330 if (!I.getType()->isVoidTy()) {
5331 if (!isa<VectorType>(Val: I.getType()))
5332 Result = lowerRangeToAssertZExt(DAG, I, Op: Result);
5333
5334 MaybeAlign Alignment = I.getRetAlign();
5335
5336 // Insert `assertalign` node if there's an alignment.
5337 if (InsertAssertAlign && Alignment) {
5338 Result =
5339 DAG.getAssertAlign(DL: getCurSDLoc(), V: Result, A: Alignment.valueOrOne());
5340 }
5341 }
5342
5343 setValue(V: &I, NewN: Result);
5344}
5345
5346/// GetSignificand - Get the significand and build it into a floating-point
5347/// number with exponent of 1:
5348///
5349/// Op = (Op & 0x007fffff) | 0x3f800000;
5350///
5351/// where Op is the hexadecimal representation of floating point value.
5352static SDValue GetSignificand(SelectionDAG &DAG, SDValue Op, const SDLoc &dl) {
5353 SDValue t1 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
5354 DAG.getConstant(0x007fffff, dl, MVT::i32));
5355 SDValue t2 = DAG.getNode(ISD::OR, dl, MVT::i32, t1,
5356 DAG.getConstant(0x3f800000, dl, MVT::i32));
5357 return DAG.getNode(ISD::BITCAST, dl, MVT::f32, t2);
5358}
5359
5360/// GetExponent - Get the exponent:
5361///
5362/// (float)(int)(((Op & 0x7f800000) >> 23) - 127);
5363///
5364/// where Op is the hexadecimal representation of floating point value.
5365static SDValue GetExponent(SelectionDAG &DAG, SDValue Op,
5366 const TargetLowering &TLI, const SDLoc &dl) {
5367 SDValue t0 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
5368 DAG.getConstant(0x7f800000, dl, MVT::i32));
5369 SDValue t1 = DAG.getNode(
5370 ISD::SRL, dl, MVT::i32, t0,
5371 DAG.getConstant(23, dl,
5372 TLI.getShiftAmountTy(MVT::i32, DAG.getDataLayout())));
5373 SDValue t2 = DAG.getNode(ISD::SUB, dl, MVT::i32, t1,
5374 DAG.getConstant(127, dl, MVT::i32));
5375 return DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, t2);
5376}
5377
5378/// getF32Constant - Get 32-bit floating point constant.
5379static SDValue getF32Constant(SelectionDAG &DAG, unsigned Flt,
5380 const SDLoc &dl) {
5381 return DAG.getConstantFP(APFloat(APFloat::IEEEsingle(), APInt(32, Flt)), dl,
5382 MVT::f32);
5383}
5384
5385static SDValue getLimitedPrecisionExp2(SDValue t0, const SDLoc &dl,
5386 SelectionDAG &DAG) {
5387 // TODO: What fast-math-flags should be set on the floating-point nodes?
5388
5389 // IntegerPartOfX = ((int32_t)(t0);
5390 SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, t0);
5391
5392 // FractionalPartOfX = t0 - (float)IntegerPartOfX;
5393 SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX);
5394 SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, t1);
5395
5396 // IntegerPartOfX <<= 23;
5397 IntegerPartOfX =
5398 DAG.getNode(ISD::SHL, dl, MVT::i32, IntegerPartOfX,
5399 DAG.getConstant(23, dl,
5400 DAG.getTargetLoweringInfo().getShiftAmountTy(
5401 MVT::i32, DAG.getDataLayout())));
5402
5403 SDValue TwoToFractionalPartOfX;
5404 if (LimitFloatPrecision <= 6) {
5405 // For floating-point precision of 6:
5406 //
5407 // TwoToFractionalPartOfX =
5408 // 0.997535578f +
5409 // (0.735607626f + 0.252464424f * x) * x;
5410 //
5411 // error 0.0144103317, which is 6 bits
5412 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5413 getF32Constant(DAG, 0x3e814304, dl));
5414 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
5415 getF32Constant(DAG, 0x3f3c50c8, dl));
5416 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5417 TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5418 getF32Constant(DAG, 0x3f7f5e7e, dl));
5419 } else if (LimitFloatPrecision <= 12) {
5420 // For floating-point precision of 12:
5421 //
5422 // TwoToFractionalPartOfX =
5423 // 0.999892986f +
5424 // (0.696457318f +
5425 // (0.224338339f + 0.792043434e-1f * x) * x) * x;
5426 //
5427 // error 0.000107046256, which is 13 to 14 bits
5428 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5429 getF32Constant(DAG, 0x3da235e3, dl));
5430 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
5431 getF32Constant(DAG, 0x3e65b8f3, dl));
5432 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5433 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5434 getF32Constant(DAG, 0x3f324b07, dl));
5435 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5436 TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
5437 getF32Constant(DAG, 0x3f7ff8fd, dl));
5438 } else { // LimitFloatPrecision <= 18
5439 // For floating-point precision of 18:
5440 //
5441 // TwoToFractionalPartOfX =
5442 // 0.999999982f +
5443 // (0.693148872f +
5444 // (0.240227044f +
5445 // (0.554906021e-1f +
5446 // (0.961591928e-2f +
5447 // (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x;
5448 // error 2.47208000*10^(-7), which is better than 18 bits
5449 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5450 getF32Constant(DAG, 0x3924b03e, dl));
5451 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
5452 getF32Constant(DAG, 0x3ab24b87, dl));
5453 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5454 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5455 getF32Constant(DAG, 0x3c1d8c17, dl));
5456 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5457 SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
5458 getF32Constant(DAG, 0x3d634a1d, dl));
5459 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
5460 SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
5461 getF32Constant(DAG, 0x3e75fe14, dl));
5462 SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
5463 SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10,
5464 getF32Constant(DAG, 0x3f317234, dl));
5465 SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X);
5466 TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12,
5467 getF32Constant(DAG, 0x3f800000, dl));
5468 }
5469
5470 // Add the exponent into the result in integer domain.
5471 SDValue t13 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, TwoToFractionalPartOfX);
5472 return DAG.getNode(ISD::BITCAST, dl, MVT::f32,
5473 DAG.getNode(ISD::ADD, dl, MVT::i32, t13, IntegerPartOfX));
5474}
5475
5476/// expandExp - Lower an exp intrinsic. Handles the special sequences for
5477/// limited-precision mode.
5478static SDValue expandExp(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
5479 const TargetLowering &TLI, SDNodeFlags Flags) {
5480 if (Op.getValueType() == MVT::f32 &&
5481 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
5482
5483 // Put the exponent in the right bit position for later addition to the
5484 // final result:
5485 //
5486 // t0 = Op * log2(e)
5487
5488 // TODO: What fast-math-flags should be set here?
5489 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, Op,
5490 DAG.getConstantFP(numbers::log2ef, dl, MVT::f32));
5491 return getLimitedPrecisionExp2(t0, dl, DAG);
5492 }
5493
5494 // No special expansion.
5495 return DAG.getNode(Opcode: ISD::FEXP, DL: dl, VT: Op.getValueType(), Operand: Op, Flags);
5496}
5497
5498/// expandLog - Lower a log intrinsic. Handles the special sequences for
5499/// limited-precision mode.
5500static SDValue expandLog(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
5501 const TargetLowering &TLI, SDNodeFlags Flags) {
5502 // TODO: What fast-math-flags should be set on the floating-point nodes?
5503
5504 if (Op.getValueType() == MVT::f32 &&
5505 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
5506 SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
5507
5508 // Scale the exponent by log(2).
5509 SDValue Exp = GetExponent(DAG, Op: Op1, TLI, dl);
5510 SDValue LogOfExponent =
5511 DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
5512 DAG.getConstantFP(numbers::ln2f, dl, MVT::f32));
5513
5514 // Get the significand and build it into a floating-point number with
5515 // exponent of 1.
5516 SDValue X = GetSignificand(DAG, Op: Op1, dl);
5517
5518 SDValue LogOfMantissa;
5519 if (LimitFloatPrecision <= 6) {
5520 // For floating-point precision of 6:
5521 //
5522 // LogofMantissa =
5523 // -1.1609546f +
5524 // (1.4034025f - 0.23903021f * x) * x;
5525 //
5526 // error 0.0034276066, which is better than 8 bits
5527 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5528 getF32Constant(DAG, 0xbe74c456, dl));
5529 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5530 getF32Constant(DAG, 0x3fb3a2b1, dl));
5531 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5532 LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5533 getF32Constant(DAG, 0x3f949a29, dl));
5534 } else if (LimitFloatPrecision <= 12) {
5535 // For floating-point precision of 12:
5536 //
5537 // LogOfMantissa =
5538 // -1.7417939f +
5539 // (2.8212026f +
5540 // (-1.4699568f +
5541 // (0.44717955f - 0.56570851e-1f * x) * x) * x) * x;
5542 //
5543 // error 0.000061011436, which is 14 bits
5544 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5545 getF32Constant(DAG, 0xbd67b6d6, dl));
5546 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5547 getF32Constant(DAG, 0x3ee4f4b8, dl));
5548 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5549 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5550 getF32Constant(DAG, 0x3fbc278b, dl));
5551 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5552 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5553 getF32Constant(DAG, 0x40348e95, dl));
5554 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5555 LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
5556 getF32Constant(DAG, 0x3fdef31a, dl));
5557 } else { // LimitFloatPrecision <= 18
5558 // For floating-point precision of 18:
5559 //
5560 // LogOfMantissa =
5561 // -2.1072184f +
5562 // (4.2372794f +
5563 // (-3.7029485f +
5564 // (2.2781945f +
5565 // (-0.87823314f +
5566 // (0.19073739f - 0.17809712e-1f * x) * x) * x) * x) * x)*x;
5567 //
5568 // error 0.0000023660568, which is better than 18 bits
5569 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5570 getF32Constant(DAG, 0xbc91e5ac, dl));
5571 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5572 getF32Constant(DAG, 0x3e4350aa, dl));
5573 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5574 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5575 getF32Constant(DAG, 0x3f60d3e3, dl));
5576 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5577 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5578 getF32Constant(DAG, 0x4011cdf0, dl));
5579 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5580 SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
5581 getF32Constant(DAG, 0x406cfd1c, dl));
5582 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
5583 SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
5584 getF32Constant(DAG, 0x408797cb, dl));
5585 SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
5586 LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
5587 getF32Constant(DAG, 0x4006dcab, dl));
5588 }
5589
5590 return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, LogOfMantissa);
5591 }
5592
5593 // No special expansion.
5594 return DAG.getNode(Opcode: ISD::FLOG, DL: dl, VT: Op.getValueType(), Operand: Op, Flags);
5595}
5596
5597/// expandLog2 - Lower a log2 intrinsic. Handles the special sequences for
5598/// limited-precision mode.
5599static SDValue expandLog2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
5600 const TargetLowering &TLI, SDNodeFlags Flags) {
5601 // TODO: What fast-math-flags should be set on the floating-point nodes?
5602
5603 if (Op.getValueType() == MVT::f32 &&
5604 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
5605 SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
5606
5607 // Get the exponent.
5608 SDValue LogOfExponent = GetExponent(DAG, Op: Op1, TLI, dl);
5609
5610 // Get the significand and build it into a floating-point number with
5611 // exponent of 1.
5612 SDValue X = GetSignificand(DAG, Op: Op1, dl);
5613
5614 // Different possible minimax approximations of significand in
5615 // floating-point for various degrees of accuracy over [1,2].
5616 SDValue Log2ofMantissa;
5617 if (LimitFloatPrecision <= 6) {
5618 // For floating-point precision of 6:
5619 //
5620 // Log2ofMantissa = -1.6749035f + (2.0246817f - .34484768f * x) * x;
5621 //
5622 // error 0.0049451742, which is more than 7 bits
5623 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5624 getF32Constant(DAG, 0xbeb08fe0, dl));
5625 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5626 getF32Constant(DAG, 0x40019463, dl));
5627 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5628 Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5629 getF32Constant(DAG, 0x3fd6633d, dl));
5630 } else if (LimitFloatPrecision <= 12) {
5631 // For floating-point precision of 12:
5632 //
5633 // Log2ofMantissa =
5634 // -2.51285454f +
5635 // (4.07009056f +
5636 // (-2.12067489f +
5637 // (.645142248f - 0.816157886e-1f * x) * x) * x) * x;
5638 //
5639 // error 0.0000876136000, which is better than 13 bits
5640 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5641 getF32Constant(DAG, 0xbda7262e, dl));
5642 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5643 getF32Constant(DAG, 0x3f25280b, dl));
5644 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5645 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5646 getF32Constant(DAG, 0x4007b923, dl));
5647 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5648 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5649 getF32Constant(DAG, 0x40823e2f, dl));
5650 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5651 Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
5652 getF32Constant(DAG, 0x4020d29c, dl));
5653 } else { // LimitFloatPrecision <= 18
5654 // For floating-point precision of 18:
5655 //
5656 // Log2ofMantissa =
5657 // -3.0400495f +
5658 // (6.1129976f +
5659 // (-5.3420409f +
5660 // (3.2865683f +
5661 // (-1.2669343f +
5662 // (0.27515199f -
5663 // 0.25691327e-1f * x) * x) * x) * x) * x) * x;
5664 //
5665 // error 0.0000018516, which is better than 18 bits
5666 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5667 getF32Constant(DAG, 0xbcd2769e, dl));
5668 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5669 getF32Constant(DAG, 0x3e8ce0b9, dl));
5670 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5671 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5672 getF32Constant(DAG, 0x3fa22ae7, dl));
5673 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5674 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5675 getF32Constant(DAG, 0x40525723, dl));
5676 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5677 SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
5678 getF32Constant(DAG, 0x40aaf200, dl));
5679 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
5680 SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
5681 getF32Constant(DAG, 0x40c39dad, dl));
5682 SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
5683 Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
5684 getF32Constant(DAG, 0x4042902c, dl));
5685 }
5686
5687 return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log2ofMantissa);
5688 }
5689
5690 // No special expansion.
5691 return DAG.getNode(Opcode: ISD::FLOG2, DL: dl, VT: Op.getValueType(), Operand: Op, Flags);
5692}
5693
5694/// expandLog10 - Lower a log10 intrinsic. Handles the special sequences for
5695/// limited-precision mode.
5696static SDValue expandLog10(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
5697 const TargetLowering &TLI, SDNodeFlags Flags) {
5698 // TODO: What fast-math-flags should be set on the floating-point nodes?
5699
5700 if (Op.getValueType() == MVT::f32 &&
5701 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
5702 SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
5703
5704 // Scale the exponent by log10(2) [0.30102999f].
5705 SDValue Exp = GetExponent(DAG, Op: Op1, TLI, dl);
5706 SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
5707 getF32Constant(DAG, 0x3e9a209a, dl));
5708
5709 // Get the significand and build it into a floating-point number with
5710 // exponent of 1.
5711 SDValue X = GetSignificand(DAG, Op: Op1, dl);
5712
5713 SDValue Log10ofMantissa;
5714 if (LimitFloatPrecision <= 6) {
5715 // For floating-point precision of 6:
5716 //
5717 // Log10ofMantissa =
5718 // -0.50419619f +
5719 // (0.60948995f - 0.10380950f * x) * x;
5720 //
5721 // error 0.0014886165, which is 6 bits
5722 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5723 getF32Constant(DAG, 0xbdd49a13, dl));
5724 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5725 getF32Constant(DAG, 0x3f1c0789, dl));
5726 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5727 Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5728 getF32Constant(DAG, 0x3f011300, dl));
5729 } else if (LimitFloatPrecision <= 12) {
5730 // For floating-point precision of 12:
5731 //
5732 // Log10ofMantissa =
5733 // -0.64831180f +
5734 // (0.91751397f +
5735 // (-0.31664806f + 0.47637168e-1f * x) * x) * x;
5736 //
5737 // error 0.00019228036, which is better than 12 bits
5738 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5739 getF32Constant(DAG, 0x3d431f31, dl));
5740 SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,
5741 getF32Constant(DAG, 0x3ea21fb2, dl));
5742 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5743 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
5744 getF32Constant(DAG, 0x3f6ae232, dl));
5745 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5746 Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,
5747 getF32Constant(DAG, 0x3f25f7c3, dl));
5748 } else { // LimitFloatPrecision <= 18
5749 // For floating-point precision of 18:
5750 //
5751 // Log10ofMantissa =
5752 // -0.84299375f +
5753 // (1.5327582f +
5754 // (-1.0688956f +
5755 // (0.49102474f +
5756 // (-0.12539807f + 0.13508273e-1f * x) * x) * x) * x) * x;
5757 //
5758 // error 0.0000037995730, which is better than 18 bits
5759 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5760 getF32Constant(DAG, 0x3c5d51ce, dl));
5761 SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,
5762 getF32Constant(DAG, 0x3e00685a, dl));
5763 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5764 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
5765 getF32Constant(DAG, 0x3efb6798, dl));
5766 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5767 SDValue t5 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,
5768 getF32Constant(DAG, 0x3f88d192, dl));
5769 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5770 SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
5771 getF32Constant(DAG, 0x3fc4316c, dl));
5772 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
5773 Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t8,
5774 getF32Constant(DAG, 0x3f57ce70, dl));
5775 }
5776
5777 return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log10ofMantissa);
5778 }
5779
5780 // No special expansion.
5781 return DAG.getNode(Opcode: ISD::FLOG10, DL: dl, VT: Op.getValueType(), Operand: Op, Flags);
5782}
5783
5784/// expandExp2 - Lower an exp2 intrinsic. Handles the special sequences for
5785/// limited-precision mode.
5786static SDValue expandExp2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
5787 const TargetLowering &TLI, SDNodeFlags Flags) {
5788 if (Op.getValueType() == MVT::f32 &&
5789 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18)
5790 return getLimitedPrecisionExp2(t0: Op, dl, DAG);
5791
5792 // No special expansion.
5793 return DAG.getNode(Opcode: ISD::FEXP2, DL: dl, VT: Op.getValueType(), Operand: Op, Flags);
5794}
5795
5796/// visitPow - Lower a pow intrinsic. Handles the special sequences for
5797/// limited-precision mode with x == 10.0f.
5798static SDValue expandPow(const SDLoc &dl, SDValue LHS, SDValue RHS,
5799 SelectionDAG &DAG, const TargetLowering &TLI,
5800 SDNodeFlags Flags) {
5801 bool IsExp10 = false;
5802 if (LHS.getValueType() == MVT::f32 && RHS.getValueType() == MVT::f32 &&
5803 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
5804 if (ConstantFPSDNode *LHSC = dyn_cast<ConstantFPSDNode>(Val&: LHS)) {
5805 APFloat Ten(10.0f);
5806 IsExp10 = LHSC->isExactlyValue(V: Ten);
5807 }
5808 }
5809
5810 // TODO: What fast-math-flags should be set on the FMUL node?
5811 if (IsExp10) {
5812 // Put the exponent in the right bit position for later addition to the
5813 // final result:
5814 //
5815 // #define LOG2OF10 3.3219281f
5816 // t0 = Op * LOG2OF10;
5817 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, RHS,
5818 getF32Constant(DAG, 0x40549a78, dl));
5819 return getLimitedPrecisionExp2(t0, dl, DAG);
5820 }
5821
5822 // No special expansion.
5823 return DAG.getNode(Opcode: ISD::FPOW, DL: dl, VT: LHS.getValueType(), N1: LHS, N2: RHS, Flags);
5824}
5825
5826/// ExpandPowI - Expand a llvm.powi intrinsic.
5827static SDValue ExpandPowI(const SDLoc &DL, SDValue LHS, SDValue RHS,
5828 SelectionDAG &DAG) {
5829 // If RHS is a constant, we can expand this out to a multiplication tree if
5830 // it's beneficial on the target, otherwise we end up lowering to a call to
5831 // __powidf2 (for example).
5832 if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(Val&: RHS)) {
5833 unsigned Val = RHSC->getSExtValue();
5834
5835 // powi(x, 0) -> 1.0
5836 if (Val == 0)
5837 return DAG.getConstantFP(Val: 1.0, DL, VT: LHS.getValueType());
5838
5839 if (DAG.getTargetLoweringInfo().isBeneficialToExpandPowI(
5840 Exponent: Val, OptForSize: DAG.shouldOptForSize())) {
5841 // Get the exponent as a positive value.
5842 if ((int)Val < 0)
5843 Val = -Val;
5844 // We use the simple binary decomposition method to generate the multiply
5845 // sequence. There are more optimal ways to do this (for example,
5846 // powi(x,15) generates one more multiply than it should), but this has
5847 // the benefit of being both really simple and much better than a libcall.
5848 SDValue Res; // Logically starts equal to 1.0
5849 SDValue CurSquare = LHS;
5850 // TODO: Intrinsics should have fast-math-flags that propagate to these
5851 // nodes.
5852 while (Val) {
5853 if (Val & 1) {
5854 if (Res.getNode())
5855 Res =
5856 DAG.getNode(Opcode: ISD::FMUL, DL, VT: Res.getValueType(), N1: Res, N2: CurSquare);
5857 else
5858 Res = CurSquare; // 1.0*CurSquare.
5859 }
5860
5861 CurSquare = DAG.getNode(Opcode: ISD::FMUL, DL, VT: CurSquare.getValueType(),
5862 N1: CurSquare, N2: CurSquare);
5863 Val >>= 1;
5864 }
5865
5866 // If the original was negative, invert the result, producing 1/(x*x*x).
5867 if (RHSC->getSExtValue() < 0)
5868 Res = DAG.getNode(Opcode: ISD::FDIV, DL, VT: LHS.getValueType(),
5869 N1: DAG.getConstantFP(Val: 1.0, DL, VT: LHS.getValueType()), N2: Res);
5870 return Res;
5871 }
5872 }
5873
5874 // Otherwise, expand to a libcall.
5875 return DAG.getNode(Opcode: ISD::FPOWI, DL, VT: LHS.getValueType(), N1: LHS, N2: RHS);
5876}
5877
5878static SDValue expandDivFix(unsigned Opcode, const SDLoc &DL,
5879 SDValue LHS, SDValue RHS, SDValue Scale,
5880 SelectionDAG &DAG, const TargetLowering &TLI) {
5881 EVT VT = LHS.getValueType();
5882 bool Signed = Opcode == ISD::SDIVFIX || Opcode == ISD::SDIVFIXSAT;
5883 bool Saturating = Opcode == ISD::SDIVFIXSAT || Opcode == ISD::UDIVFIXSAT;
5884 LLVMContext &Ctx = *DAG.getContext();
5885
5886 // If the type is legal but the operation isn't, this node might survive all
5887 // the way to operation legalization. If we end up there and we do not have
5888 // the ability to widen the type (if VT*2 is not legal), we cannot expand the
5889 // node.
5890
5891 // Coax the legalizer into expanding the node during type legalization instead
5892 // by bumping the size by one bit. This will force it to Promote, enabling the
5893 // early expansion and avoiding the need to expand later.
5894
5895 // We don't have to do this if Scale is 0; that can always be expanded, unless
5896 // it's a saturating signed operation. Those can experience true integer
5897 // division overflow, a case which we must avoid.
5898
5899 // FIXME: We wouldn't have to do this (or any of the early
5900 // expansion/promotion) if it was possible to expand a libcall of an
5901 // illegal type during operation legalization. But it's not, so things
5902 // get a bit hacky.
5903 unsigned ScaleInt = Scale->getAsZExtVal();
5904 if ((ScaleInt > 0 || (Saturating && Signed)) &&
5905 (TLI.isTypeLegal(VT) ||
5906 (VT.isVector() && TLI.isTypeLegal(VT: VT.getVectorElementType())))) {
5907 TargetLowering::LegalizeAction Action = TLI.getFixedPointOperationAction(
5908 Op: Opcode, VT, Scale: ScaleInt);
5909 if (Action != TargetLowering::Legal && Action != TargetLowering::Custom) {
5910 EVT PromVT;
5911 if (VT.isScalarInteger())
5912 PromVT = EVT::getIntegerVT(Context&: Ctx, BitWidth: VT.getSizeInBits() + 1);
5913 else if (VT.isVector()) {
5914 PromVT = VT.getVectorElementType();
5915 PromVT = EVT::getIntegerVT(Context&: Ctx, BitWidth: PromVT.getSizeInBits() + 1);
5916 PromVT = EVT::getVectorVT(Context&: Ctx, VT: PromVT, EC: VT.getVectorElementCount());
5917 } else
5918 llvm_unreachable("Wrong VT for DIVFIX?");
5919 LHS = DAG.getExtOrTrunc(IsSigned: Signed, Op: LHS, DL, VT: PromVT);
5920 RHS = DAG.getExtOrTrunc(IsSigned: Signed, Op: RHS, DL, VT: PromVT);
5921 EVT ShiftTy = TLI.getShiftAmountTy(LHSTy: PromVT, DL: DAG.getDataLayout());
5922 // For saturating operations, we need to shift up the LHS to get the
5923 // proper saturation width, and then shift down again afterwards.
5924 if (Saturating)
5925 LHS = DAG.getNode(Opcode: ISD::SHL, DL, VT: PromVT, N1: LHS,
5926 N2: DAG.getConstant(Val: 1, DL, VT: ShiftTy));
5927 SDValue Res = DAG.getNode(Opcode, DL, VT: PromVT, N1: LHS, N2: RHS, N3: Scale);
5928 if (Saturating)
5929 Res = DAG.getNode(Opcode: Signed ? ISD::SRA : ISD::SRL, DL, VT: PromVT, N1: Res,
5930 N2: DAG.getConstant(Val: 1, DL, VT: ShiftTy));
5931 return DAG.getZExtOrTrunc(Op: Res, DL, VT);
5932 }
5933 }
5934
5935 return DAG.getNode(Opcode, DL, VT, N1: LHS, N2: RHS, N3: Scale);
5936}
5937
5938// getUnderlyingArgRegs - Find underlying registers used for a truncated,
5939// bitcasted, or split argument. Returns a list of <Register, size in bits>
5940static void
5941getUnderlyingArgRegs(SmallVectorImpl<std::pair<Register, TypeSize>> &Regs,
5942 const SDValue &N) {
5943 switch (N.getOpcode()) {
5944 case ISD::CopyFromReg: {
5945 SDValue Op = N.getOperand(i: 1);
5946 Regs.emplace_back(Args: cast<RegisterSDNode>(Val&: Op)->getReg(),
5947 Args: Op.getValueType().getSizeInBits());
5948 return;
5949 }
5950 case ISD::BITCAST:
5951 case ISD::AssertZext:
5952 case ISD::AssertSext:
5953 case ISD::TRUNCATE:
5954 getUnderlyingArgRegs(Regs, N: N.getOperand(i: 0));
5955 return;
5956 case ISD::BUILD_PAIR:
5957 case ISD::BUILD_VECTOR:
5958 case ISD::CONCAT_VECTORS:
5959 for (SDValue Op : N->op_values())
5960 getUnderlyingArgRegs(Regs, N: Op);
5961 return;
5962 default:
5963 return;
5964 }
5965}
5966
5967/// If the DbgValueInst is a dbg_value of a function argument, create the
5968/// corresponding DBG_VALUE machine instruction for it now. At the end of
5969/// instruction selection, they will be inserted to the entry BB.
5970/// We don't currently support this for variadic dbg_values, as they shouldn't
5971/// appear for function arguments or in the prologue.
5972bool SelectionDAGBuilder::EmitFuncArgumentDbgValue(
5973 const Value *V, DILocalVariable *Variable, DIExpression *Expr,
5974 DILocation *DL, FuncArgumentDbgValueKind Kind, const SDValue &N) {
5975 const Argument *Arg = dyn_cast<Argument>(Val: V);
5976 if (!Arg)
5977 return false;
5978
5979 MachineFunction &MF = DAG.getMachineFunction();
5980 const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo();
5981
5982 // Helper to create DBG_INSTR_REFs or DBG_VALUEs, depending on what kind
5983 // we've been asked to pursue.
5984 auto MakeVRegDbgValue = [&](Register Reg, DIExpression *FragExpr,
5985 bool Indirect) {
5986 if (Reg.isVirtual() && MF.useDebugInstrRef()) {
5987 // For VRegs, in instruction referencing mode, create a DBG_INSTR_REF
5988 // pointing at the VReg, which will be patched up later.
5989 auto &Inst = TII->get(Opcode: TargetOpcode::DBG_INSTR_REF);
5990 SmallVector<MachineOperand, 1> MOs({MachineOperand::CreateReg(
5991 /* Reg */ Reg, /* isDef */ false, /* isImp */ false,
5992 /* isKill */ false, /* isDead */ false,
5993 /* isUndef */ false, /* isEarlyClobber */ false,
5994 /* SubReg */ 0, /* isDebug */ true)});
5995
5996 auto *NewDIExpr = FragExpr;
5997 // We don't have an "Indirect" field in DBG_INSTR_REF, fold that into
5998 // the DIExpression.
5999 if (Indirect)
6000 NewDIExpr = DIExpression::prepend(Expr: FragExpr, Flags: DIExpression::DerefBefore);
6001 SmallVector<uint64_t, 2> Ops({dwarf::DW_OP_LLVM_arg, 0});
6002 NewDIExpr = DIExpression::prependOpcodes(Expr: NewDIExpr, Ops);
6003 return BuildMI(MF, DL, MCID: Inst, IsIndirect: false, MOs, Variable, Expr: NewDIExpr);
6004 } else {
6005 // Create a completely standard DBG_VALUE.
6006 auto &Inst = TII->get(Opcode: TargetOpcode::DBG_VALUE);
6007 return BuildMI(MF, DL, MCID: Inst, IsIndirect: Indirect, Reg, Variable, Expr: FragExpr);
6008 }
6009 };
6010
6011 if (Kind == FuncArgumentDbgValueKind::Value) {
6012 // ArgDbgValues are hoisted to the beginning of the entry block. So we
6013 // should only emit as ArgDbgValue if the dbg.value intrinsic is found in
6014 // the entry block.
6015 bool IsInEntryBlock = FuncInfo.MBB == &FuncInfo.MF->front();
6016 if (!IsInEntryBlock)
6017 return false;
6018
6019 // ArgDbgValues are hoisted to the beginning of the entry block. So we
6020 // should only emit as ArgDbgValue if the dbg.value intrinsic describes a
6021 // variable that also is a param.
6022 //
6023 // Although, if we are at the top of the entry block already, we can still
6024 // emit using ArgDbgValue. This might catch some situations when the
6025 // dbg.value refers to an argument that isn't used in the entry block, so
6026 // any CopyToReg node would be optimized out and the only way to express
6027 // this DBG_VALUE is by using the physical reg (or FI) as done in this
6028 // method. ArgDbgValues are hoisted to the beginning of the entry block. So
6029 // we should only emit as ArgDbgValue if the Variable is an argument to the
6030 // current function, and the dbg.value intrinsic is found in the entry
6031 // block.
6032 bool VariableIsFunctionInputArg = Variable->isParameter() &&
6033 !DL->getInlinedAt();
6034 bool IsInPrologue = SDNodeOrder == LowestSDNodeOrder;
6035 if (!IsInPrologue && !VariableIsFunctionInputArg)
6036 return false;
6037
6038 // Here we assume that a function argument on IR level only can be used to
6039 // describe one input parameter on source level. If we for example have
6040 // source code like this
6041 //
6042 // struct A { long x, y; };
6043 // void foo(struct A a, long b) {
6044 // ...
6045 // b = a.x;
6046 // ...
6047 // }
6048 //
6049 // and IR like this
6050 //
6051 // define void @foo(i32 %a1, i32 %a2, i32 %b) {
6052 // entry:
6053 // call void @llvm.dbg.value(metadata i32 %a1, "a", DW_OP_LLVM_fragment
6054 // call void @llvm.dbg.value(metadata i32 %a2, "a", DW_OP_LLVM_fragment
6055 // call void @llvm.dbg.value(metadata i32 %b, "b",
6056 // ...
6057 // call void @llvm.dbg.value(metadata i32 %a1, "b"
6058 // ...
6059 //
6060 // then the last dbg.value is describing a parameter "b" using a value that
6061 // is an argument. But since we already has used %a1 to describe a parameter
6062 // we should not handle that last dbg.value here (that would result in an
6063 // incorrect hoisting of the DBG_VALUE to the function entry).
6064 // Notice that we allow one dbg.value per IR level argument, to accommodate
6065 // for the situation with fragments above.
6066 // If there is no node for the value being handled, we return true to skip
6067 // the normal generation of debug info, as it would kill existing debug
6068 // info for the parameter in case of duplicates.
6069 if (VariableIsFunctionInputArg) {
6070 unsigned ArgNo = Arg->getArgNo();
6071 if (ArgNo >= FuncInfo.DescribedArgs.size())
6072 FuncInfo.DescribedArgs.resize(N: ArgNo + 1, t: false);
6073 else if (!IsInPrologue && FuncInfo.DescribedArgs.test(Idx: ArgNo))
6074 return !NodeMap[V].getNode();
6075 FuncInfo.DescribedArgs.set(ArgNo);
6076 }
6077 }
6078
6079 bool IsIndirect = false;
6080 std::optional<MachineOperand> Op;
6081 // Some arguments' frame index is recorded during argument lowering.
6082 int FI = FuncInfo.getArgumentFrameIndex(A: Arg);
6083 if (FI != std::numeric_limits<int>::max())
6084 Op = MachineOperand::CreateFI(Idx: FI);
6085
6086 SmallVector<std::pair<Register, TypeSize>, 8> ArgRegsAndSizes;
6087 if (!Op && N.getNode()) {
6088 getUnderlyingArgRegs(Regs&: ArgRegsAndSizes, N);
6089 Register Reg;
6090 if (ArgRegsAndSizes.size() == 1)
6091 Reg = ArgRegsAndSizes.front().first;
6092
6093 if (Reg && Reg.isVirtual()) {
6094 MachineRegisterInfo &RegInfo = MF.getRegInfo();
6095 Register PR = RegInfo.getLiveInPhysReg(VReg: Reg);
6096 if (PR)
6097 Reg = PR;
6098 }
6099 if (Reg) {
6100 Op = MachineOperand::CreateReg(Reg, isDef: false);
6101 IsIndirect = Kind != FuncArgumentDbgValueKind::Value;
6102 }
6103 }
6104
6105 if (!Op && N.getNode()) {
6106 // Check if frame index is available.
6107 SDValue LCandidate = peekThroughBitcasts(V: N);
6108 if (LoadSDNode *LNode = dyn_cast<LoadSDNode>(Val: LCandidate.getNode()))
6109 if (FrameIndexSDNode *FINode =
6110 dyn_cast<FrameIndexSDNode>(Val: LNode->getBasePtr().getNode()))
6111 Op = MachineOperand::CreateFI(Idx: FINode->getIndex());
6112 }
6113
6114 if (!Op) {
6115 // Create a DBG_VALUE for each decomposed value in ArgRegs to cover Reg
6116 auto splitMultiRegDbgValue = [&](ArrayRef<std::pair<Register, TypeSize>>
6117 SplitRegs) {
6118 unsigned Offset = 0;
6119 for (const auto &RegAndSize : SplitRegs) {
6120 // If the expression is already a fragment, the current register
6121 // offset+size might extend beyond the fragment. In this case, only
6122 // the register bits that are inside the fragment are relevant.
6123 int RegFragmentSizeInBits = RegAndSize.second;
6124 if (auto ExprFragmentInfo = Expr->getFragmentInfo()) {
6125 uint64_t ExprFragmentSizeInBits = ExprFragmentInfo->SizeInBits;
6126 // The register is entirely outside the expression fragment,
6127 // so is irrelevant for debug info.
6128 if (Offset >= ExprFragmentSizeInBits)
6129 break;
6130 // The register is partially outside the expression fragment, only
6131 // the low bits within the fragment are relevant for debug info.
6132 if (Offset + RegFragmentSizeInBits > ExprFragmentSizeInBits) {
6133 RegFragmentSizeInBits = ExprFragmentSizeInBits - Offset;
6134 }
6135 }
6136
6137 auto FragmentExpr = DIExpression::createFragmentExpression(
6138 Expr, OffsetInBits: Offset, SizeInBits: RegFragmentSizeInBits);
6139 Offset += RegAndSize.second;
6140 // If a valid fragment expression cannot be created, the variable's
6141 // correct value cannot be determined and so it is set as poison.
6142 if (!FragmentExpr) {
6143 SDDbgValue *SDV = DAG.getConstantDbgValue(
6144 Var: Variable, Expr, C: PoisonValue::get(T: V->getType()), DL, O: SDNodeOrder);
6145 DAG.AddDbgValue(DB: SDV, isParameter: false);
6146 continue;
6147 }
6148 MachineInstr *NewMI =
6149 MakeVRegDbgValue(RegAndSize.first, *FragmentExpr,
6150 Kind != FuncArgumentDbgValueKind::Value);
6151 FuncInfo.ArgDbgValues.push_back(Elt: NewMI);
6152 }
6153 };
6154
6155 // Check if ValueMap has reg number.
6156 DenseMap<const Value *, Register>::const_iterator
6157 VMI = FuncInfo.ValueMap.find(Val: V);
6158 if (VMI != FuncInfo.ValueMap.end()) {
6159 const auto &TLI = DAG.getTargetLoweringInfo();
6160 RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), VMI->second,
6161 V->getType(), std::nullopt);
6162 if (RFV.occupiesMultipleRegs()) {
6163 splitMultiRegDbgValue(RFV.getRegsAndSizes());
6164 return true;
6165 }
6166
6167 Op = MachineOperand::CreateReg(Reg: VMI->second, isDef: false);
6168 IsIndirect = Kind != FuncArgumentDbgValueKind::Value;
6169 } else if (ArgRegsAndSizes.size() > 1) {
6170 // This was split due to the calling convention, and no virtual register
6171 // mapping exists for the value.
6172 splitMultiRegDbgValue(ArgRegsAndSizes);
6173 return true;
6174 }
6175 }
6176
6177 if (!Op)
6178 return false;
6179
6180 assert(Variable->isValidLocationForIntrinsic(DL) &&
6181 "Expected inlined-at fields to agree");
6182 MachineInstr *NewMI = nullptr;
6183
6184 if (Op->isReg())
6185 NewMI = MakeVRegDbgValue(Op->getReg(), Expr, IsIndirect);
6186 else
6187 NewMI = BuildMI(MF, DL, MCID: TII->get(Opcode: TargetOpcode::DBG_VALUE), IsIndirect: true, MOs: *Op,
6188 Variable, Expr);
6189
6190 // Otherwise, use ArgDbgValues.
6191 FuncInfo.ArgDbgValues.push_back(Elt: NewMI);
6192 return true;
6193}
6194
6195/// Return the appropriate SDDbgValue based on N.
6196SDDbgValue *SelectionDAGBuilder::getDbgValue(SDValue N,
6197 DILocalVariable *Variable,
6198 DIExpression *Expr,
6199 const DebugLoc &dl,
6200 unsigned DbgSDNodeOrder) {
6201 if (auto *FISDN = dyn_cast<FrameIndexSDNode>(Val: N.getNode())) {
6202 // Construct a FrameIndexDbgValue for FrameIndexSDNodes so we can describe
6203 // stack slot locations.
6204 //
6205 // Consider "int x = 0; int *px = &x;". There are two kinds of interesting
6206 // debug values here after optimization:
6207 //
6208 // dbg.value(i32* %px, !"int *px", !DIExpression()), and
6209 // dbg.value(i32* %px, !"int x", !DIExpression(DW_OP_deref))
6210 //
6211 // Both describe the direct values of their associated variables.
6212 return DAG.getFrameIndexDbgValue(Var: Variable, Expr, FI: FISDN->getIndex(),
6213 /*IsIndirect*/ false, DL: dl, O: DbgSDNodeOrder);
6214 }
6215 return DAG.getDbgValue(Var: Variable, Expr, N: N.getNode(), R: N.getResNo(),
6216 /*IsIndirect*/ false, DL: dl, O: DbgSDNodeOrder);
6217}
6218
6219static unsigned FixedPointIntrinsicToOpcode(unsigned Intrinsic) {
6220 switch (Intrinsic) {
6221 case Intrinsic::smul_fix:
6222 return ISD::SMULFIX;
6223 case Intrinsic::umul_fix:
6224 return ISD::UMULFIX;
6225 case Intrinsic::smul_fix_sat:
6226 return ISD::SMULFIXSAT;
6227 case Intrinsic::umul_fix_sat:
6228 return ISD::UMULFIXSAT;
6229 case Intrinsic::sdiv_fix:
6230 return ISD::SDIVFIX;
6231 case Intrinsic::udiv_fix:
6232 return ISD::UDIVFIX;
6233 case Intrinsic::sdiv_fix_sat:
6234 return ISD::SDIVFIXSAT;
6235 case Intrinsic::udiv_fix_sat:
6236 return ISD::UDIVFIXSAT;
6237 default:
6238 llvm_unreachable("Unhandled fixed point intrinsic");
6239 }
6240}
6241
6242/// Given a @llvm.call.preallocated.setup, return the corresponding
6243/// preallocated call.
6244static const CallBase *FindPreallocatedCall(const Value *PreallocatedSetup) {
6245 assert(cast<CallBase>(PreallocatedSetup)
6246 ->getCalledFunction()
6247 ->getIntrinsicID() == Intrinsic::call_preallocated_setup &&
6248 "expected call_preallocated_setup Value");
6249 for (const auto *U : PreallocatedSetup->users()) {
6250 auto *UseCall = cast<CallBase>(Val: U);
6251 const Function *Fn = UseCall->getCalledFunction();
6252 if (!Fn || Fn->getIntrinsicID() != Intrinsic::call_preallocated_arg) {
6253 return UseCall;
6254 }
6255 }
6256 llvm_unreachable("expected corresponding call to preallocated setup/arg");
6257}
6258
6259/// If DI is a debug value with an EntryValue expression, lower it using the
6260/// corresponding physical register of the associated Argument value
6261/// (guaranteed to exist by the verifier).
6262bool SelectionDAGBuilder::visitEntryValueDbgValue(
6263 ArrayRef<const Value *> Values, DILocalVariable *Variable,
6264 DIExpression *Expr, DebugLoc DbgLoc) {
6265 if (!Expr->isEntryValue() || !hasSingleElement(C&: Values))
6266 return false;
6267
6268 // These properties are guaranteed by the verifier.
6269 const Argument *Arg = cast<Argument>(Val: Values[0]);
6270 assert(Arg->hasAttribute(Attribute::AttrKind::SwiftAsync));
6271
6272 auto ArgIt = FuncInfo.ValueMap.find(Val: Arg);
6273 if (ArgIt == FuncInfo.ValueMap.end()) {
6274 LLVM_DEBUG(
6275 dbgs() << "Dropping dbg.value: expression is entry_value but "
6276 "couldn't find an associated register for the Argument\n");
6277 return true;
6278 }
6279 Register ArgVReg = ArgIt->getSecond();
6280
6281 for (auto [PhysReg, VirtReg] : FuncInfo.RegInfo->liveins())
6282 if (ArgVReg == VirtReg || ArgVReg == PhysReg) {
6283 SDDbgValue *SDV = DAG.getVRegDbgValue(
6284 Var: Variable, Expr, VReg: PhysReg, IsIndirect: false /*IsIndidrect*/, DL: DbgLoc, O: SDNodeOrder);
6285 DAG.AddDbgValue(DB: SDV, isParameter: false /*treat as dbg.declare byval parameter*/);
6286 return true;
6287 }
6288 LLVM_DEBUG(dbgs() << "Dropping dbg.value: expression is entry_value but "
6289 "couldn't find a physical register\n");
6290 return true;
6291}
6292
6293/// Lower the call to the specified intrinsic function.
6294void SelectionDAGBuilder::visitConvergenceControl(const CallInst &I,
6295 unsigned Intrinsic) {
6296 SDLoc sdl = getCurSDLoc();
6297 switch (Intrinsic) {
6298 case Intrinsic::experimental_convergence_anchor:
6299 setValue(&I, DAG.getNode(ISD::CONVERGENCECTRL_ANCHOR, sdl, MVT::Untyped));
6300 break;
6301 case Intrinsic::experimental_convergence_entry:
6302 setValue(&I, DAG.getNode(ISD::CONVERGENCECTRL_ENTRY, sdl, MVT::Untyped));
6303 break;
6304 case Intrinsic::experimental_convergence_loop: {
6305 auto Bundle = I.getOperandBundle(ID: LLVMContext::OB_convergencectrl);
6306 auto *Token = Bundle->Inputs[0].get();
6307 setValue(&I, DAG.getNode(ISD::CONVERGENCECTRL_LOOP, sdl, MVT::Untyped,
6308 getValue(Token)));
6309 break;
6310 }
6311 }
6312}
6313
6314void SelectionDAGBuilder::visitVectorHistogram(const CallInst &I,
6315 unsigned IntrinsicID) {
6316 // For now, we're only lowering an 'add' histogram.
6317 // We can add others later, e.g. saturating adds, min/max.
6318 assert(IntrinsicID == Intrinsic::experimental_vector_histogram_add &&
6319 "Tried to lower unsupported histogram type");
6320 SDLoc sdl = getCurSDLoc();
6321 Value *Ptr = I.getOperand(i_nocapture: 0);
6322 SDValue Inc = getValue(V: I.getOperand(i_nocapture: 1));
6323 SDValue Mask = getValue(V: I.getOperand(i_nocapture: 2));
6324
6325 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6326 DataLayout TargetDL = DAG.getDataLayout();
6327 EVT VT = Inc.getValueType();
6328 Align Alignment = DAG.getEVTAlign(MemoryVT: VT);
6329
6330 const MDNode *Ranges = getRangeMetadata(I);
6331
6332 SDValue Root = DAG.getRoot();
6333 SDValue Base;
6334 SDValue Index;
6335 ISD::MemIndexType IndexType;
6336 SDValue Scale;
6337 bool UniformBase = getUniformBase(Ptr, Base, Index, IndexType, Scale, SDB: this,
6338 CurBB: I.getParent(), ElemSize: VT.getScalarStoreSize());
6339
6340 unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace();
6341
6342 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
6343 PtrInfo: MachinePointerInfo(AS),
6344 F: MachineMemOperand::MOLoad | MachineMemOperand::MOStore,
6345 Size: MemoryLocation::UnknownSize, BaseAlignment: Alignment, AAInfo: I.getAAMetadata(), Ranges);
6346
6347 if (!UniformBase) {
6348 Base = DAG.getConstant(Val: 0, DL: sdl, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
6349 Index = getValue(V: Ptr);
6350 IndexType = ISD::SIGNED_SCALED;
6351 Scale =
6352 DAG.getTargetConstant(Val: 1, DL: sdl, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
6353 }
6354
6355 EVT IdxVT = Index.getValueType();
6356 EVT EltTy = IdxVT.getVectorElementType();
6357 if (TLI.shouldExtendGSIndex(VT: IdxVT, EltTy)) {
6358 EVT NewIdxVT = IdxVT.changeVectorElementType(EltVT: EltTy);
6359 Index = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: sdl, VT: NewIdxVT, Operand: Index);
6360 }
6361
6362 SDValue ID = DAG.getTargetConstant(IntrinsicID, sdl, MVT::i32);
6363
6364 SDValue Ops[] = {Root, Inc, Mask, Base, Index, Scale, ID};
6365 SDValue Histogram = DAG.getMaskedHistogram(DAG.getVTList(MVT::Other), VT, sdl,
6366 Ops, MMO, IndexType);
6367
6368 setValue(V: &I, NewN: Histogram);
6369 DAG.setRoot(Histogram);
6370}
6371
6372void SelectionDAGBuilder::visitVectorExtractLastActive(const CallInst &I,
6373 unsigned Intrinsic) {
6374 assert(Intrinsic == Intrinsic::experimental_vector_extract_last_active &&
6375 "Tried lowering invalid vector extract last");
6376 SDLoc sdl = getCurSDLoc();
6377 const DataLayout &Layout = DAG.getDataLayout();
6378 SDValue Data = getValue(V: I.getOperand(i_nocapture: 0));
6379 SDValue Mask = getValue(V: I.getOperand(i_nocapture: 1));
6380
6381 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6382 EVT ResVT = TLI.getValueType(DL: Layout, Ty: I.getType());
6383
6384 EVT ExtVT = TLI.getVectorIdxTy(DL: Layout);
6385 SDValue Idx = DAG.getNode(Opcode: ISD::VECTOR_FIND_LAST_ACTIVE, DL: sdl, VT: ExtVT, Operand: Mask);
6386 SDValue Result = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: sdl, VT: ResVT, N1: Data, N2: Idx);
6387
6388 Value *Default = I.getOperand(i_nocapture: 2);
6389 if (!isa<PoisonValue>(Val: Default) && !isa<UndefValue>(Val: Default)) {
6390 SDValue PassThru = getValue(V: Default);
6391 EVT BoolVT = Mask.getValueType().getScalarType();
6392 SDValue AnyActive = DAG.getNode(Opcode: ISD::VECREDUCE_OR, DL: sdl, VT: BoolVT, Operand: Mask);
6393 Result = DAG.getSelect(DL: sdl, VT: ResVT, Cond: AnyActive, LHS: Result, RHS: PassThru);
6394 }
6395
6396 setValue(V: &I, NewN: Result);
6397}
6398
6399/// Lower the call to the specified intrinsic function.
6400void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
6401 unsigned Intrinsic) {
6402 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6403 SDLoc sdl = getCurSDLoc();
6404 DebugLoc dl = getCurDebugLoc();
6405 SDValue Res;
6406
6407 SDNodeFlags Flags;
6408 if (auto *FPOp = dyn_cast<FPMathOperator>(Val: &I))
6409 Flags.copyFMF(FPMO: *FPOp);
6410
6411 switch (Intrinsic) {
6412 default:
6413 // By default, turn this into a target intrinsic node.
6414 visitTargetIntrinsic(I, Intrinsic);
6415 return;
6416 case Intrinsic::vscale: {
6417 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
6418 setValue(V: &I, NewN: DAG.getVScale(DL: sdl, VT, MulImm: APInt(VT.getSizeInBits(), 1)));
6419 return;
6420 }
6421 case Intrinsic::vastart: visitVAStart(I); return;
6422 case Intrinsic::vaend: visitVAEnd(I); return;
6423 case Intrinsic::vacopy: visitVACopy(I); return;
6424 case Intrinsic::returnaddress:
6425 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::RETURNADDR, DL: sdl,
6426 VT: TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType()),
6427 Operand: getValue(V: I.getArgOperand(i: 0))));
6428 return;
6429 case Intrinsic::addressofreturnaddress:
6430 setValue(V: &I,
6431 NewN: DAG.getNode(Opcode: ISD::ADDROFRETURNADDR, DL: sdl,
6432 VT: TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType())));
6433 return;
6434 case Intrinsic::sponentry:
6435 setValue(V: &I,
6436 NewN: DAG.getNode(Opcode: ISD::SPONENTRY, DL: sdl,
6437 VT: TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType())));
6438 return;
6439 case Intrinsic::frameaddress:
6440 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FRAMEADDR, DL: sdl,
6441 VT: TLI.getFrameIndexTy(DL: DAG.getDataLayout()),
6442 Operand: getValue(V: I.getArgOperand(i: 0))));
6443 return;
6444 case Intrinsic::read_volatile_register:
6445 case Intrinsic::read_register: {
6446 Value *Reg = I.getArgOperand(i: 0);
6447 SDValue Chain = getRoot();
6448 SDValue RegName =
6449 DAG.getMDNode(MD: cast<MDNode>(Val: cast<MetadataAsValue>(Val: Reg)->getMetadata()));
6450 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
6451 Res = DAG.getNode(ISD::READ_REGISTER, sdl,
6452 DAG.getVTList(VT, MVT::Other), Chain, RegName);
6453 setValue(V: &I, NewN: Res);
6454 DAG.setRoot(Res.getValue(R: 1));
6455 return;
6456 }
6457 case Intrinsic::write_register: {
6458 Value *Reg = I.getArgOperand(i: 0);
6459 Value *RegValue = I.getArgOperand(i: 1);
6460 SDValue Chain = getRoot();
6461 SDValue RegName =
6462 DAG.getMDNode(MD: cast<MDNode>(Val: cast<MetadataAsValue>(Val: Reg)->getMetadata()));
6463 DAG.setRoot(DAG.getNode(ISD::WRITE_REGISTER, sdl, MVT::Other, Chain,
6464 RegName, getValue(RegValue)));
6465 return;
6466 }
6467 case Intrinsic::memcpy:
6468 case Intrinsic::memcpy_inline: {
6469 const auto &MCI = cast<MemCpyInst>(Val: I);
6470 SDValue Dst = getValue(V: I.getArgOperand(i: 0));
6471 SDValue Src = getValue(V: I.getArgOperand(i: 1));
6472 SDValue Size = getValue(V: I.getArgOperand(i: 2));
6473 assert((!MCI.isForceInlined() || isa<ConstantSDNode>(Size)) &&
6474 "memcpy_inline needs constant size");
6475 // @llvm.memcpy.inline defines 0 and 1 to both mean no alignment.
6476 Align DstAlign = MCI.getDestAlign().valueOrOne();
6477 Align SrcAlign = MCI.getSourceAlign().valueOrOne();
6478 Align Alignment = std::min(a: DstAlign, b: SrcAlign);
6479 bool isVol = MCI.isVolatile();
6480 // FIXME: Support passing different dest/src alignments to the memcpy DAG
6481 // node.
6482 SDValue Root = isVol ? getRoot() : getMemoryRoot();
6483 SDValue MC = DAG.getMemcpy(Chain: Root, dl: sdl, Dst, Src, Size, Alignment, isVol,
6484 AlwaysInline: MCI.isForceInlined(), CI: &I, OverrideTailCall: std::nullopt,
6485 DstPtrInfo: MachinePointerInfo(I.getArgOperand(i: 0)),
6486 SrcPtrInfo: MachinePointerInfo(I.getArgOperand(i: 1)),
6487 AAInfo: I.getAAMetadata(), BatchAA);
6488 updateDAGForMaybeTailCall(MaybeTC: MC);
6489 return;
6490 }
6491 case Intrinsic::memset:
6492 case Intrinsic::memset_inline: {
6493 const auto &MSII = cast<MemSetInst>(Val: I);
6494 SDValue Dst = getValue(V: I.getArgOperand(i: 0));
6495 SDValue Value = getValue(V: I.getArgOperand(i: 1));
6496 SDValue Size = getValue(V: I.getArgOperand(i: 2));
6497 assert((!MSII.isForceInlined() || isa<ConstantSDNode>(Size)) &&
6498 "memset_inline needs constant size");
6499 // @llvm.memset defines 0 and 1 to both mean no alignment.
6500 Align DstAlign = MSII.getDestAlign().valueOrOne();
6501 bool isVol = MSII.isVolatile();
6502 SDValue Root = isVol ? getRoot() : getMemoryRoot();
6503 SDValue MC = DAG.getMemset(
6504 Chain: Root, dl: sdl, Dst, Src: Value, Size, Alignment: DstAlign, isVol, AlwaysInline: MSII.isForceInlined(),
6505 CI: &I, DstPtrInfo: MachinePointerInfo(I.getArgOperand(i: 0)), AAInfo: I.getAAMetadata());
6506 updateDAGForMaybeTailCall(MaybeTC: MC);
6507 return;
6508 }
6509 case Intrinsic::memmove: {
6510 const auto &MMI = cast<MemMoveInst>(Val: I);
6511 SDValue Op1 = getValue(V: I.getArgOperand(i: 0));
6512 SDValue Op2 = getValue(V: I.getArgOperand(i: 1));
6513 SDValue Op3 = getValue(V: I.getArgOperand(i: 2));
6514 // @llvm.memmove defines 0 and 1 to both mean no alignment.
6515 Align DstAlign = MMI.getDestAlign().valueOrOne();
6516 Align SrcAlign = MMI.getSourceAlign().valueOrOne();
6517 Align Alignment = std::min(a: DstAlign, b: SrcAlign);
6518 bool isVol = MMI.isVolatile();
6519 // FIXME: Support passing different dest/src alignments to the memmove DAG
6520 // node.
6521 SDValue Root = isVol ? getRoot() : getMemoryRoot();
6522 SDValue MM = DAG.getMemmove(Chain: Root, dl: sdl, Dst: Op1, Src: Op2, Size: Op3, Alignment, isVol, CI: &I,
6523 /* OverrideTailCall */ std::nullopt,
6524 DstPtrInfo: MachinePointerInfo(I.getArgOperand(i: 0)),
6525 SrcPtrInfo: MachinePointerInfo(I.getArgOperand(i: 1)),
6526 AAInfo: I.getAAMetadata(), BatchAA);
6527 updateDAGForMaybeTailCall(MaybeTC: MM);
6528 return;
6529 }
6530 case Intrinsic::memcpy_element_unordered_atomic: {
6531 auto &MI = cast<AnyMemCpyInst>(Val: I);
6532 SDValue Dst = getValue(V: MI.getRawDest());
6533 SDValue Src = getValue(V: MI.getRawSource());
6534 SDValue Length = getValue(V: MI.getLength());
6535
6536 Type *LengthTy = MI.getLength()->getType();
6537 unsigned ElemSz = MI.getElementSizeInBytes();
6538 bool isTC = I.isTailCall() && isInTailCallPosition(Call: I, TM: DAG.getTarget());
6539 SDValue MC =
6540 DAG.getAtomicMemcpy(Chain: getRoot(), dl: sdl, Dst, Src, Size: Length, SizeTy: LengthTy, ElemSz,
6541 isTailCall: isTC, DstPtrInfo: MachinePointerInfo(MI.getRawDest()),
6542 SrcPtrInfo: MachinePointerInfo(MI.getRawSource()));
6543 updateDAGForMaybeTailCall(MaybeTC: MC);
6544 return;
6545 }
6546 case Intrinsic::memmove_element_unordered_atomic: {
6547 auto &MI = cast<AnyMemMoveInst>(Val: I);
6548 SDValue Dst = getValue(V: MI.getRawDest());
6549 SDValue Src = getValue(V: MI.getRawSource());
6550 SDValue Length = getValue(V: MI.getLength());
6551
6552 Type *LengthTy = MI.getLength()->getType();
6553 unsigned ElemSz = MI.getElementSizeInBytes();
6554 bool isTC = I.isTailCall() && isInTailCallPosition(Call: I, TM: DAG.getTarget());
6555 SDValue MC =
6556 DAG.getAtomicMemmove(Chain: getRoot(), dl: sdl, Dst, Src, Size: Length, SizeTy: LengthTy, ElemSz,
6557 isTailCall: isTC, DstPtrInfo: MachinePointerInfo(MI.getRawDest()),
6558 SrcPtrInfo: MachinePointerInfo(MI.getRawSource()));
6559 updateDAGForMaybeTailCall(MaybeTC: MC);
6560 return;
6561 }
6562 case Intrinsic::memset_element_unordered_atomic: {
6563 auto &MI = cast<AnyMemSetInst>(Val: I);
6564 SDValue Dst = getValue(V: MI.getRawDest());
6565 SDValue Val = getValue(V: MI.getValue());
6566 SDValue Length = getValue(V: MI.getLength());
6567
6568 Type *LengthTy = MI.getLength()->getType();
6569 unsigned ElemSz = MI.getElementSizeInBytes();
6570 bool isTC = I.isTailCall() && isInTailCallPosition(Call: I, TM: DAG.getTarget());
6571 SDValue MC =
6572 DAG.getAtomicMemset(Chain: getRoot(), dl: sdl, Dst, Value: Val, Size: Length, SizeTy: LengthTy, ElemSz,
6573 isTailCall: isTC, DstPtrInfo: MachinePointerInfo(MI.getRawDest()));
6574 updateDAGForMaybeTailCall(MaybeTC: MC);
6575 return;
6576 }
6577 case Intrinsic::call_preallocated_setup: {
6578 const CallBase *PreallocatedCall = FindPreallocatedCall(PreallocatedSetup: &I);
6579 SDValue SrcValue = DAG.getSrcValue(v: PreallocatedCall);
6580 SDValue Res = DAG.getNode(ISD::PREALLOCATED_SETUP, sdl, MVT::Other,
6581 getRoot(), SrcValue);
6582 setValue(V: &I, NewN: Res);
6583 DAG.setRoot(Res);
6584 return;
6585 }
6586 case Intrinsic::call_preallocated_arg: {
6587 const CallBase *PreallocatedCall = FindPreallocatedCall(PreallocatedSetup: I.getOperand(i_nocapture: 0));
6588 SDValue SrcValue = DAG.getSrcValue(v: PreallocatedCall);
6589 SDValue Ops[3];
6590 Ops[0] = getRoot();
6591 Ops[1] = SrcValue;
6592 Ops[2] = DAG.getTargetConstant(*cast<ConstantInt>(I.getArgOperand(1)), sdl,
6593 MVT::i32); // arg index
6594 SDValue Res = DAG.getNode(
6595 ISD::PREALLOCATED_ARG, sdl,
6596 DAG.getVTList(TLI.getPointerTy(DAG.getDataLayout()), MVT::Other), Ops);
6597 setValue(V: &I, NewN: Res);
6598 DAG.setRoot(Res.getValue(R: 1));
6599 return;
6600 }
6601 case Intrinsic::dbg_declare: {
6602 const auto &DI = cast<DbgDeclareInst>(Val: I);
6603 // Debug intrinsics are handled separately in assignment tracking mode.
6604 // Some intrinsics are handled right after Argument lowering.
6605 if (AssignmentTrackingEnabled ||
6606 FuncInfo.PreprocessedDbgDeclares.count(Ptr: &DI))
6607 return;
6608 LLVM_DEBUG(dbgs() << "SelectionDAG visiting dbg_declare: " << DI << "\n");
6609 DILocalVariable *Variable = DI.getVariable();
6610 DIExpression *Expression = DI.getExpression();
6611 dropDanglingDebugInfo(Variable, Expr: Expression);
6612 // Assume dbg.declare can not currently use DIArgList, i.e.
6613 // it is non-variadic.
6614 assert(!DI.hasArgList() && "Only dbg.value should currently use DIArgList");
6615 handleDebugDeclare(Address: DI.getVariableLocationOp(OpIdx: 0), Variable, Expression,
6616 DL: DI.getDebugLoc());
6617 return;
6618 }
6619 case Intrinsic::dbg_label: {
6620 const DbgLabelInst &DI = cast<DbgLabelInst>(Val: I);
6621 DILabel *Label = DI.getLabel();
6622 assert(Label && "Missing label");
6623
6624 SDDbgLabel *SDV;
6625 SDV = DAG.getDbgLabel(Label, DL: dl, O: SDNodeOrder);
6626 DAG.AddDbgLabel(DB: SDV);
6627 return;
6628 }
6629 case Intrinsic::dbg_assign: {
6630 // Debug intrinsics are handled separately in assignment tracking mode.
6631 if (AssignmentTrackingEnabled)
6632 return;
6633 // If assignment tracking hasn't been enabled then fall through and treat
6634 // the dbg.assign as a dbg.value.
6635 [[fallthrough]];
6636 }
6637 case Intrinsic::dbg_value: {
6638 // Debug intrinsics are handled separately in assignment tracking mode.
6639 if (AssignmentTrackingEnabled)
6640 return;
6641 const DbgValueInst &DI = cast<DbgValueInst>(Val: I);
6642 assert(DI.getVariable() && "Missing variable");
6643
6644 DILocalVariable *Variable = DI.getVariable();
6645 DIExpression *Expression = DI.getExpression();
6646 dropDanglingDebugInfo(Variable, Expr: Expression);
6647
6648 if (DI.isKillLocation()) {
6649 handleKillDebugValue(Var: Variable, Expr: Expression, DbgLoc: DI.getDebugLoc(), Order: SDNodeOrder);
6650 return;
6651 }
6652
6653 SmallVector<Value *, 4> Values(DI.getValues());
6654 if (Values.empty())
6655 return;
6656
6657 bool IsVariadic = DI.hasArgList();
6658 if (!handleDebugValue(Values, Var: Variable, Expr: Expression, DbgLoc: DI.getDebugLoc(),
6659 Order: SDNodeOrder, IsVariadic))
6660 addDanglingDebugInfo(Values, Var: Variable, Expr: Expression, IsVariadic,
6661 DL: DI.getDebugLoc(), Order: SDNodeOrder);
6662 return;
6663 }
6664
6665 case Intrinsic::eh_typeid_for: {
6666 // Find the type id for the given typeinfo.
6667 GlobalValue *GV = ExtractTypeInfo(V: I.getArgOperand(i: 0));
6668 unsigned TypeID = DAG.getMachineFunction().getTypeIDFor(TI: GV);
6669 Res = DAG.getConstant(TypeID, sdl, MVT::i32);
6670 setValue(V: &I, NewN: Res);
6671 return;
6672 }
6673
6674 case Intrinsic::eh_return_i32:
6675 case Intrinsic::eh_return_i64:
6676 DAG.getMachineFunction().setCallsEHReturn(true);
6677 DAG.setRoot(DAG.getNode(ISD::EH_RETURN, sdl,
6678 MVT::Other,
6679 getControlRoot(),
6680 getValue(I.getArgOperand(0)),
6681 getValue(I.getArgOperand(1))));
6682 return;
6683 case Intrinsic::eh_unwind_init:
6684 DAG.getMachineFunction().setCallsUnwindInit(true);
6685 return;
6686 case Intrinsic::eh_dwarf_cfa:
6687 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::EH_DWARF_CFA, DL: sdl,
6688 VT: TLI.getPointerTy(DL: DAG.getDataLayout()),
6689 Operand: getValue(V: I.getArgOperand(i: 0))));
6690 return;
6691 case Intrinsic::eh_sjlj_callsite: {
6692 ConstantInt *CI = cast<ConstantInt>(Val: I.getArgOperand(i: 0));
6693 assert(FuncInfo.getCurrentCallSite() == 0 && "Overlapping call sites!");
6694
6695 FuncInfo.setCurrentCallSite(CI->getZExtValue());
6696 return;
6697 }
6698 case Intrinsic::eh_sjlj_functioncontext: {
6699 // Get and store the index of the function context.
6700 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
6701 AllocaInst *FnCtx =
6702 cast<AllocaInst>(Val: I.getArgOperand(i: 0)->stripPointerCasts());
6703 int FI = FuncInfo.StaticAllocaMap[FnCtx];
6704 MFI.setFunctionContextIndex(FI);
6705 return;
6706 }
6707 case Intrinsic::eh_sjlj_setjmp: {
6708 SDValue Ops[2];
6709 Ops[0] = getRoot();
6710 Ops[1] = getValue(V: I.getArgOperand(i: 0));
6711 SDValue Op = DAG.getNode(ISD::EH_SJLJ_SETJMP, sdl,
6712 DAG.getVTList(MVT::i32, MVT::Other), Ops);
6713 setValue(V: &I, NewN: Op.getValue(R: 0));
6714 DAG.setRoot(Op.getValue(R: 1));
6715 return;
6716 }
6717 case Intrinsic::eh_sjlj_longjmp:
6718 DAG.setRoot(DAG.getNode(ISD::EH_SJLJ_LONGJMP, sdl, MVT::Other,
6719 getRoot(), getValue(I.getArgOperand(0))));
6720 return;
6721 case Intrinsic::eh_sjlj_setup_dispatch:
6722 DAG.setRoot(DAG.getNode(ISD::EH_SJLJ_SETUP_DISPATCH, sdl, MVT::Other,
6723 getRoot()));
6724 return;
6725 case Intrinsic::masked_gather:
6726 visitMaskedGather(I);
6727 return;
6728 case Intrinsic::masked_load:
6729 visitMaskedLoad(I);
6730 return;
6731 case Intrinsic::masked_scatter:
6732 visitMaskedScatter(I);
6733 return;
6734 case Intrinsic::masked_store:
6735 visitMaskedStore(I);
6736 return;
6737 case Intrinsic::masked_expandload:
6738 visitMaskedLoad(I, IsExpanding: true /* IsExpanding */);
6739 return;
6740 case Intrinsic::masked_compressstore:
6741 visitMaskedStore(I, IsCompressing: true /* IsCompressing */);
6742 return;
6743 case Intrinsic::powi:
6744 setValue(V: &I, NewN: ExpandPowI(DL: sdl, LHS: getValue(V: I.getArgOperand(i: 0)),
6745 RHS: getValue(V: I.getArgOperand(i: 1)), DAG));
6746 return;
6747 case Intrinsic::log:
6748 setValue(V: &I, NewN: expandLog(dl: sdl, Op: getValue(V: I.getArgOperand(i: 0)), DAG, TLI, Flags));
6749 return;
6750 case Intrinsic::log2:
6751 setValue(V: &I,
6752 NewN: expandLog2(dl: sdl, Op: getValue(V: I.getArgOperand(i: 0)), DAG, TLI, Flags));
6753 return;
6754 case Intrinsic::log10:
6755 setValue(V: &I,
6756 NewN: expandLog10(dl: sdl, Op: getValue(V: I.getArgOperand(i: 0)), DAG, TLI, Flags));
6757 return;
6758 case Intrinsic::exp:
6759 setValue(V: &I, NewN: expandExp(dl: sdl, Op: getValue(V: I.getArgOperand(i: 0)), DAG, TLI, Flags));
6760 return;
6761 case Intrinsic::exp2:
6762 setValue(V: &I,
6763 NewN: expandExp2(dl: sdl, Op: getValue(V: I.getArgOperand(i: 0)), DAG, TLI, Flags));
6764 return;
6765 case Intrinsic::pow:
6766 setValue(V: &I, NewN: expandPow(dl: sdl, LHS: getValue(V: I.getArgOperand(i: 0)),
6767 RHS: getValue(V: I.getArgOperand(i: 1)), DAG, TLI, Flags));
6768 return;
6769 case Intrinsic::sqrt:
6770 case Intrinsic::fabs:
6771 case Intrinsic::sin:
6772 case Intrinsic::cos:
6773 case Intrinsic::tan:
6774 case Intrinsic::asin:
6775 case Intrinsic::acos:
6776 case Intrinsic::atan:
6777 case Intrinsic::sinh:
6778 case Intrinsic::cosh:
6779 case Intrinsic::tanh:
6780 case Intrinsic::exp10:
6781 case Intrinsic::floor:
6782 case Intrinsic::ceil:
6783 case Intrinsic::trunc:
6784 case Intrinsic::rint:
6785 case Intrinsic::nearbyint:
6786 case Intrinsic::round:
6787 case Intrinsic::roundeven:
6788 case Intrinsic::canonicalize: {
6789 unsigned Opcode;
6790 // clang-format off
6791 switch (Intrinsic) {
6792 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
6793 case Intrinsic::sqrt: Opcode = ISD::FSQRT; break;
6794 case Intrinsic::fabs: Opcode = ISD::FABS; break;
6795 case Intrinsic::sin: Opcode = ISD::FSIN; break;
6796 case Intrinsic::cos: Opcode = ISD::FCOS; break;
6797 case Intrinsic::tan: Opcode = ISD::FTAN; break;
6798 case Intrinsic::asin: Opcode = ISD::FASIN; break;
6799 case Intrinsic::acos: Opcode = ISD::FACOS; break;
6800 case Intrinsic::atan: Opcode = ISD::FATAN; break;
6801 case Intrinsic::sinh: Opcode = ISD::FSINH; break;
6802 case Intrinsic::cosh: Opcode = ISD::FCOSH; break;
6803 case Intrinsic::tanh: Opcode = ISD::FTANH; break;
6804 case Intrinsic::exp10: Opcode = ISD::FEXP10; break;
6805 case Intrinsic::floor: Opcode = ISD::FFLOOR; break;
6806 case Intrinsic::ceil: Opcode = ISD::FCEIL; break;
6807 case Intrinsic::trunc: Opcode = ISD::FTRUNC; break;
6808 case Intrinsic::rint: Opcode = ISD::FRINT; break;
6809 case Intrinsic::nearbyint: Opcode = ISD::FNEARBYINT; break;
6810 case Intrinsic::round: Opcode = ISD::FROUND; break;
6811 case Intrinsic::roundeven: Opcode = ISD::FROUNDEVEN; break;
6812 case Intrinsic::canonicalize: Opcode = ISD::FCANONICALIZE; break;
6813 }
6814 // clang-format on
6815
6816 setValue(V: &I, NewN: DAG.getNode(Opcode, DL: sdl,
6817 VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
6818 Operand: getValue(V: I.getArgOperand(i: 0)), Flags));
6819 return;
6820 }
6821 case Intrinsic::atan2:
6822 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FATAN2, DL: sdl,
6823 VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
6824 N1: getValue(V: I.getArgOperand(i: 0)),
6825 N2: getValue(V: I.getArgOperand(i: 1)), Flags));
6826 return;
6827 case Intrinsic::lround:
6828 case Intrinsic::llround:
6829 case Intrinsic::lrint:
6830 case Intrinsic::llrint: {
6831 unsigned Opcode;
6832 // clang-format off
6833 switch (Intrinsic) {
6834 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
6835 case Intrinsic::lround: Opcode = ISD::LROUND; break;
6836 case Intrinsic::llround: Opcode = ISD::LLROUND; break;
6837 case Intrinsic::lrint: Opcode = ISD::LRINT; break;
6838 case Intrinsic::llrint: Opcode = ISD::LLRINT; break;
6839 }
6840 // clang-format on
6841
6842 EVT RetVT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
6843 setValue(V: &I, NewN: DAG.getNode(Opcode, DL: sdl, VT: RetVT,
6844 Operand: getValue(V: I.getArgOperand(i: 0))));
6845 return;
6846 }
6847 case Intrinsic::minnum:
6848 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FMINNUM, DL: sdl,
6849 VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
6850 N1: getValue(V: I.getArgOperand(i: 0)),
6851 N2: getValue(V: I.getArgOperand(i: 1)), Flags));
6852 return;
6853 case Intrinsic::maxnum:
6854 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FMAXNUM, DL: sdl,
6855 VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
6856 N1: getValue(V: I.getArgOperand(i: 0)),
6857 N2: getValue(V: I.getArgOperand(i: 1)), Flags));
6858 return;
6859 case Intrinsic::minimum:
6860 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FMINIMUM, DL: sdl,
6861 VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
6862 N1: getValue(V: I.getArgOperand(i: 0)),
6863 N2: getValue(V: I.getArgOperand(i: 1)), Flags));
6864 return;
6865 case Intrinsic::maximum:
6866 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FMAXIMUM, DL: sdl,
6867 VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
6868 N1: getValue(V: I.getArgOperand(i: 0)),
6869 N2: getValue(V: I.getArgOperand(i: 1)), Flags));
6870 return;
6871 case Intrinsic::minimumnum:
6872 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FMINIMUMNUM, DL: sdl,
6873 VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
6874 N1: getValue(V: I.getArgOperand(i: 0)),
6875 N2: getValue(V: I.getArgOperand(i: 1)), Flags));
6876 return;
6877 case Intrinsic::maximumnum:
6878 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FMAXIMUMNUM, DL: sdl,
6879 VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
6880 N1: getValue(V: I.getArgOperand(i: 0)),
6881 N2: getValue(V: I.getArgOperand(i: 1)), Flags));
6882 return;
6883 case Intrinsic::copysign:
6884 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FCOPYSIGN, DL: sdl,
6885 VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
6886 N1: getValue(V: I.getArgOperand(i: 0)),
6887 N2: getValue(V: I.getArgOperand(i: 1)), Flags));
6888 return;
6889 case Intrinsic::ldexp:
6890 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FLDEXP, DL: sdl,
6891 VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
6892 N1: getValue(V: I.getArgOperand(i: 0)),
6893 N2: getValue(V: I.getArgOperand(i: 1)), Flags));
6894 return;
6895 case Intrinsic::modf:
6896 case Intrinsic::sincos:
6897 case Intrinsic::sincospi:
6898 case Intrinsic::frexp: {
6899 unsigned Opcode;
6900 switch (Intrinsic) {
6901 default:
6902 llvm_unreachable("unexpected intrinsic");
6903 case Intrinsic::sincos:
6904 Opcode = ISD::FSINCOS;
6905 break;
6906 case Intrinsic::sincospi:
6907 Opcode = ISD::FSINCOSPI;
6908 break;
6909 case Intrinsic::modf:
6910 Opcode = ISD::FMODF;
6911 break;
6912 case Intrinsic::frexp:
6913 Opcode = ISD::FFREXP;
6914 break;
6915 }
6916 SmallVector<EVT, 2> ValueVTs;
6917 ComputeValueVTs(TLI, DL: DAG.getDataLayout(), Ty: I.getType(), ValueVTs);
6918 SDVTList VTs = DAG.getVTList(VTs: ValueVTs);
6919 setValue(
6920 V: &I, NewN: DAG.getNode(Opcode, DL: sdl, VTList: VTs, Ops: getValue(V: I.getArgOperand(i: 0)), Flags));
6921 return;
6922 }
6923 case Intrinsic::arithmetic_fence: {
6924 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::ARITH_FENCE, DL: sdl,
6925 VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
6926 Operand: getValue(V: I.getArgOperand(i: 0)), Flags));
6927 return;
6928 }
6929 case Intrinsic::fma:
6930 setValue(V: &I, NewN: DAG.getNode(
6931 Opcode: ISD::FMA, DL: sdl, VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
6932 N1: getValue(V: I.getArgOperand(i: 0)), N2: getValue(V: I.getArgOperand(i: 1)),
6933 N3: getValue(V: I.getArgOperand(i: 2)), Flags));
6934 return;
6935#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \
6936 case Intrinsic::INTRINSIC:
6937#include "llvm/IR/ConstrainedOps.def"
6938 visitConstrainedFPIntrinsic(FPI: cast<ConstrainedFPIntrinsic>(Val: I));
6939 return;
6940#define BEGIN_REGISTER_VP_INTRINSIC(VPID, ...) case Intrinsic::VPID:
6941#include "llvm/IR/VPIntrinsics.def"
6942 visitVectorPredicationIntrinsic(VPIntrin: cast<VPIntrinsic>(Val: I));
6943 return;
6944 case Intrinsic::fptrunc_round: {
6945 // Get the last argument, the metadata and convert it to an integer in the
6946 // call
6947 Metadata *MD = cast<MetadataAsValue>(Val: I.getArgOperand(i: 1))->getMetadata();
6948 std::optional<RoundingMode> RoundMode =
6949 convertStrToRoundingMode(cast<MDString>(Val: MD)->getString());
6950
6951 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
6952
6953 // Propagate fast-math-flags from IR to node(s).
6954 SDNodeFlags Flags;
6955 Flags.copyFMF(FPMO: *cast<FPMathOperator>(Val: &I));
6956 SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);
6957
6958 SDValue Result;
6959 Result = DAG.getNode(
6960 ISD::FPTRUNC_ROUND, sdl, VT, getValue(I.getArgOperand(0)),
6961 DAG.getTargetConstant((int)*RoundMode, sdl, MVT::i32));
6962 setValue(V: &I, NewN: Result);
6963
6964 return;
6965 }
6966 case Intrinsic::fmuladd: {
6967 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
6968 if (TM.Options.AllowFPOpFusion != FPOpFusion::Strict &&
6969 TLI.isFMAFasterThanFMulAndFAdd(MF: DAG.getMachineFunction(), VT)) {
6970 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FMA, DL: sdl,
6971 VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
6972 N1: getValue(V: I.getArgOperand(i: 0)),
6973 N2: getValue(V: I.getArgOperand(i: 1)),
6974 N3: getValue(V: I.getArgOperand(i: 2)), Flags));
6975 } else {
6976 // TODO: Intrinsic calls should have fast-math-flags.
6977 SDValue Mul = DAG.getNode(
6978 Opcode: ISD::FMUL, DL: sdl, VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
6979 N1: getValue(V: I.getArgOperand(i: 0)), N2: getValue(V: I.getArgOperand(i: 1)), Flags);
6980 SDValue Add = DAG.getNode(Opcode: ISD::FADD, DL: sdl,
6981 VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
6982 N1: Mul, N2: getValue(V: I.getArgOperand(i: 2)), Flags);
6983 setValue(V: &I, NewN: Add);
6984 }
6985 return;
6986 }
6987 case Intrinsic::convert_to_fp16:
6988 setValue(&I, DAG.getNode(ISD::BITCAST, sdl, MVT::i16,
6989 DAG.getNode(ISD::FP_ROUND, sdl, MVT::f16,
6990 getValue(I.getArgOperand(0)),
6991 DAG.getTargetConstant(0, sdl,
6992 MVT::i32))));
6993 return;
6994 case Intrinsic::convert_from_fp16:
6995 setValue(&I, DAG.getNode(ISD::FP_EXTEND, sdl,
6996 TLI.getValueType(DAG.getDataLayout(), I.getType()),
6997 DAG.getNode(ISD::BITCAST, sdl, MVT::f16,
6998 getValue(I.getArgOperand(0)))));
6999 return;
7000 case Intrinsic::fptosi_sat: {
7001 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
7002 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FP_TO_SINT_SAT, DL: sdl, VT,
7003 N1: getValue(V: I.getArgOperand(i: 0)),
7004 N2: DAG.getValueType(VT.getScalarType())));
7005 return;
7006 }
7007 case Intrinsic::fptoui_sat: {
7008 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
7009 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FP_TO_UINT_SAT, DL: sdl, VT,
7010 N1: getValue(V: I.getArgOperand(i: 0)),
7011 N2: DAG.getValueType(VT.getScalarType())));
7012 return;
7013 }
7014 case Intrinsic::set_rounding:
7015 Res = DAG.getNode(ISD::SET_ROUNDING, sdl, MVT::Other,
7016 {getRoot(), getValue(I.getArgOperand(0))});
7017 setValue(V: &I, NewN: Res);
7018 DAG.setRoot(Res.getValue(R: 0));
7019 return;
7020 case Intrinsic::is_fpclass: {
7021 const DataLayout DLayout = DAG.getDataLayout();
7022 EVT DestVT = TLI.getValueType(DL: DLayout, Ty: I.getType());
7023 EVT ArgVT = TLI.getValueType(DL: DLayout, Ty: I.getArgOperand(i: 0)->getType());
7024 FPClassTest Test = static_cast<FPClassTest>(
7025 cast<ConstantInt>(Val: I.getArgOperand(i: 1))->getZExtValue());
7026 MachineFunction &MF = DAG.getMachineFunction();
7027 const Function &F = MF.getFunction();
7028 SDValue Op = getValue(V: I.getArgOperand(i: 0));
7029 SDNodeFlags Flags;
7030 Flags.setNoFPExcept(
7031 !F.getAttributes().hasFnAttr(llvm::Attribute::StrictFP));
7032 // If ISD::IS_FPCLASS should be expanded, do it right now, because the
7033 // expansion can use illegal types. Making expansion early allows
7034 // legalizing these types prior to selection.
7035 if (!TLI.isOperationLegal(Op: ISD::IS_FPCLASS, VT: ArgVT) &&
7036 !TLI.isOperationCustom(Op: ISD::IS_FPCLASS, VT: ArgVT)) {
7037 SDValue Result = TLI.expandIS_FPCLASS(ResultVT: DestVT, Op, Test, Flags, DL: sdl, DAG);
7038 setValue(V: &I, NewN: Result);
7039 return;
7040 }
7041
7042 SDValue Check = DAG.getTargetConstant(Test, sdl, MVT::i32);
7043 SDValue V = DAG.getNode(Opcode: ISD::IS_FPCLASS, DL: sdl, VT: DestVT, Ops: {Op, Check}, Flags);
7044 setValue(V: &I, NewN: V);
7045 return;
7046 }
7047 case Intrinsic::get_fpenv: {
7048 const DataLayout DLayout = DAG.getDataLayout();
7049 EVT EnvVT = TLI.getValueType(DL: DLayout, Ty: I.getType());
7050 Align TempAlign = DAG.getEVTAlign(MemoryVT: EnvVT);
7051 SDValue Chain = getRoot();
7052 // Use GET_FPENV if it is legal or custom. Otherwise use memory-based node
7053 // and temporary storage in stack.
7054 if (TLI.isOperationLegalOrCustom(Op: ISD::GET_FPENV, VT: EnvVT)) {
7055 Res = DAG.getNode(
7056 ISD::GET_FPENV, sdl,
7057 DAG.getVTList(TLI.getValueType(DAG.getDataLayout(), I.getType()),
7058 MVT::Other),
7059 Chain);
7060 } else {
7061 SDValue Temp = DAG.CreateStackTemporary(VT: EnvVT, minAlign: TempAlign.value());
7062 int SPFI = cast<FrameIndexSDNode>(Val: Temp.getNode())->getIndex();
7063 auto MPI =
7064 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: SPFI);
7065 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
7066 PtrInfo: MPI, F: MachineMemOperand::MOStore, Size: LocationSize::beforeOrAfterPointer(),
7067 BaseAlignment: TempAlign);
7068 Chain = DAG.getGetFPEnv(Chain, dl: sdl, Ptr: Temp, MemVT: EnvVT, MMO);
7069 Res = DAG.getLoad(VT: EnvVT, dl: sdl, Chain, Ptr: Temp, PtrInfo: MPI);
7070 }
7071 setValue(V: &I, NewN: Res);
7072 DAG.setRoot(Res.getValue(R: 1));
7073 return;
7074 }
7075 case Intrinsic::set_fpenv: {
7076 const DataLayout DLayout = DAG.getDataLayout();
7077 SDValue Env = getValue(V: I.getArgOperand(i: 0));
7078 EVT EnvVT = Env.getValueType();
7079 Align TempAlign = DAG.getEVTAlign(MemoryVT: EnvVT);
7080 SDValue Chain = getRoot();
7081 // If SET_FPENV is custom or legal, use it. Otherwise use loading
7082 // environment from memory.
7083 if (TLI.isOperationLegalOrCustom(Op: ISD::SET_FPENV, VT: EnvVT)) {
7084 Chain = DAG.getNode(ISD::SET_FPENV, sdl, MVT::Other, Chain, Env);
7085 } else {
7086 // Allocate space in stack, copy environment bits into it and use this
7087 // memory in SET_FPENV_MEM.
7088 SDValue Temp = DAG.CreateStackTemporary(VT: EnvVT, minAlign: TempAlign.value());
7089 int SPFI = cast<FrameIndexSDNode>(Val: Temp.getNode())->getIndex();
7090 auto MPI =
7091 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: SPFI);
7092 Chain = DAG.getStore(Chain, dl: sdl, Val: Env, Ptr: Temp, PtrInfo: MPI, Alignment: TempAlign,
7093 MMOFlags: MachineMemOperand::MOStore);
7094 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
7095 PtrInfo: MPI, F: MachineMemOperand::MOLoad, Size: LocationSize::beforeOrAfterPointer(),
7096 BaseAlignment: TempAlign);
7097 Chain = DAG.getSetFPEnv(Chain, dl: sdl, Ptr: Temp, MemVT: EnvVT, MMO);
7098 }
7099 DAG.setRoot(Chain);
7100 return;
7101 }
7102 case Intrinsic::reset_fpenv:
7103 DAG.setRoot(DAG.getNode(ISD::RESET_FPENV, sdl, MVT::Other, getRoot()));
7104 return;
7105 case Intrinsic::get_fpmode:
7106 Res = DAG.getNode(
7107 ISD::GET_FPMODE, sdl,
7108 DAG.getVTList(TLI.getValueType(DAG.getDataLayout(), I.getType()),
7109 MVT::Other),
7110 DAG.getRoot());
7111 setValue(V: &I, NewN: Res);
7112 DAG.setRoot(Res.getValue(R: 1));
7113 return;
7114 case Intrinsic::set_fpmode:
7115 Res = DAG.getNode(ISD::SET_FPMODE, sdl, MVT::Other, {DAG.getRoot()},
7116 getValue(I.getArgOperand(0)));
7117 DAG.setRoot(Res);
7118 return;
7119 case Intrinsic::reset_fpmode: {
7120 Res = DAG.getNode(ISD::RESET_FPMODE, sdl, MVT::Other, getRoot());
7121 DAG.setRoot(Res);
7122 return;
7123 }
7124 case Intrinsic::pcmarker: {
7125 SDValue Tmp = getValue(V: I.getArgOperand(i: 0));
7126 DAG.setRoot(DAG.getNode(ISD::PCMARKER, sdl, MVT::Other, getRoot(), Tmp));
7127 return;
7128 }
7129 case Intrinsic::readcyclecounter: {
7130 SDValue Op = getRoot();
7131 Res = DAG.getNode(ISD::READCYCLECOUNTER, sdl,
7132 DAG.getVTList(MVT::i64, MVT::Other), Op);
7133 setValue(V: &I, NewN: Res);
7134 DAG.setRoot(Res.getValue(R: 1));
7135 return;
7136 }
7137 case Intrinsic::readsteadycounter: {
7138 SDValue Op = getRoot();
7139 Res = DAG.getNode(ISD::READSTEADYCOUNTER, sdl,
7140 DAG.getVTList(MVT::i64, MVT::Other), Op);
7141 setValue(V: &I, NewN: Res);
7142 DAG.setRoot(Res.getValue(R: 1));
7143 return;
7144 }
7145 case Intrinsic::bitreverse:
7146 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::BITREVERSE, DL: sdl,
7147 VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
7148 Operand: getValue(V: I.getArgOperand(i: 0))));
7149 return;
7150 case Intrinsic::bswap:
7151 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::BSWAP, DL: sdl,
7152 VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
7153 Operand: getValue(V: I.getArgOperand(i: 0))));
7154 return;
7155 case Intrinsic::cttz: {
7156 SDValue Arg = getValue(V: I.getArgOperand(i: 0));
7157 ConstantInt *CI = cast<ConstantInt>(Val: I.getArgOperand(i: 1));
7158 EVT Ty = Arg.getValueType();
7159 setValue(V: &I, NewN: DAG.getNode(Opcode: CI->isZero() ? ISD::CTTZ : ISD::CTTZ_ZERO_UNDEF,
7160 DL: sdl, VT: Ty, Operand: Arg));
7161 return;
7162 }
7163 case Intrinsic::ctlz: {
7164 SDValue Arg = getValue(V: I.getArgOperand(i: 0));
7165 ConstantInt *CI = cast<ConstantInt>(Val: I.getArgOperand(i: 1));
7166 EVT Ty = Arg.getValueType();
7167 setValue(V: &I, NewN: DAG.getNode(Opcode: CI->isZero() ? ISD::CTLZ : ISD::CTLZ_ZERO_UNDEF,
7168 DL: sdl, VT: Ty, Operand: Arg));
7169 return;
7170 }
7171 case Intrinsic::ctpop: {
7172 SDValue Arg = getValue(V: I.getArgOperand(i: 0));
7173 EVT Ty = Arg.getValueType();
7174 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::CTPOP, DL: sdl, VT: Ty, Operand: Arg));
7175 return;
7176 }
7177 case Intrinsic::fshl:
7178 case Intrinsic::fshr: {
7179 bool IsFSHL = Intrinsic == Intrinsic::fshl;
7180 SDValue X = getValue(V: I.getArgOperand(i: 0));
7181 SDValue Y = getValue(V: I.getArgOperand(i: 1));
7182 SDValue Z = getValue(V: I.getArgOperand(i: 2));
7183 EVT VT = X.getValueType();
7184
7185 if (X == Y) {
7186 auto RotateOpcode = IsFSHL ? ISD::ROTL : ISD::ROTR;
7187 setValue(V: &I, NewN: DAG.getNode(Opcode: RotateOpcode, DL: sdl, VT, N1: X, N2: Z));
7188 } else {
7189 auto FunnelOpcode = IsFSHL ? ISD::FSHL : ISD::FSHR;
7190 setValue(V: &I, NewN: DAG.getNode(Opcode: FunnelOpcode, DL: sdl, VT, N1: X, N2: Y, N3: Z));
7191 }
7192 return;
7193 }
7194 case Intrinsic::sadd_sat: {
7195 SDValue Op1 = getValue(V: I.getArgOperand(i: 0));
7196 SDValue Op2 = getValue(V: I.getArgOperand(i: 1));
7197 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::SADDSAT, DL: sdl, VT: Op1.getValueType(), N1: Op1, N2: Op2));
7198 return;
7199 }
7200 case Intrinsic::uadd_sat: {
7201 SDValue Op1 = getValue(V: I.getArgOperand(i: 0));
7202 SDValue Op2 = getValue(V: I.getArgOperand(i: 1));
7203 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::UADDSAT, DL: sdl, VT: Op1.getValueType(), N1: Op1, N2: Op2));
7204 return;
7205 }
7206 case Intrinsic::ssub_sat: {
7207 SDValue Op1 = getValue(V: I.getArgOperand(i: 0));
7208 SDValue Op2 = getValue(V: I.getArgOperand(i: 1));
7209 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::SSUBSAT, DL: sdl, VT: Op1.getValueType(), N1: Op1, N2: Op2));
7210 return;
7211 }
7212 case Intrinsic::usub_sat: {
7213 SDValue Op1 = getValue(V: I.getArgOperand(i: 0));
7214 SDValue Op2 = getValue(V: I.getArgOperand(i: 1));
7215 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::USUBSAT, DL: sdl, VT: Op1.getValueType(), N1: Op1, N2: Op2));
7216 return;
7217 }
7218 case Intrinsic::sshl_sat: {
7219 SDValue Op1 = getValue(V: I.getArgOperand(i: 0));
7220 SDValue Op2 = getValue(V: I.getArgOperand(i: 1));
7221 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::SSHLSAT, DL: sdl, VT: Op1.getValueType(), N1: Op1, N2: Op2));
7222 return;
7223 }
7224 case Intrinsic::ushl_sat: {
7225 SDValue Op1 = getValue(V: I.getArgOperand(i: 0));
7226 SDValue Op2 = getValue(V: I.getArgOperand(i: 1));
7227 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::USHLSAT, DL: sdl, VT: Op1.getValueType(), N1: Op1, N2: Op2));
7228 return;
7229 }
7230 case Intrinsic::smul_fix:
7231 case Intrinsic::umul_fix:
7232 case Intrinsic::smul_fix_sat:
7233 case Intrinsic::umul_fix_sat: {
7234 SDValue Op1 = getValue(V: I.getArgOperand(i: 0));
7235 SDValue Op2 = getValue(V: I.getArgOperand(i: 1));
7236 SDValue Op3 = getValue(V: I.getArgOperand(i: 2));
7237 setValue(V: &I, NewN: DAG.getNode(Opcode: FixedPointIntrinsicToOpcode(Intrinsic), DL: sdl,
7238 VT: Op1.getValueType(), N1: Op1, N2: Op2, N3: Op3));
7239 return;
7240 }
7241 case Intrinsic::sdiv_fix:
7242 case Intrinsic::udiv_fix:
7243 case Intrinsic::sdiv_fix_sat:
7244 case Intrinsic::udiv_fix_sat: {
7245 SDValue Op1 = getValue(V: I.getArgOperand(i: 0));
7246 SDValue Op2 = getValue(V: I.getArgOperand(i: 1));
7247 SDValue Op3 = getValue(V: I.getArgOperand(i: 2));
7248 setValue(V: &I, NewN: expandDivFix(Opcode: FixedPointIntrinsicToOpcode(Intrinsic), DL: sdl,
7249 LHS: Op1, RHS: Op2, Scale: Op3, DAG, TLI));
7250 return;
7251 }
7252 case Intrinsic::smax: {
7253 SDValue Op1 = getValue(V: I.getArgOperand(i: 0));
7254 SDValue Op2 = getValue(V: I.getArgOperand(i: 1));
7255 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::SMAX, DL: sdl, VT: Op1.getValueType(), N1: Op1, N2: Op2));
7256 return;
7257 }
7258 case Intrinsic::smin: {
7259 SDValue Op1 = getValue(V: I.getArgOperand(i: 0));
7260 SDValue Op2 = getValue(V: I.getArgOperand(i: 1));
7261 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::SMIN, DL: sdl, VT: Op1.getValueType(), N1: Op1, N2: Op2));
7262 return;
7263 }
7264 case Intrinsic::umax: {
7265 SDValue Op1 = getValue(V: I.getArgOperand(i: 0));
7266 SDValue Op2 = getValue(V: I.getArgOperand(i: 1));
7267 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::UMAX, DL: sdl, VT: Op1.getValueType(), N1: Op1, N2: Op2));
7268 return;
7269 }
7270 case Intrinsic::umin: {
7271 SDValue Op1 = getValue(V: I.getArgOperand(i: 0));
7272 SDValue Op2 = getValue(V: I.getArgOperand(i: 1));
7273 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::UMIN, DL: sdl, VT: Op1.getValueType(), N1: Op1, N2: Op2));
7274 return;
7275 }
7276 case Intrinsic::abs: {
7277 // TODO: Preserve "int min is poison" arg in SDAG?
7278 SDValue Op1 = getValue(V: I.getArgOperand(i: 0));
7279 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::ABS, DL: sdl, VT: Op1.getValueType(), Operand: Op1));
7280 return;
7281 }
7282 case Intrinsic::scmp: {
7283 SDValue Op1 = getValue(V: I.getArgOperand(i: 0));
7284 SDValue Op2 = getValue(V: I.getArgOperand(i: 1));
7285 EVT DestVT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
7286 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::SCMP, DL: sdl, VT: DestVT, N1: Op1, N2: Op2));
7287 break;
7288 }
7289 case Intrinsic::ucmp: {
7290 SDValue Op1 = getValue(V: I.getArgOperand(i: 0));
7291 SDValue Op2 = getValue(V: I.getArgOperand(i: 1));
7292 EVT DestVT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
7293 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::UCMP, DL: sdl, VT: DestVT, N1: Op1, N2: Op2));
7294 break;
7295 }
7296 case Intrinsic::stacksave: {
7297 SDValue Op = getRoot();
7298 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
7299 Res = DAG.getNode(ISD::STACKSAVE, sdl, DAG.getVTList(VT, MVT::Other), Op);
7300 setValue(V: &I, NewN: Res);
7301 DAG.setRoot(Res.getValue(R: 1));
7302 return;
7303 }
7304 case Intrinsic::stackrestore:
7305 Res = getValue(V: I.getArgOperand(i: 0));
7306 DAG.setRoot(DAG.getNode(ISD::STACKRESTORE, sdl, MVT::Other, getRoot(), Res));
7307 return;
7308 case Intrinsic::get_dynamic_area_offset: {
7309 SDValue Op = getRoot();
7310 EVT PtrTy = TLI.getFrameIndexTy(DL: DAG.getDataLayout());
7311 EVT ResTy = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
7312 // Result type for @llvm.get.dynamic.area.offset should match PtrTy for
7313 // target.
7314 if (PtrTy.getFixedSizeInBits() < ResTy.getFixedSizeInBits())
7315 report_fatal_error(reason: "Wrong result type for @llvm.get.dynamic.area.offset"
7316 " intrinsic!");
7317 Res = DAG.getNode(Opcode: ISD::GET_DYNAMIC_AREA_OFFSET, DL: sdl, VTList: DAG.getVTList(VT: ResTy),
7318 N: Op);
7319 DAG.setRoot(Op);
7320 setValue(V: &I, NewN: Res);
7321 return;
7322 }
7323 case Intrinsic::stackguard: {
7324 MachineFunction &MF = DAG.getMachineFunction();
7325 const Module &M = *MF.getFunction().getParent();
7326 EVT PtrTy = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
7327 SDValue Chain = getRoot();
7328 if (TLI.useLoadStackGuardNode(M)) {
7329 Res = getLoadStackGuard(DAG, DL: sdl, Chain);
7330 Res = DAG.getPtrExtOrTrunc(Op: Res, DL: sdl, VT: PtrTy);
7331 } else {
7332 const Value *Global = TLI.getSDagStackGuard(M);
7333 Align Align = DAG.getDataLayout().getPrefTypeAlign(Ty: Global->getType());
7334 Res = DAG.getLoad(VT: PtrTy, dl: sdl, Chain, Ptr: getValue(V: Global),
7335 PtrInfo: MachinePointerInfo(Global, 0), Alignment: Align,
7336 MMOFlags: MachineMemOperand::MOVolatile);
7337 }
7338 if (TLI.useStackGuardXorFP())
7339 Res = TLI.emitStackGuardXorFP(DAG, Val: Res, DL: sdl);
7340 DAG.setRoot(Chain);
7341 setValue(V: &I, NewN: Res);
7342 return;
7343 }
7344 case Intrinsic::stackprotector: {
7345 // Emit code into the DAG to store the stack guard onto the stack.
7346 MachineFunction &MF = DAG.getMachineFunction();
7347 MachineFrameInfo &MFI = MF.getFrameInfo();
7348 const Module &M = *MF.getFunction().getParent();
7349 SDValue Src, Chain = getRoot();
7350
7351 if (TLI.useLoadStackGuardNode(M))
7352 Src = getLoadStackGuard(DAG, DL: sdl, Chain);
7353 else
7354 Src = getValue(V: I.getArgOperand(i: 0)); // The guard's value.
7355
7356 AllocaInst *Slot = cast<AllocaInst>(Val: I.getArgOperand(i: 1));
7357
7358 int FI = FuncInfo.StaticAllocaMap[Slot];
7359 MFI.setStackProtectorIndex(FI);
7360 EVT PtrTy = TLI.getFrameIndexTy(DL: DAG.getDataLayout());
7361
7362 SDValue FIN = DAG.getFrameIndex(FI, VT: PtrTy);
7363
7364 // Store the stack protector onto the stack.
7365 Res = DAG.getStore(
7366 Chain, dl: sdl, Val: Src, Ptr: FIN,
7367 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI),
7368 Alignment: MaybeAlign(), MMOFlags: MachineMemOperand::MOVolatile);
7369 setValue(V: &I, NewN: Res);
7370 DAG.setRoot(Res);
7371 return;
7372 }
7373 case Intrinsic::objectsize:
7374 llvm_unreachable("llvm.objectsize.* should have been lowered already");
7375
7376 case Intrinsic::is_constant:
7377 llvm_unreachable("llvm.is.constant.* should have been lowered already");
7378
7379 case Intrinsic::annotation:
7380 case Intrinsic::ptr_annotation:
7381 case Intrinsic::launder_invariant_group:
7382 case Intrinsic::strip_invariant_group:
7383 // Drop the intrinsic, but forward the value
7384 setValue(V: &I, NewN: getValue(V: I.getOperand(i_nocapture: 0)));
7385 return;
7386
7387 case Intrinsic::type_test:
7388 case Intrinsic::public_type_test:
7389 setValue(V: &I, NewN: getValue(V: ConstantInt::getTrue(Ty: I.getType())));
7390 return;
7391
7392 case Intrinsic::assume:
7393 case Intrinsic::experimental_noalias_scope_decl:
7394 case Intrinsic::var_annotation:
7395 case Intrinsic::sideeffect:
7396 // Discard annotate attributes, noalias scope declarations, assumptions, and
7397 // artificial side-effects.
7398 return;
7399
7400 case Intrinsic::codeview_annotation: {
7401 // Emit a label associated with this metadata.
7402 MachineFunction &MF = DAG.getMachineFunction();
7403 MCSymbol *Label = MF.getContext().createTempSymbol(Name: "annotation", AlwaysAddSuffix: true);
7404 Metadata *MD = cast<MetadataAsValue>(Val: I.getArgOperand(i: 0))->getMetadata();
7405 MF.addCodeViewAnnotation(Label, MD: cast<MDNode>(Val: MD));
7406 Res = DAG.getLabelNode(Opcode: ISD::ANNOTATION_LABEL, dl: sdl, Root: getRoot(), Label);
7407 DAG.setRoot(Res);
7408 return;
7409 }
7410
7411 case Intrinsic::init_trampoline: {
7412 const Function *F = cast<Function>(Val: I.getArgOperand(i: 1)->stripPointerCasts());
7413
7414 SDValue Ops[6];
7415 Ops[0] = getRoot();
7416 Ops[1] = getValue(V: I.getArgOperand(i: 0));
7417 Ops[2] = getValue(V: I.getArgOperand(i: 1));
7418 Ops[3] = getValue(V: I.getArgOperand(i: 2));
7419 Ops[4] = DAG.getSrcValue(v: I.getArgOperand(i: 0));
7420 Ops[5] = DAG.getSrcValue(v: F);
7421
7422 Res = DAG.getNode(ISD::INIT_TRAMPOLINE, sdl, MVT::Other, Ops);
7423
7424 DAG.setRoot(Res);
7425 return;
7426 }
7427 case Intrinsic::adjust_trampoline:
7428 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::ADJUST_TRAMPOLINE, DL: sdl,
7429 VT: TLI.getPointerTy(DL: DAG.getDataLayout()),
7430 Operand: getValue(V: I.getArgOperand(i: 0))));
7431 return;
7432 case Intrinsic::gcroot: {
7433 assert(DAG.getMachineFunction().getFunction().hasGC() &&
7434 "only valid in functions with gc specified, enforced by Verifier");
7435 assert(GFI && "implied by previous");
7436 const Value *Alloca = I.getArgOperand(i: 0)->stripPointerCasts();
7437 const Constant *TypeMap = cast<Constant>(Val: I.getArgOperand(i: 1));
7438
7439 FrameIndexSDNode *FI = cast<FrameIndexSDNode>(Val: getValue(V: Alloca).getNode());
7440 GFI->addStackRoot(Num: FI->getIndex(), Metadata: TypeMap);
7441 return;
7442 }
7443 case Intrinsic::gcread:
7444 case Intrinsic::gcwrite:
7445 llvm_unreachable("GC failed to lower gcread/gcwrite intrinsics!");
7446 case Intrinsic::get_rounding:
7447 Res = DAG.getNode(ISD::GET_ROUNDING, sdl, {MVT::i32, MVT::Other}, getRoot());
7448 setValue(V: &I, NewN: Res);
7449 DAG.setRoot(Res.getValue(R: 1));
7450 return;
7451
7452 case Intrinsic::expect:
7453 case Intrinsic::expect_with_probability:
7454 // Just replace __builtin_expect(exp, c) and
7455 // __builtin_expect_with_probability(exp, c, p) with EXP.
7456 setValue(V: &I, NewN: getValue(V: I.getArgOperand(i: 0)));
7457 return;
7458
7459 case Intrinsic::ubsantrap:
7460 case Intrinsic::debugtrap:
7461 case Intrinsic::trap: {
7462 StringRef TrapFuncName =
7463 I.getAttributes().getFnAttr(Kind: "trap-func-name").getValueAsString();
7464 if (TrapFuncName.empty()) {
7465 switch (Intrinsic) {
7466 case Intrinsic::trap:
7467 DAG.setRoot(DAG.getNode(ISD::TRAP, sdl, MVT::Other, getRoot()));
7468 break;
7469 case Intrinsic::debugtrap:
7470 DAG.setRoot(DAG.getNode(ISD::DEBUGTRAP, sdl, MVT::Other, getRoot()));
7471 break;
7472 case Intrinsic::ubsantrap:
7473 DAG.setRoot(DAG.getNode(
7474 ISD::UBSANTRAP, sdl, MVT::Other, getRoot(),
7475 DAG.getTargetConstant(
7476 cast<ConstantInt>(I.getArgOperand(0))->getZExtValue(), sdl,
7477 MVT::i32)));
7478 break;
7479 default: llvm_unreachable("unknown trap intrinsic");
7480 }
7481 DAG.addNoMergeSiteInfo(DAG.getRoot().getNode(),
7482 I.hasFnAttr(Attribute::NoMerge));
7483 return;
7484 }
7485 TargetLowering::ArgListTy Args;
7486 if (Intrinsic == Intrinsic::ubsantrap) {
7487 Args.push_back(x: TargetLoweringBase::ArgListEntry());
7488 Args[0].Val = I.getArgOperand(i: 0);
7489 Args[0].Node = getValue(V: Args[0].Val);
7490 Args[0].Ty = Args[0].Val->getType();
7491 }
7492
7493 TargetLowering::CallLoweringInfo CLI(DAG);
7494 CLI.setDebugLoc(sdl).setChain(getRoot()).setLibCallee(
7495 CC: CallingConv::C, ResultType: I.getType(),
7496 Target: DAG.getExternalSymbol(Sym: TrapFuncName.data(),
7497 VT: TLI.getPointerTy(DL: DAG.getDataLayout())),
7498 ArgsList: std::move(Args));
7499 CLI.NoMerge = I.hasFnAttr(Attribute::NoMerge);
7500 std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
7501 DAG.setRoot(Result.second);
7502 return;
7503 }
7504
7505 case Intrinsic::allow_runtime_check:
7506 case Intrinsic::allow_ubsan_check:
7507 setValue(V: &I, NewN: getValue(V: ConstantInt::getTrue(Ty: I.getType())));
7508 return;
7509
7510 case Intrinsic::uadd_with_overflow:
7511 case Intrinsic::sadd_with_overflow:
7512 case Intrinsic::usub_with_overflow:
7513 case Intrinsic::ssub_with_overflow:
7514 case Intrinsic::umul_with_overflow:
7515 case Intrinsic::smul_with_overflow: {
7516 ISD::NodeType Op;
7517 switch (Intrinsic) {
7518 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
7519 case Intrinsic::uadd_with_overflow: Op = ISD::UADDO; break;
7520 case Intrinsic::sadd_with_overflow: Op = ISD::SADDO; break;
7521 case Intrinsic::usub_with_overflow: Op = ISD::USUBO; break;
7522 case Intrinsic::ssub_with_overflow: Op = ISD::SSUBO; break;
7523 case Intrinsic::umul_with_overflow: Op = ISD::UMULO; break;
7524 case Intrinsic::smul_with_overflow: Op = ISD::SMULO; break;
7525 }
7526 SDValue Op1 = getValue(V: I.getArgOperand(i: 0));
7527 SDValue Op2 = getValue(V: I.getArgOperand(i: 1));
7528
7529 EVT ResultVT = Op1.getValueType();
7530 EVT OverflowVT = MVT::i1;
7531 if (ResultVT.isVector())
7532 OverflowVT = EVT::getVectorVT(
7533 Context&: *Context, VT: OverflowVT, EC: ResultVT.getVectorElementCount());
7534
7535 SDVTList VTs = DAG.getVTList(VT1: ResultVT, VT2: OverflowVT);
7536 setValue(V: &I, NewN: DAG.getNode(Opcode: Op, DL: sdl, VTList: VTs, N1: Op1, N2: Op2));
7537 return;
7538 }
7539 case Intrinsic::prefetch: {
7540 SDValue Ops[5];
7541 unsigned rw = cast<ConstantInt>(Val: I.getArgOperand(i: 1))->getZExtValue();
7542 auto Flags = rw == 0 ? MachineMemOperand::MOLoad :MachineMemOperand::MOStore;
7543 Ops[0] = DAG.getRoot();
7544 Ops[1] = getValue(V: I.getArgOperand(i: 0));
7545 Ops[2] = DAG.getTargetConstant(*cast<ConstantInt>(I.getArgOperand(1)), sdl,
7546 MVT::i32);
7547 Ops[3] = DAG.getTargetConstant(*cast<ConstantInt>(I.getArgOperand(2)), sdl,
7548 MVT::i32);
7549 Ops[4] = DAG.getTargetConstant(*cast<ConstantInt>(I.getArgOperand(3)), sdl,
7550 MVT::i32);
7551 SDValue Result = DAG.getMemIntrinsicNode(
7552 ISD::PREFETCH, sdl, DAG.getVTList(MVT::Other), Ops,
7553 EVT::getIntegerVT(*Context, 8), MachinePointerInfo(I.getArgOperand(0)),
7554 /* align */ std::nullopt, Flags);
7555
7556 // Chain the prefetch in parallel with any pending loads, to stay out of
7557 // the way of later optimizations.
7558 PendingLoads.push_back(Elt: Result);
7559 Result = getRoot();
7560 DAG.setRoot(Result);
7561 return;
7562 }
7563 case Intrinsic::lifetime_start:
7564 case Intrinsic::lifetime_end: {
7565 bool IsStart = (Intrinsic == Intrinsic::lifetime_start);
7566 // Stack coloring is not enabled in O0, discard region information.
7567 if (TM.getOptLevel() == CodeGenOptLevel::None)
7568 return;
7569
7570 const int64_t ObjectSize =
7571 cast<ConstantInt>(Val: I.getArgOperand(i: 0))->getSExtValue();
7572 Value *const ObjectPtr = I.getArgOperand(i: 1);
7573 SmallVector<const Value *, 4> Allocas;
7574 getUnderlyingObjects(V: ObjectPtr, Objects&: Allocas);
7575
7576 for (const Value *Alloca : Allocas) {
7577 const AllocaInst *LifetimeObject = dyn_cast_or_null<AllocaInst>(Val: Alloca);
7578
7579 // Could not find an Alloca.
7580 if (!LifetimeObject)
7581 continue;
7582
7583 // First check that the Alloca is static, otherwise it won't have a
7584 // valid frame index.
7585 auto SI = FuncInfo.StaticAllocaMap.find(Val: LifetimeObject);
7586 if (SI == FuncInfo.StaticAllocaMap.end())
7587 return;
7588
7589 const int FrameIndex = SI->second;
7590 int64_t Offset;
7591 if (GetPointerBaseWithConstantOffset(
7592 Ptr: ObjectPtr, Offset, DL: DAG.getDataLayout()) != LifetimeObject)
7593 Offset = -1; // Cannot determine offset from alloca to lifetime object.
7594 Res = DAG.getLifetimeNode(IsStart, dl: sdl, Chain: getRoot(), FrameIndex, Size: ObjectSize,
7595 Offset);
7596 DAG.setRoot(Res);
7597 }
7598 return;
7599 }
7600 case Intrinsic::pseudoprobe: {
7601 auto Guid = cast<ConstantInt>(Val: I.getArgOperand(i: 0))->getZExtValue();
7602 auto Index = cast<ConstantInt>(Val: I.getArgOperand(i: 1))->getZExtValue();
7603 auto Attr = cast<ConstantInt>(Val: I.getArgOperand(i: 2))->getZExtValue();
7604 Res = DAG.getPseudoProbeNode(Dl: sdl, Chain: getRoot(), Guid, Index, Attr);
7605 DAG.setRoot(Res);
7606 return;
7607 }
7608 case Intrinsic::invariant_start:
7609 // Discard region information.
7610 setValue(V: &I,
7611 NewN: DAG.getUNDEF(VT: TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType())));
7612 return;
7613 case Intrinsic::invariant_end:
7614 // Discard region information.
7615 return;
7616 case Intrinsic::clear_cache: {
7617 SDValue InputChain = DAG.getRoot();
7618 SDValue StartVal = getValue(V: I.getArgOperand(i: 0));
7619 SDValue EndVal = getValue(V: I.getArgOperand(i: 1));
7620 Res = DAG.getNode(ISD::CLEAR_CACHE, sdl, DAG.getVTList(MVT::Other),
7621 {InputChain, StartVal, EndVal});
7622 setValue(V: &I, NewN: Res);
7623 DAG.setRoot(Res);
7624 return;
7625 }
7626 case Intrinsic::donothing:
7627 case Intrinsic::seh_try_begin:
7628 case Intrinsic::seh_scope_begin:
7629 case Intrinsic::seh_try_end:
7630 case Intrinsic::seh_scope_end:
7631 // ignore
7632 return;
7633 case Intrinsic::experimental_stackmap:
7634 visitStackmap(I);
7635 return;
7636 case Intrinsic::experimental_patchpoint_void:
7637 case Intrinsic::experimental_patchpoint:
7638 visitPatchpoint(CB: I);
7639 return;
7640 case Intrinsic::experimental_gc_statepoint:
7641 LowerStatepoint(I: cast<GCStatepointInst>(Val: I));
7642 return;
7643 case Intrinsic::experimental_gc_result:
7644 visitGCResult(I: cast<GCResultInst>(Val: I));
7645 return;
7646 case Intrinsic::experimental_gc_relocate:
7647 visitGCRelocate(Relocate: cast<GCRelocateInst>(Val: I));
7648 return;
7649 case Intrinsic::instrprof_cover:
7650 llvm_unreachable("instrprof failed to lower a cover");
7651 case Intrinsic::instrprof_increment:
7652 llvm_unreachable("instrprof failed to lower an increment");
7653 case Intrinsic::instrprof_timestamp:
7654 llvm_unreachable("instrprof failed to lower a timestamp");
7655 case Intrinsic::instrprof_value_profile:
7656 llvm_unreachable("instrprof failed to lower a value profiling call");
7657 case Intrinsic::instrprof_mcdc_parameters:
7658 llvm_unreachable("instrprof failed to lower mcdc parameters");
7659 case Intrinsic::instrprof_mcdc_tvbitmap_update:
7660 llvm_unreachable("instrprof failed to lower an mcdc tvbitmap update");
7661 case Intrinsic::localescape: {
7662 MachineFunction &MF = DAG.getMachineFunction();
7663 const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo();
7664
7665 // Directly emit some LOCAL_ESCAPE machine instrs. Label assignment emission
7666 // is the same on all targets.
7667 for (unsigned Idx = 0, E = I.arg_size(); Idx < E; ++Idx) {
7668 Value *Arg = I.getArgOperand(i: Idx)->stripPointerCasts();
7669 if (isa<ConstantPointerNull>(Val: Arg))
7670 continue; // Skip null pointers. They represent a hole in index space.
7671 AllocaInst *Slot = cast<AllocaInst>(Val: Arg);
7672 assert(FuncInfo.StaticAllocaMap.count(Slot) &&
7673 "can only escape static allocas");
7674 int FI = FuncInfo.StaticAllocaMap[Slot];
7675 MCSymbol *FrameAllocSym = MF.getContext().getOrCreateFrameAllocSymbol(
7676 FuncName: GlobalValue::dropLLVMManglingEscape(Name: MF.getName()), Idx);
7677 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD: dl,
7678 MCID: TII->get(Opcode: TargetOpcode::LOCAL_ESCAPE))
7679 .addSym(Sym: FrameAllocSym)
7680 .addFrameIndex(Idx: FI);
7681 }
7682
7683 return;
7684 }
7685
7686 case Intrinsic::localrecover: {
7687 // i8* @llvm.localrecover(i8* %fn, i8* %fp, i32 %idx)
7688 MachineFunction &MF = DAG.getMachineFunction();
7689
7690 // Get the symbol that defines the frame offset.
7691 auto *Fn = cast<Function>(Val: I.getArgOperand(i: 0)->stripPointerCasts());
7692 auto *Idx = cast<ConstantInt>(Val: I.getArgOperand(i: 2));
7693 unsigned IdxVal =
7694 unsigned(Idx->getLimitedValue(Limit: std::numeric_limits<int>::max()));
7695 MCSymbol *FrameAllocSym = MF.getContext().getOrCreateFrameAllocSymbol(
7696 FuncName: GlobalValue::dropLLVMManglingEscape(Name: Fn->getName()), Idx: IdxVal);
7697
7698 Value *FP = I.getArgOperand(i: 1);
7699 SDValue FPVal = getValue(V: FP);
7700 EVT PtrVT = FPVal.getValueType();
7701
7702 // Create a MCSymbol for the label to avoid any target lowering
7703 // that would make this PC relative.
7704 SDValue OffsetSym = DAG.getMCSymbol(Sym: FrameAllocSym, VT: PtrVT);
7705 SDValue OffsetVal =
7706 DAG.getNode(Opcode: ISD::LOCAL_RECOVER, DL: sdl, VT: PtrVT, Operand: OffsetSym);
7707
7708 // Add the offset to the FP.
7709 SDValue Add = DAG.getMemBasePlusOffset(Base: FPVal, Offset: OffsetVal, DL: sdl);
7710 setValue(V: &I, NewN: Add);
7711
7712 return;
7713 }
7714
7715 case Intrinsic::fake_use: {
7716 Value *V = I.getArgOperand(i: 0);
7717 SDValue Ops[2];
7718 // For Values not declared or previously used in this basic block, the
7719 // NodeMap will not have an entry, and `getValue` will assert if V has no
7720 // valid register value.
7721 auto FakeUseValue = [&]() -> SDValue {
7722 SDValue &N = NodeMap[V];
7723 if (N.getNode())
7724 return N;
7725
7726 // If there's a virtual register allocated and initialized for this
7727 // value, use it.
7728 if (SDValue copyFromReg = getCopyFromRegs(V, Ty: V->getType()))
7729 return copyFromReg;
7730 // FIXME: Do we want to preserve constants? It seems pointless.
7731 if (isa<Constant>(Val: V))
7732 return getValue(V);
7733 return SDValue();
7734 }();
7735 if (!FakeUseValue || FakeUseValue.isUndef())
7736 return;
7737 Ops[0] = getRoot();
7738 Ops[1] = FakeUseValue;
7739 // Also, do not translate a fake use with an undef operand, or any other
7740 // empty SDValues.
7741 if (!Ops[1] || Ops[1].isUndef())
7742 return;
7743 DAG.setRoot(DAG.getNode(ISD::FAKE_USE, sdl, MVT::Other, Ops));
7744 return;
7745 }
7746
7747 case Intrinsic::eh_exceptionpointer:
7748 case Intrinsic::eh_exceptioncode: {
7749 // Get the exception pointer vreg, copy from it, and resize it to fit.
7750 const auto *CPI = cast<CatchPadInst>(Val: I.getArgOperand(i: 0));
7751 MVT PtrVT = TLI.getPointerTy(DL: DAG.getDataLayout());
7752 const TargetRegisterClass *PtrRC = TLI.getRegClassFor(VT: PtrVT);
7753 Register VReg = FuncInfo.getCatchPadExceptionPointerVReg(CPI, RC: PtrRC);
7754 SDValue N = DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl: sdl, Reg: VReg, VT: PtrVT);
7755 if (Intrinsic == Intrinsic::eh_exceptioncode)
7756 N = DAG.getZExtOrTrunc(N, sdl, MVT::i32);
7757 setValue(V: &I, NewN: N);
7758 return;
7759 }
7760 case Intrinsic::xray_customevent: {
7761 // Here we want to make sure that the intrinsic behaves as if it has a
7762 // specific calling convention.
7763 const auto &Triple = DAG.getTarget().getTargetTriple();
7764 if (!Triple.isAArch64(PointerWidth: 64) && Triple.getArch() != Triple::x86_64)
7765 return;
7766
7767 SmallVector<SDValue, 8> Ops;
7768
7769 // We want to say that we always want the arguments in registers.
7770 SDValue LogEntryVal = getValue(V: I.getArgOperand(i: 0));
7771 SDValue StrSizeVal = getValue(V: I.getArgOperand(i: 1));
7772 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
7773 SDValue Chain = getRoot();
7774 Ops.push_back(Elt: LogEntryVal);
7775 Ops.push_back(Elt: StrSizeVal);
7776 Ops.push_back(Elt: Chain);
7777
7778 // We need to enforce the calling convention for the callsite, so that
7779 // argument ordering is enforced correctly, and that register allocation can
7780 // see that some registers may be assumed clobbered and have to preserve
7781 // them across calls to the intrinsic.
7782 MachineSDNode *MN = DAG.getMachineNode(Opcode: TargetOpcode::PATCHABLE_EVENT_CALL,
7783 dl: sdl, VTs: NodeTys, Ops);
7784 SDValue patchableNode = SDValue(MN, 0);
7785 DAG.setRoot(patchableNode);
7786 setValue(V: &I, NewN: patchableNode);
7787 return;
7788 }
7789 case Intrinsic::xray_typedevent: {
7790 // Here we want to make sure that the intrinsic behaves as if it has a
7791 // specific calling convention.
7792 const auto &Triple = DAG.getTarget().getTargetTriple();
7793 if (!Triple.isAArch64(PointerWidth: 64) && Triple.getArch() != Triple::x86_64)
7794 return;
7795
7796 SmallVector<SDValue, 8> Ops;
7797
7798 // We want to say that we always want the arguments in registers.
7799 // It's unclear to me how manipulating the selection DAG here forces callers
7800 // to provide arguments in registers instead of on the stack.
7801 SDValue LogTypeId = getValue(V: I.getArgOperand(i: 0));
7802 SDValue LogEntryVal = getValue(V: I.getArgOperand(i: 1));
7803 SDValue StrSizeVal = getValue(V: I.getArgOperand(i: 2));
7804 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
7805 SDValue Chain = getRoot();
7806 Ops.push_back(Elt: LogTypeId);
7807 Ops.push_back(Elt: LogEntryVal);
7808 Ops.push_back(Elt: StrSizeVal);
7809 Ops.push_back(Elt: Chain);
7810
7811 // We need to enforce the calling convention for the callsite, so that
7812 // argument ordering is enforced correctly, and that register allocation can
7813 // see that some registers may be assumed clobbered and have to preserve
7814 // them across calls to the intrinsic.
7815 MachineSDNode *MN = DAG.getMachineNode(
7816 Opcode: TargetOpcode::PATCHABLE_TYPED_EVENT_CALL, dl: sdl, VTs: NodeTys, Ops);
7817 SDValue patchableNode = SDValue(MN, 0);
7818 DAG.setRoot(patchableNode);
7819 setValue(V: &I, NewN: patchableNode);
7820 return;
7821 }
7822 case Intrinsic::experimental_deoptimize:
7823 LowerDeoptimizeCall(CI: &I);
7824 return;
7825 case Intrinsic::stepvector:
7826 visitStepVector(I);
7827 return;
7828 case Intrinsic::vector_reduce_fadd:
7829 case Intrinsic::vector_reduce_fmul:
7830 case Intrinsic::vector_reduce_add:
7831 case Intrinsic::vector_reduce_mul:
7832 case Intrinsic::vector_reduce_and:
7833 case Intrinsic::vector_reduce_or:
7834 case Intrinsic::vector_reduce_xor:
7835 case Intrinsic::vector_reduce_smax:
7836 case Intrinsic::vector_reduce_smin:
7837 case Intrinsic::vector_reduce_umax:
7838 case Intrinsic::vector_reduce_umin:
7839 case Intrinsic::vector_reduce_fmax:
7840 case Intrinsic::vector_reduce_fmin:
7841 case Intrinsic::vector_reduce_fmaximum:
7842 case Intrinsic::vector_reduce_fminimum:
7843 visitVectorReduce(I, Intrinsic);
7844 return;
7845
7846 case Intrinsic::icall_branch_funnel: {
7847 SmallVector<SDValue, 16> Ops;
7848 Ops.push_back(Elt: getValue(V: I.getArgOperand(i: 0)));
7849
7850 int64_t Offset;
7851 auto *Base = dyn_cast<GlobalObject>(Val: GetPointerBaseWithConstantOffset(
7852 Ptr: I.getArgOperand(i: 1), Offset, DL: DAG.getDataLayout()));
7853 if (!Base)
7854 report_fatal_error(
7855 reason: "llvm.icall.branch.funnel operand must be a GlobalValue");
7856 Ops.push_back(DAG.getTargetGlobalAddress(Base, sdl, MVT::i64, 0));
7857
7858 struct BranchFunnelTarget {
7859 int64_t Offset;
7860 SDValue Target;
7861 };
7862 SmallVector<BranchFunnelTarget, 8> Targets;
7863
7864 for (unsigned Op = 1, N = I.arg_size(); Op != N; Op += 2) {
7865 auto *ElemBase = dyn_cast<GlobalObject>(Val: GetPointerBaseWithConstantOffset(
7866 Ptr: I.getArgOperand(i: Op), Offset, DL: DAG.getDataLayout()));
7867 if (ElemBase != Base)
7868 report_fatal_error(reason: "all llvm.icall.branch.funnel operands must refer "
7869 "to the same GlobalValue");
7870
7871 SDValue Val = getValue(V: I.getArgOperand(i: Op + 1));
7872 auto *GA = dyn_cast<GlobalAddressSDNode>(Val);
7873 if (!GA)
7874 report_fatal_error(
7875 reason: "llvm.icall.branch.funnel operand must be a GlobalValue");
7876 Targets.push_back(Elt: {.Offset: Offset, .Target: DAG.getTargetGlobalAddress(
7877 GV: GA->getGlobal(), DL: sdl, VT: Val.getValueType(),
7878 offset: GA->getOffset())});
7879 }
7880 llvm::sort(C&: Targets,
7881 Comp: [](const BranchFunnelTarget &T1, const BranchFunnelTarget &T2) {
7882 return T1.Offset < T2.Offset;
7883 });
7884
7885 for (auto &T : Targets) {
7886 Ops.push_back(DAG.getTargetConstant(T.Offset, sdl, MVT::i32));
7887 Ops.push_back(Elt: T.Target);
7888 }
7889
7890 Ops.push_back(Elt: DAG.getRoot()); // Chain
7891 SDValue N(DAG.getMachineNode(TargetOpcode::ICALL_BRANCH_FUNNEL, sdl,
7892 MVT::Other, Ops),
7893 0);
7894 DAG.setRoot(N);
7895 setValue(V: &I, NewN: N);
7896 HasTailCall = true;
7897 return;
7898 }
7899
7900 case Intrinsic::wasm_landingpad_index:
7901 // Information this intrinsic contained has been transferred to
7902 // MachineFunction in SelectionDAGISel::PrepareEHLandingPad. We can safely
7903 // delete it now.
7904 return;
7905
7906 case Intrinsic::aarch64_settag:
7907 case Intrinsic::aarch64_settag_zero: {
7908 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
7909 bool ZeroMemory = Intrinsic == Intrinsic::aarch64_settag_zero;
7910 SDValue Val = TSI.EmitTargetCodeForSetTag(
7911 DAG, dl: sdl, Chain: getRoot(), Addr: getValue(V: I.getArgOperand(i: 0)),
7912 Size: getValue(V: I.getArgOperand(i: 1)), DstPtrInfo: MachinePointerInfo(I.getArgOperand(i: 0)),
7913 ZeroData: ZeroMemory);
7914 DAG.setRoot(Val);
7915 setValue(V: &I, NewN: Val);
7916 return;
7917 }
7918 case Intrinsic::amdgcn_cs_chain: {
7919 // At this point we don't care if it's amdgpu_cs_chain or
7920 // amdgpu_cs_chain_preserve.
7921 CallingConv::ID CC = CallingConv::AMDGPU_CS_Chain;
7922
7923 Type *RetTy = I.getType();
7924 assert(RetTy->isVoidTy() && "Should not return");
7925
7926 SDValue Callee = getValue(V: I.getOperand(i_nocapture: 0));
7927
7928 // We only have 2 actual args: one for the SGPRs and one for the VGPRs.
7929 // We'll also tack the value of the EXEC mask at the end.
7930 TargetLowering::ArgListTy Args;
7931 Args.reserve(n: 3);
7932
7933 for (unsigned Idx : {2, 3, 1}) {
7934 TargetLowering::ArgListEntry Arg;
7935 Arg.Node = getValue(V: I.getOperand(i_nocapture: Idx));
7936 Arg.Ty = I.getOperand(i_nocapture: Idx)->getType();
7937 Arg.setAttributes(Call: &I, ArgIdx: Idx);
7938 Args.push_back(x: Arg);
7939 }
7940
7941 assert(Args[0].IsInReg && "SGPR args should be marked inreg");
7942 assert(!Args[1].IsInReg && "VGPR args should not be marked inreg");
7943 Args[2].IsInReg = true; // EXEC should be inreg
7944
7945 // Forward the flags and any additional arguments.
7946 for (unsigned Idx = 4; Idx < I.arg_size(); ++Idx) {
7947 TargetLowering::ArgListEntry Arg;
7948 Arg.Node = getValue(V: I.getOperand(i_nocapture: Idx));
7949 Arg.Ty = I.getOperand(i_nocapture: Idx)->getType();
7950 Arg.setAttributes(Call: &I, ArgIdx: Idx);
7951 Args.push_back(x: Arg);
7952 }
7953
7954 TargetLowering::CallLoweringInfo CLI(DAG);
7955 CLI.setDebugLoc(getCurSDLoc())
7956 .setChain(getRoot())
7957 .setCallee(CC, ResultType: RetTy, Target: Callee, ArgsList: std::move(Args))
7958 .setNoReturn(true)
7959 .setTailCall(true)
7960 .setConvergent(I.isConvergent());
7961 CLI.CB = &I;
7962 std::pair<SDValue, SDValue> Result =
7963 lowerInvokable(CLI, /*EHPadBB*/ nullptr);
7964 (void)Result;
7965 assert(!Result.first.getNode() && !Result.second.getNode() &&
7966 "Should've lowered as tail call");
7967
7968 HasTailCall = true;
7969 return;
7970 }
7971 case Intrinsic::ptrmask: {
7972 SDValue Ptr = getValue(V: I.getOperand(i_nocapture: 0));
7973 SDValue Mask = getValue(V: I.getOperand(i_nocapture: 1));
7974
7975 // On arm64_32, pointers are 32 bits when stored in memory, but
7976 // zero-extended to 64 bits when in registers. Thus the mask is 32 bits to
7977 // match the index type, but the pointer is 64 bits, so the mask must be
7978 // zero-extended up to 64 bits to match the pointer.
7979 EVT PtrVT =
7980 TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getOperand(i_nocapture: 0)->getType());
7981 EVT MemVT =
7982 TLI.getMemValueType(DL: DAG.getDataLayout(), Ty: I.getOperand(i_nocapture: 0)->getType());
7983 assert(PtrVT == Ptr.getValueType());
7984 if (Mask.getValueType().getFixedSizeInBits() < MemVT.getFixedSizeInBits()) {
7985 // For AMDGPU buffer descriptors the mask is 48 bits, but the pointer is
7986 // 128-bit, so we have to pad the mask with ones for unused bits.
7987 auto HighOnes = DAG.getNode(
7988 Opcode: ISD::SHL, DL: sdl, VT: PtrVT, N1: DAG.getAllOnesConstant(DL: sdl, VT: PtrVT),
7989 N2: DAG.getShiftAmountConstant(Val: Mask.getValueType().getFixedSizeInBits(),
7990 VT: PtrVT, DL: sdl));
7991 Mask = DAG.getNode(Opcode: ISD::OR, DL: sdl, VT: PtrVT,
7992 N1: DAG.getZExtOrTrunc(Op: Mask, DL: sdl, VT: PtrVT), N2: HighOnes);
7993 } else if (Mask.getValueType() != PtrVT)
7994 Mask = DAG.getPtrExtOrTrunc(Op: Mask, DL: sdl, VT: PtrVT);
7995
7996 assert(Mask.getValueType() == PtrVT);
7997 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::AND, DL: sdl, VT: PtrVT, N1: Ptr, N2: Mask));
7998 return;
7999 }
8000 case Intrinsic::threadlocal_address: {
8001 setValue(V: &I, NewN: getValue(V: I.getOperand(i_nocapture: 0)));
8002 return;
8003 }
8004 case Intrinsic::get_active_lane_mask: {
8005 EVT CCVT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
8006 SDValue Index = getValue(V: I.getOperand(i_nocapture: 0));
8007 SDValue TripCount = getValue(V: I.getOperand(i_nocapture: 1));
8008 EVT ElementVT = Index.getValueType();
8009
8010 if (!TLI.shouldExpandGetActiveLaneMask(VT: CCVT, OpVT: ElementVT)) {
8011 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::GET_ACTIVE_LANE_MASK, DL: sdl, VT: CCVT, N1: Index,
8012 N2: TripCount));
8013 return;
8014 }
8015
8016 EVT VecTy = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ElementVT,
8017 EC: CCVT.getVectorElementCount());
8018
8019 SDValue VectorIndex = DAG.getSplat(VT: VecTy, DL: sdl, Op: Index);
8020 SDValue VectorTripCount = DAG.getSplat(VT: VecTy, DL: sdl, Op: TripCount);
8021 SDValue VectorStep = DAG.getStepVector(DL: sdl, ResVT: VecTy);
8022 SDValue VectorInduction = DAG.getNode(
8023 Opcode: ISD::UADDSAT, DL: sdl, VT: VecTy, N1: VectorIndex, N2: VectorStep);
8024 SDValue SetCC = DAG.getSetCC(DL: sdl, VT: CCVT, LHS: VectorInduction,
8025 RHS: VectorTripCount, Cond: ISD::CondCode::SETULT);
8026 setValue(V: &I, NewN: SetCC);
8027 return;
8028 }
8029 case Intrinsic::experimental_get_vector_length: {
8030 assert(cast<ConstantInt>(I.getOperand(1))->getSExtValue() > 0 &&
8031 "Expected positive VF");
8032 unsigned VF = cast<ConstantInt>(Val: I.getOperand(i_nocapture: 1))->getZExtValue();
8033 bool IsScalable = cast<ConstantInt>(Val: I.getOperand(i_nocapture: 2))->isOne();
8034
8035 SDValue Count = getValue(V: I.getOperand(i_nocapture: 0));
8036 EVT CountVT = Count.getValueType();
8037
8038 if (!TLI.shouldExpandGetVectorLength(CountVT, VF, IsScalable)) {
8039 visitTargetIntrinsic(I, Intrinsic);
8040 return;
8041 }
8042
8043 // Expand to a umin between the trip count and the maximum elements the type
8044 // can hold.
8045 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
8046
8047 // Extend the trip count to at least the result VT.
8048 if (CountVT.bitsLT(VT)) {
8049 Count = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: sdl, VT, Operand: Count);
8050 CountVT = VT;
8051 }
8052
8053 SDValue MaxEVL = DAG.getElementCount(DL: sdl, VT: CountVT,
8054 EC: ElementCount::get(MinVal: VF, Scalable: IsScalable));
8055
8056 SDValue UMin = DAG.getNode(Opcode: ISD::UMIN, DL: sdl, VT: CountVT, N1: Count, N2: MaxEVL);
8057 // Clip to the result type if needed.
8058 SDValue Trunc = DAG.getNode(Opcode: ISD::TRUNCATE, DL: sdl, VT, Operand: UMin);
8059
8060 setValue(V: &I, NewN: Trunc);
8061 return;
8062 }
8063 case Intrinsic::experimental_vector_partial_reduce_add: {
8064 if (!TLI.shouldExpandPartialReductionIntrinsic(I: cast<IntrinsicInst>(Val: &I))) {
8065 visitTargetIntrinsic(I, Intrinsic);
8066 return;
8067 }
8068 SDValue Acc = getValue(V: I.getOperand(i_nocapture: 0));
8069 SDValue Input = getValue(V: I.getOperand(i_nocapture: 1));
8070 setValue(V: &I,
8071 NewN: DAG.getNode(Opcode: ISD::PARTIAL_REDUCE_UMLA, DL: sdl, VT: Acc.getValueType(), N1: Acc,
8072 N2: Input, N3: DAG.getConstant(Val: 1, DL: sdl, VT: Input.getValueType())));
8073 return;
8074 }
8075 case Intrinsic::experimental_cttz_elts: {
8076 auto DL = getCurSDLoc();
8077 SDValue Op = getValue(V: I.getOperand(i_nocapture: 0));
8078 EVT OpVT = Op.getValueType();
8079
8080 if (!TLI.shouldExpandCttzElements(VT: OpVT)) {
8081 visitTargetIntrinsic(I, Intrinsic);
8082 return;
8083 }
8084
8085 if (OpVT.getScalarType() != MVT::i1) {
8086 // Compare the input vector elements to zero & use to count trailing zeros
8087 SDValue AllZero = DAG.getConstant(Val: 0, DL, VT: OpVT);
8088 OpVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
8089 OpVT.getVectorElementCount());
8090 Op = DAG.getSetCC(DL, VT: OpVT, LHS: Op, RHS: AllZero, Cond: ISD::SETNE);
8091 }
8092
8093 // If the zero-is-poison flag is set, we can assume the upper limit
8094 // of the result is VF-1.
8095 bool ZeroIsPoison =
8096 !cast<ConstantSDNode>(Val: getValue(V: I.getOperand(i_nocapture: 1)))->isZero();
8097 ConstantRange VScaleRange(1, true); // Dummy value.
8098 if (isa<ScalableVectorType>(Val: I.getOperand(i_nocapture: 0)->getType()))
8099 VScaleRange = getVScaleRange(F: I.getCaller(), BitWidth: 64);
8100 unsigned EltWidth = TLI.getBitWidthForCttzElements(
8101 RetTy: I.getType(), EC: OpVT.getVectorElementCount(), ZeroIsPoison, VScaleRange: &VScaleRange);
8102
8103 MVT NewEltTy = MVT::getIntegerVT(BitWidth: EltWidth);
8104
8105 // Create the new vector type & get the vector length
8106 EVT NewVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: NewEltTy,
8107 EC: OpVT.getVectorElementCount());
8108
8109 SDValue VL =
8110 DAG.getElementCount(DL, VT: NewEltTy, EC: OpVT.getVectorElementCount());
8111
8112 SDValue StepVec = DAG.getStepVector(DL, ResVT: NewVT);
8113 SDValue SplatVL = DAG.getSplat(VT: NewVT, DL, Op: VL);
8114 SDValue StepVL = DAG.getNode(Opcode: ISD::SUB, DL, VT: NewVT, N1: SplatVL, N2: StepVec);
8115 SDValue Ext = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: NewVT, Operand: Op);
8116 SDValue And = DAG.getNode(Opcode: ISD::AND, DL, VT: NewVT, N1: StepVL, N2: Ext);
8117 SDValue Max = DAG.getNode(Opcode: ISD::VECREDUCE_UMAX, DL, VT: NewEltTy, Operand: And);
8118 SDValue Sub = DAG.getNode(Opcode: ISD::SUB, DL, VT: NewEltTy, N1: VL, N2: Max);
8119
8120 EVT RetTy = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
8121 SDValue Ret = DAG.getZExtOrTrunc(Op: Sub, DL, VT: RetTy);
8122
8123 setValue(V: &I, NewN: Ret);
8124 return;
8125 }
8126 case Intrinsic::vector_insert: {
8127 SDValue Vec = getValue(V: I.getOperand(i_nocapture: 0));
8128 SDValue SubVec = getValue(V: I.getOperand(i_nocapture: 1));
8129 SDValue Index = getValue(V: I.getOperand(i_nocapture: 2));
8130
8131 // The intrinsic's index type is i64, but the SDNode requires an index type
8132 // suitable for the target. Convert the index as required.
8133 MVT VectorIdxTy = TLI.getVectorIdxTy(DL: DAG.getDataLayout());
8134 if (Index.getValueType() != VectorIdxTy)
8135 Index = DAG.getVectorIdxConstant(Val: Index->getAsZExtVal(), DL: sdl);
8136
8137 EVT ResultVT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
8138 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: sdl, VT: ResultVT, N1: Vec, N2: SubVec,
8139 N3: Index));
8140 return;
8141 }
8142 case Intrinsic::vector_extract: {
8143 SDValue Vec = getValue(V: I.getOperand(i_nocapture: 0));
8144 SDValue Index = getValue(V: I.getOperand(i_nocapture: 1));
8145 EVT ResultVT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
8146
8147 // The intrinsic's index type is i64, but the SDNode requires an index type
8148 // suitable for the target. Convert the index as required.
8149 MVT VectorIdxTy = TLI.getVectorIdxTy(DL: DAG.getDataLayout());
8150 if (Index.getValueType() != VectorIdxTy)
8151 Index = DAG.getVectorIdxConstant(Val: Index->getAsZExtVal(), DL: sdl);
8152
8153 setValue(V: &I,
8154 NewN: DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: sdl, VT: ResultVT, N1: Vec, N2: Index));
8155 return;
8156 }
8157 case Intrinsic::experimental_vector_match: {
8158 SDValue Op1 = getValue(V: I.getOperand(i_nocapture: 0));
8159 SDValue Op2 = getValue(V: I.getOperand(i_nocapture: 1));
8160 SDValue Mask = getValue(V: I.getOperand(i_nocapture: 2));
8161 EVT Op1VT = Op1.getValueType();
8162 EVT Op2VT = Op2.getValueType();
8163 EVT ResVT = Mask.getValueType();
8164 unsigned SearchSize = Op2VT.getVectorNumElements();
8165
8166 // If the target has native support for this vector match operation, lower
8167 // the intrinsic untouched; otherwise, expand it below.
8168 if (!TLI.shouldExpandVectorMatch(VT: Op1VT, SearchSize)) {
8169 visitTargetIntrinsic(I, Intrinsic);
8170 return;
8171 }
8172
8173 SDValue Ret = DAG.getConstant(Val: 0, DL: sdl, VT: ResVT);
8174
8175 for (unsigned i = 0; i < SearchSize; ++i) {
8176 SDValue Op2Elem = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: sdl,
8177 VT: Op2VT.getVectorElementType(), N1: Op2,
8178 N2: DAG.getVectorIdxConstant(Val: i, DL: sdl));
8179 SDValue Splat = DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL: sdl, VT: Op1VT, Operand: Op2Elem);
8180 SDValue Cmp = DAG.getSetCC(DL: sdl, VT: ResVT, LHS: Op1, RHS: Splat, Cond: ISD::SETEQ);
8181 Ret = DAG.getNode(Opcode: ISD::OR, DL: sdl, VT: ResVT, N1: Ret, N2: Cmp);
8182 }
8183
8184 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::AND, DL: sdl, VT: ResVT, N1: Ret, N2: Mask));
8185 return;
8186 }
8187 case Intrinsic::vector_reverse:
8188 visitVectorReverse(I);
8189 return;
8190 case Intrinsic::vector_splice:
8191 visitVectorSplice(I);
8192 return;
8193 case Intrinsic::callbr_landingpad:
8194 visitCallBrLandingPad(I);
8195 return;
8196 case Intrinsic::vector_interleave2:
8197 visitVectorInterleave(I, Factor: 2);
8198 return;
8199 case Intrinsic::vector_interleave3:
8200 visitVectorInterleave(I, Factor: 3);
8201 return;
8202 case Intrinsic::vector_interleave4:
8203 visitVectorInterleave(I, Factor: 4);
8204 return;
8205 case Intrinsic::vector_interleave5:
8206 visitVectorInterleave(I, Factor: 5);
8207 return;
8208 case Intrinsic::vector_interleave6:
8209 visitVectorInterleave(I, Factor: 6);
8210 return;
8211 case Intrinsic::vector_interleave7:
8212 visitVectorInterleave(I, Factor: 7);
8213 return;
8214 case Intrinsic::vector_interleave8:
8215 visitVectorInterleave(I, Factor: 8);
8216 return;
8217 case Intrinsic::vector_deinterleave2:
8218 visitVectorDeinterleave(I, Factor: 2);
8219 return;
8220 case Intrinsic::vector_deinterleave3:
8221 visitVectorDeinterleave(I, Factor: 3);
8222 return;
8223 case Intrinsic::vector_deinterleave4:
8224 visitVectorDeinterleave(I, Factor: 4);
8225 return;
8226 case Intrinsic::vector_deinterleave5:
8227 visitVectorDeinterleave(I, Factor: 5);
8228 return;
8229 case Intrinsic::vector_deinterleave6:
8230 visitVectorDeinterleave(I, Factor: 6);
8231 return;
8232 case Intrinsic::vector_deinterleave7:
8233 visitVectorDeinterleave(I, Factor: 7);
8234 return;
8235 case Intrinsic::vector_deinterleave8:
8236 visitVectorDeinterleave(I, Factor: 8);
8237 return;
8238 case Intrinsic::experimental_vector_compress:
8239 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::VECTOR_COMPRESS, DL: sdl,
8240 VT: getValue(V: I.getArgOperand(i: 0)).getValueType(),
8241 N1: getValue(V: I.getArgOperand(i: 0)),
8242 N2: getValue(V: I.getArgOperand(i: 1)),
8243 N3: getValue(V: I.getArgOperand(i: 2)), Flags));
8244 return;
8245 case Intrinsic::experimental_convergence_anchor:
8246 case Intrinsic::experimental_convergence_entry:
8247 case Intrinsic::experimental_convergence_loop:
8248 visitConvergenceControl(I, Intrinsic);
8249 return;
8250 case Intrinsic::experimental_vector_histogram_add: {
8251 visitVectorHistogram(I, IntrinsicID: Intrinsic);
8252 return;
8253 }
8254 case Intrinsic::experimental_vector_extract_last_active: {
8255 visitVectorExtractLastActive(I, Intrinsic);
8256 return;
8257 }
8258 }
8259}
8260
8261void SelectionDAGBuilder::visitConstrainedFPIntrinsic(
8262 const ConstrainedFPIntrinsic &FPI) {
8263 SDLoc sdl = getCurSDLoc();
8264
8265 // We do not need to serialize constrained FP intrinsics against
8266 // each other or against (nonvolatile) loads, so they can be
8267 // chained like loads.
8268 SDValue Chain = DAG.getRoot();
8269 SmallVector<SDValue, 4> Opers;
8270 Opers.push_back(Elt: Chain);
8271 for (unsigned I = 0, E = FPI.getNonMetadataArgCount(); I != E; ++I)
8272 Opers.push_back(Elt: getValue(V: FPI.getArgOperand(i: I)));
8273
8274 auto pushOutChain = [this](SDValue Result, fp::ExceptionBehavior EB) {
8275 assert(Result.getNode()->getNumValues() == 2);
8276
8277 // Push node to the appropriate list so that future instructions can be
8278 // chained up correctly.
8279 SDValue OutChain = Result.getValue(R: 1);
8280 switch (EB) {
8281 case fp::ExceptionBehavior::ebIgnore:
8282 // The only reason why ebIgnore nodes still need to be chained is that
8283 // they might depend on the current rounding mode, and therefore must
8284 // not be moved across instruction that may change that mode.
8285 [[fallthrough]];
8286 case fp::ExceptionBehavior::ebMayTrap:
8287 // These must not be moved across calls or instructions that may change
8288 // floating-point exception masks.
8289 PendingConstrainedFP.push_back(Elt: OutChain);
8290 break;
8291 case fp::ExceptionBehavior::ebStrict:
8292 // These must not be moved across calls or instructions that may change
8293 // floating-point exception masks or read floating-point exception flags.
8294 // In addition, they cannot be optimized out even if unused.
8295 PendingConstrainedFPStrict.push_back(Elt: OutChain);
8296 break;
8297 }
8298 };
8299
8300 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8301 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: FPI.getType());
8302 SDVTList VTs = DAG.getVTList(VT, MVT::Other);
8303 fp::ExceptionBehavior EB = *FPI.getExceptionBehavior();
8304
8305 SDNodeFlags Flags;
8306 if (EB == fp::ExceptionBehavior::ebIgnore)
8307 Flags.setNoFPExcept(true);
8308
8309 if (auto *FPOp = dyn_cast<FPMathOperator>(Val: &FPI))
8310 Flags.copyFMF(FPMO: *FPOp);
8311
8312 unsigned Opcode;
8313 switch (FPI.getIntrinsicID()) {
8314 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
8315#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
8316 case Intrinsic::INTRINSIC: \
8317 Opcode = ISD::STRICT_##DAGN; \
8318 break;
8319#include "llvm/IR/ConstrainedOps.def"
8320 case Intrinsic::experimental_constrained_fmuladd: {
8321 Opcode = ISD::STRICT_FMA;
8322 // Break fmuladd into fmul and fadd.
8323 if (TM.Options.AllowFPOpFusion == FPOpFusion::Strict ||
8324 !TLI.isFMAFasterThanFMulAndFAdd(MF: DAG.getMachineFunction(), VT)) {
8325 Opers.pop_back();
8326 SDValue Mul = DAG.getNode(Opcode: ISD::STRICT_FMUL, DL: sdl, VTList: VTs, Ops: Opers, Flags);
8327 pushOutChain(Mul, EB);
8328 Opcode = ISD::STRICT_FADD;
8329 Opers.clear();
8330 Opers.push_back(Elt: Mul.getValue(R: 1));
8331 Opers.push_back(Elt: Mul.getValue(R: 0));
8332 Opers.push_back(Elt: getValue(V: FPI.getArgOperand(i: 2)));
8333 }
8334 break;
8335 }
8336 }
8337
8338 // A few strict DAG nodes carry additional operands that are not
8339 // set up by the default code above.
8340 switch (Opcode) {
8341 default: break;
8342 case ISD::STRICT_FP_ROUND:
8343 Opers.push_back(
8344 Elt: DAG.getTargetConstant(Val: 0, DL: sdl, VT: TLI.getPointerTy(DL: DAG.getDataLayout())));
8345 break;
8346 case ISD::STRICT_FSETCC:
8347 case ISD::STRICT_FSETCCS: {
8348 auto *FPCmp = dyn_cast<ConstrainedFPCmpIntrinsic>(Val: &FPI);
8349 ISD::CondCode Condition = getFCmpCondCode(Pred: FPCmp->getPredicate());
8350 if (TM.Options.NoNaNsFPMath)
8351 Condition = getFCmpCodeWithoutNaN(CC: Condition);
8352 Opers.push_back(Elt: DAG.getCondCode(Cond: Condition));
8353 break;
8354 }
8355 }
8356
8357 SDValue Result = DAG.getNode(Opcode, DL: sdl, VTList: VTs, Ops: Opers, Flags);
8358 pushOutChain(Result, EB);
8359
8360 SDValue FPResult = Result.getValue(R: 0);
8361 setValue(V: &FPI, NewN: FPResult);
8362}
8363
8364static unsigned getISDForVPIntrinsic(const VPIntrinsic &VPIntrin) {
8365 std::optional<unsigned> ResOPC;
8366 switch (VPIntrin.getIntrinsicID()) {
8367 case Intrinsic::vp_ctlz: {
8368 bool IsZeroUndef = cast<ConstantInt>(Val: VPIntrin.getArgOperand(i: 1))->isOne();
8369 ResOPC = IsZeroUndef ? ISD::VP_CTLZ_ZERO_UNDEF : ISD::VP_CTLZ;
8370 break;
8371 }
8372 case Intrinsic::vp_cttz: {
8373 bool IsZeroUndef = cast<ConstantInt>(Val: VPIntrin.getArgOperand(i: 1))->isOne();
8374 ResOPC = IsZeroUndef ? ISD::VP_CTTZ_ZERO_UNDEF : ISD::VP_CTTZ;
8375 break;
8376 }
8377 case Intrinsic::vp_cttz_elts: {
8378 bool IsZeroPoison = cast<ConstantInt>(Val: VPIntrin.getArgOperand(i: 1))->isOne();
8379 ResOPC = IsZeroPoison ? ISD::VP_CTTZ_ELTS_ZERO_UNDEF : ISD::VP_CTTZ_ELTS;
8380 break;
8381 }
8382#define HELPER_MAP_VPID_TO_VPSD(VPID, VPSD) \
8383 case Intrinsic::VPID: \
8384 ResOPC = ISD::VPSD; \
8385 break;
8386#include "llvm/IR/VPIntrinsics.def"
8387 }
8388
8389 if (!ResOPC)
8390 llvm_unreachable(
8391 "Inconsistency: no SDNode available for this VPIntrinsic!");
8392
8393 if (*ResOPC == ISD::VP_REDUCE_SEQ_FADD ||
8394 *ResOPC == ISD::VP_REDUCE_SEQ_FMUL) {
8395 if (VPIntrin.getFastMathFlags().allowReassoc())
8396 return *ResOPC == ISD::VP_REDUCE_SEQ_FADD ? ISD::VP_REDUCE_FADD
8397 : ISD::VP_REDUCE_FMUL;
8398 }
8399
8400 return *ResOPC;
8401}
8402
8403void SelectionDAGBuilder::visitVPLoad(
8404 const VPIntrinsic &VPIntrin, EVT VT,
8405 const SmallVectorImpl<SDValue> &OpValues) {
8406 SDLoc DL = getCurSDLoc();
8407 Value *PtrOperand = VPIntrin.getArgOperand(i: 0);
8408 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
8409 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
8410 const MDNode *Ranges = getRangeMetadata(I: VPIntrin);
8411 SDValue LD;
8412 // Do not serialize variable-length loads of constant memory with
8413 // anything.
8414 if (!Alignment)
8415 Alignment = DAG.getEVTAlign(MemoryVT: VT);
8416 MemoryLocation ML = MemoryLocation::getAfter(Ptr: PtrOperand, AATags: AAInfo);
8417 bool AddToChain = !BatchAA || !BatchAA->pointsToConstantMemory(Loc: ML);
8418 SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
8419 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
8420 PtrInfo: MachinePointerInfo(PtrOperand), F: MachineMemOperand::MOLoad,
8421 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: *Alignment, AAInfo, Ranges);
8422 LD = DAG.getLoadVP(VT, dl: DL, Chain: InChain, Ptr: OpValues[0], Mask: OpValues[1], EVL: OpValues[2],
8423 MMO, IsExpanding: false /*IsExpanding */);
8424 if (AddToChain)
8425 PendingLoads.push_back(Elt: LD.getValue(R: 1));
8426 setValue(V: &VPIntrin, NewN: LD);
8427}
8428
8429void SelectionDAGBuilder::visitVPGather(
8430 const VPIntrinsic &VPIntrin, EVT VT,
8431 const SmallVectorImpl<SDValue> &OpValues) {
8432 SDLoc DL = getCurSDLoc();
8433 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8434 Value *PtrOperand = VPIntrin.getArgOperand(i: 0);
8435 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
8436 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
8437 const MDNode *Ranges = getRangeMetadata(I: VPIntrin);
8438 SDValue LD;
8439 if (!Alignment)
8440 Alignment = DAG.getEVTAlign(MemoryVT: VT.getScalarType());
8441 unsigned AS =
8442 PtrOperand->getType()->getScalarType()->getPointerAddressSpace();
8443 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
8444 PtrInfo: MachinePointerInfo(AS), F: MachineMemOperand::MOLoad,
8445 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: *Alignment, AAInfo, Ranges);
8446 SDValue Base, Index, Scale;
8447 ISD::MemIndexType IndexType;
8448 bool UniformBase = getUniformBase(Ptr: PtrOperand, Base, Index, IndexType, Scale,
8449 SDB: this, CurBB: VPIntrin.getParent(),
8450 ElemSize: VT.getScalarStoreSize());
8451 if (!UniformBase) {
8452 Base = DAG.getConstant(Val: 0, DL, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
8453 Index = getValue(V: PtrOperand);
8454 IndexType = ISD::SIGNED_SCALED;
8455 Scale = DAG.getTargetConstant(Val: 1, DL, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
8456 }
8457 EVT IdxVT = Index.getValueType();
8458 EVT EltTy = IdxVT.getVectorElementType();
8459 if (TLI.shouldExtendGSIndex(VT: IdxVT, EltTy)) {
8460 EVT NewIdxVT = IdxVT.changeVectorElementType(EltVT: EltTy);
8461 Index = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: NewIdxVT, Operand: Index);
8462 }
8463 LD = DAG.getGatherVP(
8464 DAG.getVTList(VT, MVT::Other), VT, DL,
8465 {DAG.getRoot(), Base, Index, Scale, OpValues[1], OpValues[2]}, MMO,
8466 IndexType);
8467 PendingLoads.push_back(Elt: LD.getValue(R: 1));
8468 setValue(V: &VPIntrin, NewN: LD);
8469}
8470
8471void SelectionDAGBuilder::visitVPStore(
8472 const VPIntrinsic &VPIntrin, const SmallVectorImpl<SDValue> &OpValues) {
8473 SDLoc DL = getCurSDLoc();
8474 Value *PtrOperand = VPIntrin.getArgOperand(i: 1);
8475 EVT VT = OpValues[0].getValueType();
8476 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
8477 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
8478 SDValue ST;
8479 if (!Alignment)
8480 Alignment = DAG.getEVTAlign(MemoryVT: VT);
8481 SDValue Ptr = OpValues[1];
8482 SDValue Offset = DAG.getUNDEF(VT: Ptr.getValueType());
8483 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
8484 PtrInfo: MachinePointerInfo(PtrOperand), F: MachineMemOperand::MOStore,
8485 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: *Alignment, AAInfo);
8486 ST = DAG.getStoreVP(Chain: getMemoryRoot(), dl: DL, Val: OpValues[0], Ptr, Offset,
8487 Mask: OpValues[2], EVL: OpValues[3], MemVT: VT, MMO, AM: ISD::UNINDEXED,
8488 /* IsTruncating */ false, /*IsCompressing*/ false);
8489 DAG.setRoot(ST);
8490 setValue(V: &VPIntrin, NewN: ST);
8491}
8492
8493void SelectionDAGBuilder::visitVPScatter(
8494 const VPIntrinsic &VPIntrin, const SmallVectorImpl<SDValue> &OpValues) {
8495 SDLoc DL = getCurSDLoc();
8496 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8497 Value *PtrOperand = VPIntrin.getArgOperand(i: 1);
8498 EVT VT = OpValues[0].getValueType();
8499 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
8500 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
8501 SDValue ST;
8502 if (!Alignment)
8503 Alignment = DAG.getEVTAlign(MemoryVT: VT.getScalarType());
8504 unsigned AS =
8505 PtrOperand->getType()->getScalarType()->getPointerAddressSpace();
8506 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
8507 PtrInfo: MachinePointerInfo(AS), F: MachineMemOperand::MOStore,
8508 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: *Alignment, AAInfo);
8509 SDValue Base, Index, Scale;
8510 ISD::MemIndexType IndexType;
8511 bool UniformBase = getUniformBase(Ptr: PtrOperand, Base, Index, IndexType, Scale,
8512 SDB: this, CurBB: VPIntrin.getParent(),
8513 ElemSize: VT.getScalarStoreSize());
8514 if (!UniformBase) {
8515 Base = DAG.getConstant(Val: 0, DL, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
8516 Index = getValue(V: PtrOperand);
8517 IndexType = ISD::SIGNED_SCALED;
8518 Scale =
8519 DAG.getTargetConstant(Val: 1, DL, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
8520 }
8521 EVT IdxVT = Index.getValueType();
8522 EVT EltTy = IdxVT.getVectorElementType();
8523 if (TLI.shouldExtendGSIndex(VT: IdxVT, EltTy)) {
8524 EVT NewIdxVT = IdxVT.changeVectorElementType(EltVT: EltTy);
8525 Index = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: NewIdxVT, Operand: Index);
8526 }
8527 ST = DAG.getScatterVP(DAG.getVTList(MVT::Other), VT, DL,
8528 {getMemoryRoot(), OpValues[0], Base, Index, Scale,
8529 OpValues[2], OpValues[3]},
8530 MMO, IndexType);
8531 DAG.setRoot(ST);
8532 setValue(V: &VPIntrin, NewN: ST);
8533}
8534
8535void SelectionDAGBuilder::visitVPStridedLoad(
8536 const VPIntrinsic &VPIntrin, EVT VT,
8537 const SmallVectorImpl<SDValue> &OpValues) {
8538 SDLoc DL = getCurSDLoc();
8539 Value *PtrOperand = VPIntrin.getArgOperand(i: 0);
8540 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
8541 if (!Alignment)
8542 Alignment = DAG.getEVTAlign(MemoryVT: VT.getScalarType());
8543 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
8544 const MDNode *Ranges = getRangeMetadata(I: VPIntrin);
8545 MemoryLocation ML = MemoryLocation::getAfter(Ptr: PtrOperand, AATags: AAInfo);
8546 bool AddToChain = !BatchAA || !BatchAA->pointsToConstantMemory(Loc: ML);
8547 SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
8548 unsigned AS = PtrOperand->getType()->getPointerAddressSpace();
8549 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
8550 PtrInfo: MachinePointerInfo(AS), F: MachineMemOperand::MOLoad,
8551 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: *Alignment, AAInfo, Ranges);
8552
8553 SDValue LD = DAG.getStridedLoadVP(VT, DL, Chain: InChain, Ptr: OpValues[0], Stride: OpValues[1],
8554 Mask: OpValues[2], EVL: OpValues[3], MMO,
8555 IsExpanding: false /*IsExpanding*/);
8556
8557 if (AddToChain)
8558 PendingLoads.push_back(Elt: LD.getValue(R: 1));
8559 setValue(V: &VPIntrin, NewN: LD);
8560}
8561
8562void SelectionDAGBuilder::visitVPStridedStore(
8563 const VPIntrinsic &VPIntrin, const SmallVectorImpl<SDValue> &OpValues) {
8564 SDLoc DL = getCurSDLoc();
8565 Value *PtrOperand = VPIntrin.getArgOperand(i: 1);
8566 EVT VT = OpValues[0].getValueType();
8567 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
8568 if (!Alignment)
8569 Alignment = DAG.getEVTAlign(MemoryVT: VT.getScalarType());
8570 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
8571 unsigned AS = PtrOperand->getType()->getPointerAddressSpace();
8572 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
8573 PtrInfo: MachinePointerInfo(AS), F: MachineMemOperand::MOStore,
8574 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: *Alignment, AAInfo);
8575
8576 SDValue ST = DAG.getStridedStoreVP(
8577 Chain: getMemoryRoot(), DL, Val: OpValues[0], Ptr: OpValues[1],
8578 Offset: DAG.getUNDEF(VT: OpValues[1].getValueType()), Stride: OpValues[2], Mask: OpValues[3],
8579 EVL: OpValues[4], MemVT: VT, MMO, AM: ISD::UNINDEXED, /*IsTruncating*/ false,
8580 /*IsCompressing*/ false);
8581
8582 DAG.setRoot(ST);
8583 setValue(V: &VPIntrin, NewN: ST);
8584}
8585
8586void SelectionDAGBuilder::visitVPCmp(const VPCmpIntrinsic &VPIntrin) {
8587 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8588 SDLoc DL = getCurSDLoc();
8589
8590 ISD::CondCode Condition;
8591 CmpInst::Predicate CondCode = VPIntrin.getPredicate();
8592 bool IsFP = VPIntrin.getOperand(i_nocapture: 0)->getType()->isFPOrFPVectorTy();
8593 if (IsFP) {
8594 // FIXME: Regular fcmps are FPMathOperators which may have fast-math (nnan)
8595 // flags, but calls that don't return floating-point types can't be
8596 // FPMathOperators, like vp.fcmp. This affects constrained fcmp too.
8597 Condition = getFCmpCondCode(Pred: CondCode);
8598 if (TM.Options.NoNaNsFPMath)
8599 Condition = getFCmpCodeWithoutNaN(CC: Condition);
8600 } else {
8601 Condition = getICmpCondCode(Pred: CondCode);
8602 }
8603
8604 SDValue Op1 = getValue(V: VPIntrin.getOperand(i_nocapture: 0));
8605 SDValue Op2 = getValue(V: VPIntrin.getOperand(i_nocapture: 1));
8606 // #2 is the condition code
8607 SDValue MaskOp = getValue(V: VPIntrin.getOperand(i_nocapture: 3));
8608 SDValue EVL = getValue(V: VPIntrin.getOperand(i_nocapture: 4));
8609 MVT EVLParamVT = TLI.getVPExplicitVectorLengthTy();
8610 assert(EVLParamVT.isScalarInteger() && EVLParamVT.bitsGE(MVT::i32) &&
8611 "Unexpected target EVL type");
8612 EVL = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: EVLParamVT, Operand: EVL);
8613
8614 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DL: DAG.getDataLayout(),
8615 Ty: VPIntrin.getType());
8616 setValue(V: &VPIntrin,
8617 NewN: DAG.getSetCCVP(DL, VT: DestVT, LHS: Op1, RHS: Op2, Cond: Condition, Mask: MaskOp, EVL));
8618}
8619
8620void SelectionDAGBuilder::visitVectorPredicationIntrinsic(
8621 const VPIntrinsic &VPIntrin) {
8622 SDLoc DL = getCurSDLoc();
8623 unsigned Opcode = getISDForVPIntrinsic(VPIntrin);
8624
8625 auto IID = VPIntrin.getIntrinsicID();
8626
8627 if (const auto *CmpI = dyn_cast<VPCmpIntrinsic>(Val: &VPIntrin))
8628 return visitVPCmp(VPIntrin: *CmpI);
8629
8630 SmallVector<EVT, 4> ValueVTs;
8631 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8632 ComputeValueVTs(TLI, DL: DAG.getDataLayout(), Ty: VPIntrin.getType(), ValueVTs);
8633 SDVTList VTs = DAG.getVTList(VTs: ValueVTs);
8634
8635 auto EVLParamPos = VPIntrinsic::getVectorLengthParamPos(IntrinsicID: IID);
8636
8637 MVT EVLParamVT = TLI.getVPExplicitVectorLengthTy();
8638 assert(EVLParamVT.isScalarInteger() && EVLParamVT.bitsGE(MVT::i32) &&
8639 "Unexpected target EVL type");
8640
8641 // Request operands.
8642 SmallVector<SDValue, 7> OpValues;
8643 for (unsigned I = 0; I < VPIntrin.arg_size(); ++I) {
8644 auto Op = getValue(V: VPIntrin.getArgOperand(i: I));
8645 if (I == EVLParamPos)
8646 Op = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: EVLParamVT, Operand: Op);
8647 OpValues.push_back(Elt: Op);
8648 }
8649
8650 switch (Opcode) {
8651 default: {
8652 SDNodeFlags SDFlags;
8653 if (auto *FPMO = dyn_cast<FPMathOperator>(Val: &VPIntrin))
8654 SDFlags.copyFMF(FPMO: *FPMO);
8655 SDValue Result = DAG.getNode(Opcode, DL, VTList: VTs, Ops: OpValues, Flags: SDFlags);
8656 setValue(V: &VPIntrin, NewN: Result);
8657 break;
8658 }
8659 case ISD::VP_LOAD:
8660 visitVPLoad(VPIntrin, VT: ValueVTs[0], OpValues);
8661 break;
8662 case ISD::VP_GATHER:
8663 visitVPGather(VPIntrin, VT: ValueVTs[0], OpValues);
8664 break;
8665 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
8666 visitVPStridedLoad(VPIntrin, VT: ValueVTs[0], OpValues);
8667 break;
8668 case ISD::VP_STORE:
8669 visitVPStore(VPIntrin, OpValues);
8670 break;
8671 case ISD::VP_SCATTER:
8672 visitVPScatter(VPIntrin, OpValues);
8673 break;
8674 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
8675 visitVPStridedStore(VPIntrin, OpValues);
8676 break;
8677 case ISD::VP_FMULADD: {
8678 assert(OpValues.size() == 5 && "Unexpected number of operands");
8679 SDNodeFlags SDFlags;
8680 if (auto *FPMO = dyn_cast<FPMathOperator>(Val: &VPIntrin))
8681 SDFlags.copyFMF(FPMO: *FPMO);
8682 if (TM.Options.AllowFPOpFusion != FPOpFusion::Strict &&
8683 TLI.isFMAFasterThanFMulAndFAdd(MF: DAG.getMachineFunction(), ValueVTs[0])) {
8684 setValue(V: &VPIntrin, NewN: DAG.getNode(Opcode: ISD::VP_FMA, DL, VTList: VTs, Ops: OpValues, Flags: SDFlags));
8685 } else {
8686 SDValue Mul = DAG.getNode(
8687 Opcode: ISD::VP_FMUL, DL, VTList: VTs,
8688 Ops: {OpValues[0], OpValues[1], OpValues[3], OpValues[4]}, Flags: SDFlags);
8689 SDValue Add =
8690 DAG.getNode(Opcode: ISD::VP_FADD, DL, VTList: VTs,
8691 Ops: {Mul, OpValues[2], OpValues[3], OpValues[4]}, Flags: SDFlags);
8692 setValue(V: &VPIntrin, NewN: Add);
8693 }
8694 break;
8695 }
8696 case ISD::VP_IS_FPCLASS: {
8697 const DataLayout DLayout = DAG.getDataLayout();
8698 EVT DestVT = TLI.getValueType(DL: DLayout, Ty: VPIntrin.getType());
8699 auto Constant = OpValues[1]->getAsZExtVal();
8700 SDValue Check = DAG.getTargetConstant(Constant, DL, MVT::i32);
8701 SDValue V = DAG.getNode(Opcode: ISD::VP_IS_FPCLASS, DL, VT: DestVT,
8702 Ops: {OpValues[0], Check, OpValues[2], OpValues[3]});
8703 setValue(V: &VPIntrin, NewN: V);
8704 return;
8705 }
8706 case ISD::VP_INTTOPTR: {
8707 SDValue N = OpValues[0];
8708 EVT DestVT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: VPIntrin.getType());
8709 EVT PtrMemVT = TLI.getMemValueType(DL: DAG.getDataLayout(), Ty: VPIntrin.getType());
8710 N = DAG.getVPPtrExtOrTrunc(DL: getCurSDLoc(), VT: DestVT, Op: N, Mask: OpValues[1],
8711 EVL: OpValues[2]);
8712 N = DAG.getVPZExtOrTrunc(DL: getCurSDLoc(), VT: PtrMemVT, Op: N, Mask: OpValues[1],
8713 EVL: OpValues[2]);
8714 setValue(V: &VPIntrin, NewN: N);
8715 break;
8716 }
8717 case ISD::VP_PTRTOINT: {
8718 SDValue N = OpValues[0];
8719 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DL: DAG.getDataLayout(),
8720 Ty: VPIntrin.getType());
8721 EVT PtrMemVT = TLI.getMemValueType(DL: DAG.getDataLayout(),
8722 Ty: VPIntrin.getOperand(i_nocapture: 0)->getType());
8723 N = DAG.getVPPtrExtOrTrunc(DL: getCurSDLoc(), VT: PtrMemVT, Op: N, Mask: OpValues[1],
8724 EVL: OpValues[2]);
8725 N = DAG.getVPZExtOrTrunc(DL: getCurSDLoc(), VT: DestVT, Op: N, Mask: OpValues[1],
8726 EVL: OpValues[2]);
8727 setValue(V: &VPIntrin, NewN: N);
8728 break;
8729 }
8730 case ISD::VP_ABS:
8731 case ISD::VP_CTLZ:
8732 case ISD::VP_CTLZ_ZERO_UNDEF:
8733 case ISD::VP_CTTZ:
8734 case ISD::VP_CTTZ_ZERO_UNDEF:
8735 case ISD::VP_CTTZ_ELTS_ZERO_UNDEF:
8736 case ISD::VP_CTTZ_ELTS: {
8737 SDValue Result =
8738 DAG.getNode(Opcode, DL, VTList: VTs, Ops: {OpValues[0], OpValues[2], OpValues[3]});
8739 setValue(V: &VPIntrin, NewN: Result);
8740 break;
8741 }
8742 }
8743}
8744
8745SDValue SelectionDAGBuilder::lowerStartEH(SDValue Chain,
8746 const BasicBlock *EHPadBB,
8747 MCSymbol *&BeginLabel) {
8748 MachineFunction &MF = DAG.getMachineFunction();
8749
8750 // Insert a label before the invoke call to mark the try range. This can be
8751 // used to detect deletion of the invoke via the MachineModuleInfo.
8752 BeginLabel = MF.getContext().createTempSymbol();
8753
8754 // For SjLj, keep track of which landing pads go with which invokes
8755 // so as to maintain the ordering of pads in the LSDA.
8756 unsigned CallSiteIndex = FuncInfo.getCurrentCallSite();
8757 if (CallSiteIndex) {
8758 MF.setCallSiteBeginLabel(BeginLabel, Site: CallSiteIndex);
8759 LPadToCallSiteMap[FuncInfo.getMBB(BB: EHPadBB)].push_back(Elt: CallSiteIndex);
8760
8761 // Now that the call site is handled, stop tracking it.
8762 FuncInfo.setCurrentCallSite(0);
8763 }
8764
8765 return DAG.getEHLabel(dl: getCurSDLoc(), Root: Chain, Label: BeginLabel);
8766}
8767
8768SDValue SelectionDAGBuilder::lowerEndEH(SDValue Chain, const InvokeInst *II,
8769 const BasicBlock *EHPadBB,
8770 MCSymbol *BeginLabel) {
8771 assert(BeginLabel && "BeginLabel should've been set");
8772
8773 MachineFunction &MF = DAG.getMachineFunction();
8774
8775 // Insert a label at the end of the invoke call to mark the try range. This
8776 // can be used to detect deletion of the invoke via the MachineModuleInfo.
8777 MCSymbol *EndLabel = MF.getContext().createTempSymbol();
8778 Chain = DAG.getEHLabel(dl: getCurSDLoc(), Root: Chain, Label: EndLabel);
8779
8780 // Inform MachineModuleInfo of range.
8781 auto Pers = classifyEHPersonality(Pers: FuncInfo.Fn->getPersonalityFn());
8782 // There is a platform (e.g. wasm) that uses funclet style IR but does not
8783 // actually use outlined funclets and their LSDA info style.
8784 if (MF.hasEHFunclets() && isFuncletEHPersonality(Pers)) {
8785 assert(II && "II should've been set");
8786 WinEHFuncInfo *EHInfo = MF.getWinEHFuncInfo();
8787 EHInfo->addIPToStateRange(II, InvokeBegin: BeginLabel, InvokeEnd: EndLabel);
8788 } else if (!isScopedEHPersonality(Pers)) {
8789 assert(EHPadBB);
8790 MF.addInvoke(LandingPad: FuncInfo.getMBB(BB: EHPadBB), BeginLabel, EndLabel);
8791 }
8792
8793 return Chain;
8794}
8795
8796std::pair<SDValue, SDValue>
8797SelectionDAGBuilder::lowerInvokable(TargetLowering::CallLoweringInfo &CLI,
8798 const BasicBlock *EHPadBB) {
8799 MCSymbol *BeginLabel = nullptr;
8800
8801 if (EHPadBB) {
8802 // Both PendingLoads and PendingExports must be flushed here;
8803 // this call might not return.
8804 (void)getRoot();
8805 DAG.setRoot(lowerStartEH(Chain: getControlRoot(), EHPadBB, BeginLabel));
8806 CLI.setChain(getRoot());
8807 }
8808
8809 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8810 std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
8811
8812 assert((CLI.IsTailCall || Result.second.getNode()) &&
8813 "Non-null chain expected with non-tail call!");
8814 assert((Result.second.getNode() || !Result.first.getNode()) &&
8815 "Null value expected with tail call!");
8816
8817 if (!Result.second.getNode()) {
8818 // As a special case, a null chain means that a tail call has been emitted
8819 // and the DAG root is already updated.
8820 HasTailCall = true;
8821
8822 // Since there's no actual continuation from this block, nothing can be
8823 // relying on us setting vregs for them.
8824 PendingExports.clear();
8825 } else {
8826 DAG.setRoot(Result.second);
8827 }
8828
8829 if (EHPadBB) {
8830 DAG.setRoot(lowerEndEH(Chain: getRoot(), II: cast_or_null<InvokeInst>(Val: CLI.CB), EHPadBB,
8831 BeginLabel));
8832 Result.second = getRoot();
8833 }
8834
8835 return Result;
8836}
8837
8838void SelectionDAGBuilder::LowerCallTo(const CallBase &CB, SDValue Callee,
8839 bool isTailCall, bool isMustTailCall,
8840 const BasicBlock *EHPadBB,
8841 const TargetLowering::PtrAuthInfo *PAI) {
8842 auto &DL = DAG.getDataLayout();
8843 FunctionType *FTy = CB.getFunctionType();
8844 Type *RetTy = CB.getType();
8845
8846 TargetLowering::ArgListTy Args;
8847 Args.reserve(n: CB.arg_size());
8848
8849 const Value *SwiftErrorVal = nullptr;
8850 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8851
8852 if (isTailCall) {
8853 // Avoid emitting tail calls in functions with the disable-tail-calls
8854 // attribute.
8855 auto *Caller = CB.getParent()->getParent();
8856 if (Caller->getFnAttribute(Kind: "disable-tail-calls").getValueAsString() ==
8857 "true" && !isMustTailCall)
8858 isTailCall = false;
8859
8860 // We can't tail call inside a function with a swifterror argument. Lowering
8861 // does not support this yet. It would have to move into the swifterror
8862 // register before the call.
8863 if (TLI.supportSwiftError() &&
8864 Caller->getAttributes().hasAttrSomewhere(Attribute::SwiftError))
8865 isTailCall = false;
8866 }
8867
8868 for (auto I = CB.arg_begin(), E = CB.arg_end(); I != E; ++I) {
8869 TargetLowering::ArgListEntry Entry;
8870 const Value *V = *I;
8871
8872 // Skip empty types
8873 if (V->getType()->isEmptyTy())
8874 continue;
8875
8876 SDValue ArgNode = getValue(V);
8877 Entry.Node = ArgNode; Entry.Ty = V->getType();
8878
8879 Entry.setAttributes(Call: &CB, ArgIdx: I - CB.arg_begin());
8880
8881 // Use swifterror virtual register as input to the call.
8882 if (Entry.IsSwiftError && TLI.supportSwiftError()) {
8883 SwiftErrorVal = V;
8884 // We find the virtual register for the actual swifterror argument.
8885 // Instead of using the Value, we use the virtual register instead.
8886 Entry.Node =
8887 DAG.getRegister(Reg: SwiftError.getOrCreateVRegUseAt(&CB, FuncInfo.MBB, V),
8888 VT: EVT(TLI.getPointerTy(DL)));
8889 }
8890
8891 Args.push_back(x: Entry);
8892
8893 // If we have an explicit sret argument that is an Instruction, (i.e., it
8894 // might point to function-local memory), we can't meaningfully tail-call.
8895 if (Entry.IsSRet && isa<Instruction>(Val: V))
8896 isTailCall = false;
8897 }
8898
8899 // If call site has a cfguardtarget operand bundle, create and add an
8900 // additional ArgListEntry.
8901 if (auto Bundle = CB.getOperandBundle(ID: LLVMContext::OB_cfguardtarget)) {
8902 TargetLowering::ArgListEntry Entry;
8903 Value *V = Bundle->Inputs[0];
8904 SDValue ArgNode = getValue(V);
8905 Entry.Node = ArgNode;
8906 Entry.Ty = V->getType();
8907 Entry.IsCFGuardTarget = true;
8908 Args.push_back(x: Entry);
8909 }
8910
8911 // Check if target-independent constraints permit a tail call here.
8912 // Target-dependent constraints are checked within TLI->LowerCallTo.
8913 if (isTailCall && !isInTailCallPosition(Call: CB, TM: DAG.getTarget()))
8914 isTailCall = false;
8915
8916 // Disable tail calls if there is an swifterror argument. Targets have not
8917 // been updated to support tail calls.
8918 if (TLI.supportSwiftError() && SwiftErrorVal)
8919 isTailCall = false;
8920
8921 ConstantInt *CFIType = nullptr;
8922 if (CB.isIndirectCall()) {
8923 if (auto Bundle = CB.getOperandBundle(ID: LLVMContext::OB_kcfi)) {
8924 if (!TLI.supportKCFIBundles())
8925 report_fatal_error(
8926 reason: "Target doesn't support calls with kcfi operand bundles.");
8927 CFIType = cast<ConstantInt>(Val: Bundle->Inputs[0]);
8928 assert(CFIType->getType()->isIntegerTy(32) && "Invalid CFI type");
8929 }
8930 }
8931
8932 SDValue ConvControlToken;
8933 if (auto Bundle = CB.getOperandBundle(ID: LLVMContext::OB_convergencectrl)) {
8934 auto *Token = Bundle->Inputs[0].get();
8935 ConvControlToken = getValue(V: Token);
8936 }
8937
8938 TargetLowering::CallLoweringInfo CLI(DAG);
8939 CLI.setDebugLoc(getCurSDLoc())
8940 .setChain(getRoot())
8941 .setCallee(ResultType: RetTy, FTy, Target: Callee, ArgsList: std::move(Args), Call: CB)
8942 .setTailCall(isTailCall)
8943 .setConvergent(CB.isConvergent())
8944 .setIsPreallocated(
8945 CB.countOperandBundlesOfType(ID: LLVMContext::OB_preallocated) != 0)
8946 .setCFIType(CFIType)
8947 .setConvergenceControlToken(ConvControlToken);
8948
8949 // Set the pointer authentication info if we have it.
8950 if (PAI) {
8951 if (!TLI.supportPtrAuthBundles())
8952 report_fatal_error(
8953 reason: "This target doesn't support calls with ptrauth operand bundles.");
8954 CLI.setPtrAuth(*PAI);
8955 }
8956
8957 std::pair<SDValue, SDValue> Result = lowerInvokable(CLI, EHPadBB);
8958
8959 if (Result.first.getNode()) {
8960 Result.first = lowerRangeToAssertZExt(DAG, I: CB, Op: Result.first);
8961 setValue(V: &CB, NewN: Result.first);
8962 }
8963
8964 // The last element of CLI.InVals has the SDValue for swifterror return.
8965 // Here we copy it to a virtual register and update SwiftErrorMap for
8966 // book-keeping.
8967 if (SwiftErrorVal && TLI.supportSwiftError()) {
8968 // Get the last element of InVals.
8969 SDValue Src = CLI.InVals.back();
8970 Register VReg =
8971 SwiftError.getOrCreateVRegDefAt(&CB, FuncInfo.MBB, SwiftErrorVal);
8972 SDValue CopyNode = CLI.DAG.getCopyToReg(Chain: Result.second, dl: CLI.DL, Reg: VReg, N: Src);
8973 DAG.setRoot(CopyNode);
8974 }
8975}
8976
8977static SDValue getMemCmpLoad(const Value *PtrVal, MVT LoadVT,
8978 SelectionDAGBuilder &Builder) {
8979 // Check to see if this load can be trivially constant folded, e.g. if the
8980 // input is from a string literal.
8981 if (const Constant *LoadInput = dyn_cast<Constant>(Val: PtrVal)) {
8982 // Cast pointer to the type we really want to load.
8983 Type *LoadTy =
8984 Type::getIntNTy(C&: PtrVal->getContext(), N: LoadVT.getScalarSizeInBits());
8985 if (LoadVT.isVector())
8986 LoadTy = FixedVectorType::get(ElementType: LoadTy, NumElts: LoadVT.getVectorNumElements());
8987 if (const Constant *LoadCst =
8988 ConstantFoldLoadFromConstPtr(C: const_cast<Constant *>(LoadInput),
8989 Ty: LoadTy, DL: Builder.DAG.getDataLayout()))
8990 return Builder.getValue(V: LoadCst);
8991 }
8992
8993 // Otherwise, we have to emit the load. If the pointer is to unfoldable but
8994 // still constant memory, the input chain can be the entry node.
8995 SDValue Root;
8996 bool ConstantMemory = false;
8997
8998 // Do not serialize (non-volatile) loads of constant memory with anything.
8999 if (Builder.BatchAA && Builder.BatchAA->pointsToConstantMemory(P: PtrVal)) {
9000 Root = Builder.DAG.getEntryNode();
9001 ConstantMemory = true;
9002 } else {
9003 // Do not serialize non-volatile loads against each other.
9004 Root = Builder.DAG.getRoot();
9005 }
9006
9007 SDValue Ptr = Builder.getValue(V: PtrVal);
9008 SDValue LoadVal =
9009 Builder.DAG.getLoad(VT: LoadVT, dl: Builder.getCurSDLoc(), Chain: Root, Ptr,
9010 PtrInfo: MachinePointerInfo(PtrVal), Alignment: Align(1));
9011
9012 if (!ConstantMemory)
9013 Builder.PendingLoads.push_back(Elt: LoadVal.getValue(R: 1));
9014 return LoadVal;
9015}
9016
9017/// Record the value for an instruction that produces an integer result,
9018/// converting the type where necessary.
9019void SelectionDAGBuilder::processIntegerCallValue(const Instruction &I,
9020 SDValue Value,
9021 bool IsSigned) {
9022 EVT VT = DAG.getTargetLoweringInfo().getValueType(DL: DAG.getDataLayout(),
9023 Ty: I.getType(), AllowUnknown: true);
9024 Value = DAG.getExtOrTrunc(IsSigned, Op: Value, DL: getCurSDLoc(), VT);
9025 setValue(V: &I, NewN: Value);
9026}
9027
9028/// See if we can lower a memcmp/bcmp call into an optimized form. If so, return
9029/// true and lower it. Otherwise return false, and it will be lowered like a
9030/// normal call.
9031/// The caller already checked that \p I calls the appropriate LibFunc with a
9032/// correct prototype.
9033bool SelectionDAGBuilder::visitMemCmpBCmpCall(const CallInst &I) {
9034 const Value *LHS = I.getArgOperand(i: 0), *RHS = I.getArgOperand(i: 1);
9035 const Value *Size = I.getArgOperand(i: 2);
9036 const ConstantSDNode *CSize = dyn_cast<ConstantSDNode>(Val: getValue(V: Size));
9037 if (CSize && CSize->getZExtValue() == 0) {
9038 EVT CallVT = DAG.getTargetLoweringInfo().getValueType(DL: DAG.getDataLayout(),
9039 Ty: I.getType(), AllowUnknown: true);
9040 setValue(V: &I, NewN: DAG.getConstant(Val: 0, DL: getCurSDLoc(), VT: CallVT));
9041 return true;
9042 }
9043
9044 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
9045 std::pair<SDValue, SDValue> Res = TSI.EmitTargetCodeForMemcmp(
9046 DAG, dl: getCurSDLoc(), Chain: DAG.getRoot(), Op1: getValue(V: LHS), Op2: getValue(V: RHS),
9047 Op3: getValue(V: Size), Op1PtrInfo: MachinePointerInfo(LHS), Op2PtrInfo: MachinePointerInfo(RHS));
9048 if (Res.first.getNode()) {
9049 processIntegerCallValue(I, Value: Res.first, IsSigned: true);
9050 PendingLoads.push_back(Elt: Res.second);
9051 return true;
9052 }
9053
9054 // memcmp(S1,S2,2) != 0 -> (*(short*)LHS != *(short*)RHS) != 0
9055 // memcmp(S1,S2,4) != 0 -> (*(int*)LHS != *(int*)RHS) != 0
9056 if (!CSize || !isOnlyUsedInZeroEqualityComparison(CxtI: &I))
9057 return false;
9058
9059 // If the target has a fast compare for the given size, it will return a
9060 // preferred load type for that size. Require that the load VT is legal and
9061 // that the target supports unaligned loads of that type. Otherwise, return
9062 // INVALID.
9063 auto hasFastLoadsAndCompare = [&](unsigned NumBits) {
9064 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9065 MVT LVT = TLI.hasFastEqualityCompare(NumBits);
9066 if (LVT != MVT::INVALID_SIMPLE_VALUE_TYPE) {
9067 // TODO: Handle 5 byte compare as 4-byte + 1 byte.
9068 // TODO: Handle 8 byte compare on x86-32 as two 32-bit loads.
9069 // TODO: Check alignment of src and dest ptrs.
9070 unsigned DstAS = LHS->getType()->getPointerAddressSpace();
9071 unsigned SrcAS = RHS->getType()->getPointerAddressSpace();
9072 if (!TLI.isTypeLegal(VT: LVT) ||
9073 !TLI.allowsMisalignedMemoryAccesses(LVT, AddrSpace: SrcAS) ||
9074 !TLI.allowsMisalignedMemoryAccesses(LVT, AddrSpace: DstAS))
9075 LVT = MVT::INVALID_SIMPLE_VALUE_TYPE;
9076 }
9077
9078 return LVT;
9079 };
9080
9081 // This turns into unaligned loads. We only do this if the target natively
9082 // supports the MVT we'll be loading or if it is small enough (<= 4) that
9083 // we'll only produce a small number of byte loads.
9084 MVT LoadVT;
9085 unsigned NumBitsToCompare = CSize->getZExtValue() * 8;
9086 switch (NumBitsToCompare) {
9087 default:
9088 return false;
9089 case 16:
9090 LoadVT = MVT::i16;
9091 break;
9092 case 32:
9093 LoadVT = MVT::i32;
9094 break;
9095 case 64:
9096 case 128:
9097 case 256:
9098 LoadVT = hasFastLoadsAndCompare(NumBitsToCompare);
9099 break;
9100 }
9101
9102 if (LoadVT == MVT::INVALID_SIMPLE_VALUE_TYPE)
9103 return false;
9104
9105 SDValue LoadL = getMemCmpLoad(PtrVal: LHS, LoadVT, Builder&: *this);
9106 SDValue LoadR = getMemCmpLoad(PtrVal: RHS, LoadVT, Builder&: *this);
9107
9108 // Bitcast to a wide integer type if the loads are vectors.
9109 if (LoadVT.isVector()) {
9110 EVT CmpVT = EVT::getIntegerVT(Context&: LHS->getContext(), BitWidth: LoadVT.getSizeInBits());
9111 LoadL = DAG.getBitcast(VT: CmpVT, V: LoadL);
9112 LoadR = DAG.getBitcast(VT: CmpVT, V: LoadR);
9113 }
9114
9115 SDValue Cmp = DAG.getSetCC(getCurSDLoc(), MVT::i1, LoadL, LoadR, ISD::SETNE);
9116 processIntegerCallValue(I, Value: Cmp, IsSigned: false);
9117 return true;
9118}
9119
9120/// See if we can lower a memchr call into an optimized form. If so, return
9121/// true and lower it. Otherwise return false, and it will be lowered like a
9122/// normal call.
9123/// The caller already checked that \p I calls the appropriate LibFunc with a
9124/// correct prototype.
9125bool SelectionDAGBuilder::visitMemChrCall(const CallInst &I) {
9126 const Value *Src = I.getArgOperand(i: 0);
9127 const Value *Char = I.getArgOperand(i: 1);
9128 const Value *Length = I.getArgOperand(i: 2);
9129
9130 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
9131 std::pair<SDValue, SDValue> Res =
9132 TSI.EmitTargetCodeForMemchr(DAG, dl: getCurSDLoc(), Chain: DAG.getRoot(),
9133 Src: getValue(V: Src), Char: getValue(V: Char), Length: getValue(V: Length),
9134 SrcPtrInfo: MachinePointerInfo(Src));
9135 if (Res.first.getNode()) {
9136 setValue(V: &I, NewN: Res.first);
9137 PendingLoads.push_back(Elt: Res.second);
9138 return true;
9139 }
9140
9141 return false;
9142}
9143
9144/// See if we can lower a mempcpy call into an optimized form. If so, return
9145/// true and lower it. Otherwise return false, and it will be lowered like a
9146/// normal call.
9147/// The caller already checked that \p I calls the appropriate LibFunc with a
9148/// correct prototype.
9149bool SelectionDAGBuilder::visitMemPCpyCall(const CallInst &I) {
9150 SDValue Dst = getValue(V: I.getArgOperand(i: 0));
9151 SDValue Src = getValue(V: I.getArgOperand(i: 1));
9152 SDValue Size = getValue(V: I.getArgOperand(i: 2));
9153
9154 Align DstAlign = DAG.InferPtrAlign(Ptr: Dst).valueOrOne();
9155 Align SrcAlign = DAG.InferPtrAlign(Ptr: Src).valueOrOne();
9156 // DAG::getMemcpy needs Alignment to be defined.
9157 Align Alignment = std::min(a: DstAlign, b: SrcAlign);
9158
9159 SDLoc sdl = getCurSDLoc();
9160
9161 // In the mempcpy context we need to pass in a false value for isTailCall
9162 // because the return pointer needs to be adjusted by the size of
9163 // the copied memory.
9164 SDValue Root = getMemoryRoot();
9165 SDValue MC = DAG.getMemcpy(
9166 Chain: Root, dl: sdl, Dst, Src, Size, Alignment, isVol: false, AlwaysInline: false, /*CI=*/nullptr,
9167 OverrideTailCall: std::nullopt, DstPtrInfo: MachinePointerInfo(I.getArgOperand(i: 0)),
9168 SrcPtrInfo: MachinePointerInfo(I.getArgOperand(i: 1)), AAInfo: I.getAAMetadata());
9169 assert(MC.getNode() != nullptr &&
9170 "** memcpy should not be lowered as TailCall in mempcpy context **");
9171 DAG.setRoot(MC);
9172
9173 // Check if Size needs to be truncated or extended.
9174 Size = DAG.getSExtOrTrunc(Op: Size, DL: sdl, VT: Dst.getValueType());
9175
9176 // Adjust return pointer to point just past the last dst byte.
9177 SDValue DstPlusSize = DAG.getMemBasePlusOffset(Base: Dst, Offset: Size, DL: sdl);
9178 setValue(V: &I, NewN: DstPlusSize);
9179 return true;
9180}
9181
9182/// See if we can lower a strcpy call into an optimized form. If so, return
9183/// true and lower it, otherwise return false and it will be lowered like a
9184/// normal call.
9185/// The caller already checked that \p I calls the appropriate LibFunc with a
9186/// correct prototype.
9187bool SelectionDAGBuilder::visitStrCpyCall(const CallInst &I, bool isStpcpy) {
9188 const Value *Arg0 = I.getArgOperand(i: 0), *Arg1 = I.getArgOperand(i: 1);
9189
9190 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
9191 std::pair<SDValue, SDValue> Res =
9192 TSI.EmitTargetCodeForStrcpy(DAG, DL: getCurSDLoc(), Chain: getRoot(),
9193 Dest: getValue(V: Arg0), Src: getValue(V: Arg1),
9194 DestPtrInfo: MachinePointerInfo(Arg0),
9195 SrcPtrInfo: MachinePointerInfo(Arg1), isStpcpy);
9196 if (Res.first.getNode()) {
9197 setValue(V: &I, NewN: Res.first);
9198 DAG.setRoot(Res.second);
9199 return true;
9200 }
9201
9202 return false;
9203}
9204
9205/// See if we can lower a strcmp call into an optimized form. If so, return
9206/// true and lower it, otherwise return false and it will be lowered like a
9207/// normal call.
9208/// The caller already checked that \p I calls the appropriate LibFunc with a
9209/// correct prototype.
9210bool SelectionDAGBuilder::visitStrCmpCall(const CallInst &I) {
9211 const Value *Arg0 = I.getArgOperand(i: 0), *Arg1 = I.getArgOperand(i: 1);
9212
9213 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
9214 std::pair<SDValue, SDValue> Res =
9215 TSI.EmitTargetCodeForStrcmp(DAG, dl: getCurSDLoc(), Chain: DAG.getRoot(),
9216 Op1: getValue(V: Arg0), Op2: getValue(V: Arg1),
9217 Op1PtrInfo: MachinePointerInfo(Arg0),
9218 Op2PtrInfo: MachinePointerInfo(Arg1));
9219 if (Res.first.getNode()) {
9220 processIntegerCallValue(I, Value: Res.first, IsSigned: true);
9221 PendingLoads.push_back(Elt: Res.second);
9222 return true;
9223 }
9224
9225 return false;
9226}
9227
9228/// See if we can lower a strlen call into an optimized form. If so, return
9229/// true and lower it, otherwise return false and it will be lowered like a
9230/// normal call.
9231/// The caller already checked that \p I calls the appropriate LibFunc with a
9232/// correct prototype.
9233bool SelectionDAGBuilder::visitStrLenCall(const CallInst &I) {
9234 const Value *Arg0 = I.getArgOperand(i: 0);
9235
9236 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
9237 std::pair<SDValue, SDValue> Res =
9238 TSI.EmitTargetCodeForStrlen(DAG, DL: getCurSDLoc(), Chain: DAG.getRoot(),
9239 Src: getValue(V: Arg0), SrcPtrInfo: MachinePointerInfo(Arg0));
9240 if (Res.first.getNode()) {
9241 processIntegerCallValue(I, Value: Res.first, IsSigned: false);
9242 PendingLoads.push_back(Elt: Res.second);
9243 return true;
9244 }
9245
9246 return false;
9247}
9248
9249/// See if we can lower a strnlen call into an optimized form. If so, return
9250/// true and lower it, otherwise return false and it will be lowered like a
9251/// normal call.
9252/// The caller already checked that \p I calls the appropriate LibFunc with a
9253/// correct prototype.
9254bool SelectionDAGBuilder::visitStrNLenCall(const CallInst &I) {
9255 const Value *Arg0 = I.getArgOperand(i: 0), *Arg1 = I.getArgOperand(i: 1);
9256
9257 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
9258 std::pair<SDValue, SDValue> Res =
9259 TSI.EmitTargetCodeForStrnlen(DAG, DL: getCurSDLoc(), Chain: DAG.getRoot(),
9260 Src: getValue(V: Arg0), MaxLength: getValue(V: Arg1),
9261 SrcPtrInfo: MachinePointerInfo(Arg0));
9262 if (Res.first.getNode()) {
9263 processIntegerCallValue(I, Value: Res.first, IsSigned: false);
9264 PendingLoads.push_back(Elt: Res.second);
9265 return true;
9266 }
9267
9268 return false;
9269}
9270
9271/// See if we can lower a unary floating-point operation into an SDNode with
9272/// the specified Opcode. If so, return true and lower it, otherwise return
9273/// false and it will be lowered like a normal call.
9274/// The caller already checked that \p I calls the appropriate LibFunc with a
9275/// correct prototype.
9276bool SelectionDAGBuilder::visitUnaryFloatCall(const CallInst &I,
9277 unsigned Opcode) {
9278 // We already checked this call's prototype; verify it doesn't modify errno.
9279 if (!I.onlyReadsMemory())
9280 return false;
9281
9282 SDNodeFlags Flags;
9283 Flags.copyFMF(FPMO: cast<FPMathOperator>(Val: I));
9284
9285 SDValue Tmp = getValue(V: I.getArgOperand(i: 0));
9286 setValue(V: &I,
9287 NewN: DAG.getNode(Opcode, DL: getCurSDLoc(), VT: Tmp.getValueType(), Operand: Tmp, Flags));
9288 return true;
9289}
9290
9291/// See if we can lower a binary floating-point operation into an SDNode with
9292/// the specified Opcode. If so, return true and lower it. Otherwise return
9293/// false, and it will be lowered like a normal call.
9294/// The caller already checked that \p I calls the appropriate LibFunc with a
9295/// correct prototype.
9296bool SelectionDAGBuilder::visitBinaryFloatCall(const CallInst &I,
9297 unsigned Opcode) {
9298 // We already checked this call's prototype; verify it doesn't modify errno.
9299 if (!I.onlyReadsMemory())
9300 return false;
9301
9302 SDNodeFlags Flags;
9303 Flags.copyFMF(FPMO: cast<FPMathOperator>(Val: I));
9304
9305 SDValue Tmp0 = getValue(V: I.getArgOperand(i: 0));
9306 SDValue Tmp1 = getValue(V: I.getArgOperand(i: 1));
9307 EVT VT = Tmp0.getValueType();
9308 setValue(V: &I, NewN: DAG.getNode(Opcode, DL: getCurSDLoc(), VT, N1: Tmp0, N2: Tmp1, Flags));
9309 return true;
9310}
9311
9312void SelectionDAGBuilder::visitCall(const CallInst &I) {
9313 // Handle inline assembly differently.
9314 if (I.isInlineAsm()) {
9315 visitInlineAsm(Call: I);
9316 return;
9317 }
9318
9319 diagnoseDontCall(CI: I);
9320
9321 if (Function *F = I.getCalledFunction()) {
9322 if (F->isDeclaration()) {
9323 // Is this an LLVM intrinsic?
9324 if (unsigned IID = F->getIntrinsicID()) {
9325 visitIntrinsicCall(I, Intrinsic: IID);
9326 return;
9327 }
9328 }
9329
9330 // Check for well-known libc/libm calls. If the function is internal, it
9331 // can't be a library call. Don't do the check if marked as nobuiltin for
9332 // some reason or the call site requires strict floating point semantics.
9333 LibFunc Func;
9334 if (!I.isNoBuiltin() && !I.isStrictFP() && !F->hasLocalLinkage() &&
9335 F->hasName() && LibInfo->getLibFunc(FDecl: *F, F&: Func) &&
9336 LibInfo->hasOptimizedCodeGen(F: Func)) {
9337 switch (Func) {
9338 default: break;
9339 case LibFunc_bcmp:
9340 if (visitMemCmpBCmpCall(I))
9341 return;
9342 break;
9343 case LibFunc_copysign:
9344 case LibFunc_copysignf:
9345 case LibFunc_copysignl:
9346 // We already checked this call's prototype; verify it doesn't modify
9347 // errno.
9348 if (I.onlyReadsMemory()) {
9349 SDValue LHS = getValue(V: I.getArgOperand(i: 0));
9350 SDValue RHS = getValue(V: I.getArgOperand(i: 1));
9351 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::FCOPYSIGN, DL: getCurSDLoc(),
9352 VT: LHS.getValueType(), N1: LHS, N2: RHS));
9353 return;
9354 }
9355 break;
9356 case LibFunc_fabs:
9357 case LibFunc_fabsf:
9358 case LibFunc_fabsl:
9359 if (visitUnaryFloatCall(I, Opcode: ISD::FABS))
9360 return;
9361 break;
9362 case LibFunc_fmin:
9363 case LibFunc_fminf:
9364 case LibFunc_fminl:
9365 if (visitBinaryFloatCall(I, Opcode: ISD::FMINNUM))
9366 return;
9367 break;
9368 case LibFunc_fmax:
9369 case LibFunc_fmaxf:
9370 case LibFunc_fmaxl:
9371 if (visitBinaryFloatCall(I, Opcode: ISD::FMAXNUM))
9372 return;
9373 break;
9374 case LibFunc_fminimum_num:
9375 case LibFunc_fminimum_numf:
9376 case LibFunc_fminimum_numl:
9377 if (visitBinaryFloatCall(I, Opcode: ISD::FMINIMUMNUM))
9378 return;
9379 break;
9380 case LibFunc_fmaximum_num:
9381 case LibFunc_fmaximum_numf:
9382 case LibFunc_fmaximum_numl:
9383 if (visitBinaryFloatCall(I, Opcode: ISD::FMAXIMUMNUM))
9384 return;
9385 break;
9386 case LibFunc_sin:
9387 case LibFunc_sinf:
9388 case LibFunc_sinl:
9389 if (visitUnaryFloatCall(I, Opcode: ISD::FSIN))
9390 return;
9391 break;
9392 case LibFunc_cos:
9393 case LibFunc_cosf:
9394 case LibFunc_cosl:
9395 if (visitUnaryFloatCall(I, Opcode: ISD::FCOS))
9396 return;
9397 break;
9398 case LibFunc_tan:
9399 case LibFunc_tanf:
9400 case LibFunc_tanl:
9401 if (visitUnaryFloatCall(I, Opcode: ISD::FTAN))
9402 return;
9403 break;
9404 case LibFunc_asin:
9405 case LibFunc_asinf:
9406 case LibFunc_asinl:
9407 if (visitUnaryFloatCall(I, Opcode: ISD::FASIN))
9408 return;
9409 break;
9410 case LibFunc_acos:
9411 case LibFunc_acosf:
9412 case LibFunc_acosl:
9413 if (visitUnaryFloatCall(I, Opcode: ISD::FACOS))
9414 return;
9415 break;
9416 case LibFunc_atan:
9417 case LibFunc_atanf:
9418 case LibFunc_atanl:
9419 if (visitUnaryFloatCall(I, Opcode: ISD::FATAN))
9420 return;
9421 break;
9422 case LibFunc_atan2:
9423 case LibFunc_atan2f:
9424 case LibFunc_atan2l:
9425 if (visitBinaryFloatCall(I, Opcode: ISD::FATAN2))
9426 return;
9427 break;
9428 case LibFunc_sinh:
9429 case LibFunc_sinhf:
9430 case LibFunc_sinhl:
9431 if (visitUnaryFloatCall(I, Opcode: ISD::FSINH))
9432 return;
9433 break;
9434 case LibFunc_cosh:
9435 case LibFunc_coshf:
9436 case LibFunc_coshl:
9437 if (visitUnaryFloatCall(I, Opcode: ISD::FCOSH))
9438 return;
9439 break;
9440 case LibFunc_tanh:
9441 case LibFunc_tanhf:
9442 case LibFunc_tanhl:
9443 if (visitUnaryFloatCall(I, Opcode: ISD::FTANH))
9444 return;
9445 break;
9446 case LibFunc_sqrt:
9447 case LibFunc_sqrtf:
9448 case LibFunc_sqrtl:
9449 case LibFunc_sqrt_finite:
9450 case LibFunc_sqrtf_finite:
9451 case LibFunc_sqrtl_finite:
9452 if (visitUnaryFloatCall(I, Opcode: ISD::FSQRT))
9453 return;
9454 break;
9455 case LibFunc_floor:
9456 case LibFunc_floorf:
9457 case LibFunc_floorl:
9458 if (visitUnaryFloatCall(I, Opcode: ISD::FFLOOR))
9459 return;
9460 break;
9461 case LibFunc_nearbyint:
9462 case LibFunc_nearbyintf:
9463 case LibFunc_nearbyintl:
9464 if (visitUnaryFloatCall(I, Opcode: ISD::FNEARBYINT))
9465 return;
9466 break;
9467 case LibFunc_ceil:
9468 case LibFunc_ceilf:
9469 case LibFunc_ceill:
9470 if (visitUnaryFloatCall(I, Opcode: ISD::FCEIL))
9471 return;
9472 break;
9473 case LibFunc_rint:
9474 case LibFunc_rintf:
9475 case LibFunc_rintl:
9476 if (visitUnaryFloatCall(I, Opcode: ISD::FRINT))
9477 return;
9478 break;
9479 case LibFunc_round:
9480 case LibFunc_roundf:
9481 case LibFunc_roundl:
9482 if (visitUnaryFloatCall(I, Opcode: ISD::FROUND))
9483 return;
9484 break;
9485 case LibFunc_trunc:
9486 case LibFunc_truncf:
9487 case LibFunc_truncl:
9488 if (visitUnaryFloatCall(I, Opcode: ISD::FTRUNC))
9489 return;
9490 break;
9491 case LibFunc_log2:
9492 case LibFunc_log2f:
9493 case LibFunc_log2l:
9494 if (visitUnaryFloatCall(I, Opcode: ISD::FLOG2))
9495 return;
9496 break;
9497 case LibFunc_exp2:
9498 case LibFunc_exp2f:
9499 case LibFunc_exp2l:
9500 if (visitUnaryFloatCall(I, Opcode: ISD::FEXP2))
9501 return;
9502 break;
9503 case LibFunc_exp10:
9504 case LibFunc_exp10f:
9505 case LibFunc_exp10l:
9506 if (visitUnaryFloatCall(I, Opcode: ISD::FEXP10))
9507 return;
9508 break;
9509 case LibFunc_ldexp:
9510 case LibFunc_ldexpf:
9511 case LibFunc_ldexpl:
9512 if (visitBinaryFloatCall(I, Opcode: ISD::FLDEXP))
9513 return;
9514 break;
9515 case LibFunc_memcmp:
9516 if (visitMemCmpBCmpCall(I))
9517 return;
9518 break;
9519 case LibFunc_mempcpy:
9520 if (visitMemPCpyCall(I))
9521 return;
9522 break;
9523 case LibFunc_memchr:
9524 if (visitMemChrCall(I))
9525 return;
9526 break;
9527 case LibFunc_strcpy:
9528 if (visitStrCpyCall(I, isStpcpy: false))
9529 return;
9530 break;
9531 case LibFunc_stpcpy:
9532 if (visitStrCpyCall(I, isStpcpy: true))
9533 return;
9534 break;
9535 case LibFunc_strcmp:
9536 if (visitStrCmpCall(I))
9537 return;
9538 break;
9539 case LibFunc_strlen:
9540 if (visitStrLenCall(I))
9541 return;
9542 break;
9543 case LibFunc_strnlen:
9544 if (visitStrNLenCall(I))
9545 return;
9546 break;
9547 }
9548 }
9549 }
9550
9551 if (I.countOperandBundlesOfType(ID: LLVMContext::OB_ptrauth)) {
9552 LowerCallSiteWithPtrAuthBundle(CB: cast<CallBase>(Val: I), /*EHPadBB=*/nullptr);
9553 return;
9554 }
9555
9556 // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
9557 // have to do anything here to lower funclet bundles.
9558 // CFGuardTarget bundles are lowered in LowerCallTo.
9559 if (I.hasOperandBundlesOtherThan(
9560 IDs: {LLVMContext::OB_deopt, LLVMContext::OB_funclet,
9561 LLVMContext::OB_cfguardtarget, LLVMContext::OB_preallocated,
9562 LLVMContext::OB_clang_arc_attachedcall, LLVMContext::OB_kcfi,
9563 LLVMContext::OB_convergencectrl}))
9564 reportFatalUsageError(reason: "cannot lower calls with arbitrary operand bundles!");
9565
9566 SDValue Callee = getValue(V: I.getCalledOperand());
9567
9568 if (I.hasDeoptState())
9569 LowerCallSiteWithDeoptBundle(Call: &I, Callee, EHPadBB: nullptr);
9570 else
9571 // Check if we can potentially perform a tail call. More detailed checking
9572 // is be done within LowerCallTo, after more information about the call is
9573 // known.
9574 LowerCallTo(CB: I, Callee, isTailCall: I.isTailCall(), isMustTailCall: I.isMustTailCall());
9575}
9576
9577void SelectionDAGBuilder::LowerCallSiteWithPtrAuthBundle(
9578 const CallBase &CB, const BasicBlock *EHPadBB) {
9579 auto PAB = CB.getOperandBundle(Name: "ptrauth");
9580 const Value *CalleeV = CB.getCalledOperand();
9581
9582 // Gather the call ptrauth data from the operand bundle:
9583 // [ i32 <key>, i64 <discriminator> ]
9584 const auto *Key = cast<ConstantInt>(Val: PAB->Inputs[0]);
9585 const Value *Discriminator = PAB->Inputs[1];
9586
9587 assert(Key->getType()->isIntegerTy(32) && "Invalid ptrauth key");
9588 assert(Discriminator->getType()->isIntegerTy(64) &&
9589 "Invalid ptrauth discriminator");
9590
9591 // Look through ptrauth constants to find the raw callee.
9592 // Do a direct unauthenticated call if we found it and everything matches.
9593 if (const auto *CalleeCPA = dyn_cast<ConstantPtrAuth>(Val: CalleeV))
9594 if (CalleeCPA->isKnownCompatibleWith(Key, Discriminator,
9595 DL: DAG.getDataLayout()))
9596 return LowerCallTo(CB, Callee: getValue(V: CalleeCPA->getPointer()), isTailCall: CB.isTailCall(),
9597 isMustTailCall: CB.isMustTailCall(), EHPadBB);
9598
9599 // Functions should never be ptrauth-called directly.
9600 assert(!isa<Function>(CalleeV) && "invalid direct ptrauth call");
9601
9602 // Otherwise, do an authenticated indirect call.
9603 TargetLowering::PtrAuthInfo PAI = {.Key: Key->getZExtValue(),
9604 .Discriminator: getValue(V: Discriminator)};
9605
9606 LowerCallTo(CB, Callee: getValue(V: CalleeV), isTailCall: CB.isTailCall(), isMustTailCall: CB.isMustTailCall(),
9607 EHPadBB, PAI: &PAI);
9608}
9609
9610namespace {
9611
9612/// AsmOperandInfo - This contains information for each constraint that we are
9613/// lowering.
9614class SDISelAsmOperandInfo : public TargetLowering::AsmOperandInfo {
9615public:
9616 /// CallOperand - If this is the result output operand or a clobber
9617 /// this is null, otherwise it is the incoming operand to the CallInst.
9618 /// This gets modified as the asm is processed.
9619 SDValue CallOperand;
9620
9621 /// AssignedRegs - If this is a register or register class operand, this
9622 /// contains the set of register corresponding to the operand.
9623 RegsForValue AssignedRegs;
9624
9625 explicit SDISelAsmOperandInfo(const TargetLowering::AsmOperandInfo &info)
9626 : TargetLowering::AsmOperandInfo(info), CallOperand(nullptr, 0) {
9627 }
9628
9629 /// Whether or not this operand accesses memory
9630 bool hasMemory(const TargetLowering &TLI) const {
9631 // Indirect operand accesses access memory.
9632 if (isIndirect)
9633 return true;
9634
9635 for (const auto &Code : Codes)
9636 if (TLI.getConstraintType(Constraint: Code) == TargetLowering::C_Memory)
9637 return true;
9638
9639 return false;
9640 }
9641};
9642
9643
9644} // end anonymous namespace
9645
9646/// Make sure that the output operand \p OpInfo and its corresponding input
9647/// operand \p MatchingOpInfo have compatible constraint types (otherwise error
9648/// out).
9649static void patchMatchingInput(const SDISelAsmOperandInfo &OpInfo,
9650 SDISelAsmOperandInfo &MatchingOpInfo,
9651 SelectionDAG &DAG) {
9652 if (OpInfo.ConstraintVT == MatchingOpInfo.ConstraintVT)
9653 return;
9654
9655 const TargetRegisterInfo *TRI = DAG.getSubtarget().getRegisterInfo();
9656 const auto &TLI = DAG.getTargetLoweringInfo();
9657
9658 std::pair<unsigned, const TargetRegisterClass *> MatchRC =
9659 TLI.getRegForInlineAsmConstraint(TRI, Constraint: OpInfo.ConstraintCode,
9660 VT: OpInfo.ConstraintVT);
9661 std::pair<unsigned, const TargetRegisterClass *> InputRC =
9662 TLI.getRegForInlineAsmConstraint(TRI, Constraint: MatchingOpInfo.ConstraintCode,
9663 VT: MatchingOpInfo.ConstraintVT);
9664 const bool OutOpIsIntOrFP =
9665 OpInfo.ConstraintVT.isInteger() || OpInfo.ConstraintVT.isFloatingPoint();
9666 const bool InOpIsIntOrFP = MatchingOpInfo.ConstraintVT.isInteger() ||
9667 MatchingOpInfo.ConstraintVT.isFloatingPoint();
9668 if ((OutOpIsIntOrFP != InOpIsIntOrFP) || (MatchRC.second != InputRC.second)) {
9669 // FIXME: error out in a more elegant fashion
9670 report_fatal_error(reason: "Unsupported asm: input constraint"
9671 " with a matching output constraint of"
9672 " incompatible type!");
9673 }
9674 MatchingOpInfo.ConstraintVT = OpInfo.ConstraintVT;
9675}
9676
9677/// Get a direct memory input to behave well as an indirect operand.
9678/// This may introduce stores, hence the need for a \p Chain.
9679/// \return The (possibly updated) chain.
9680static SDValue getAddressForMemoryInput(SDValue Chain, const SDLoc &Location,
9681 SDISelAsmOperandInfo &OpInfo,
9682 SelectionDAG &DAG) {
9683 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9684
9685 // If we don't have an indirect input, put it in the constpool if we can,
9686 // otherwise spill it to a stack slot.
9687 // TODO: This isn't quite right. We need to handle these according to
9688 // the addressing mode that the constraint wants. Also, this may take
9689 // an additional register for the computation and we don't want that
9690 // either.
9691
9692 // If the operand is a float, integer, or vector constant, spill to a
9693 // constant pool entry to get its address.
9694 const Value *OpVal = OpInfo.CallOperandVal;
9695 if (isa<ConstantFP>(Val: OpVal) || isa<ConstantInt>(Val: OpVal) ||
9696 isa<ConstantVector>(Val: OpVal) || isa<ConstantDataVector>(Val: OpVal)) {
9697 OpInfo.CallOperand = DAG.getConstantPool(
9698 C: cast<Constant>(Val: OpVal), VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
9699 return Chain;
9700 }
9701
9702 // Otherwise, create a stack slot and emit a store to it before the asm.
9703 Type *Ty = OpVal->getType();
9704 auto &DL = DAG.getDataLayout();
9705 TypeSize TySize = DL.getTypeAllocSize(Ty);
9706 MachineFunction &MF = DAG.getMachineFunction();
9707 const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering();
9708 int StackID = 0;
9709 if (TySize.isScalable())
9710 StackID = TFI->getStackIDForScalableVectors();
9711 int SSFI = MF.getFrameInfo().CreateStackObject(Size: TySize.getKnownMinValue(),
9712 Alignment: DL.getPrefTypeAlign(Ty), isSpillSlot: false,
9713 Alloca: nullptr, ID: StackID);
9714 SDValue StackSlot = DAG.getFrameIndex(FI: SSFI, VT: TLI.getFrameIndexTy(DL));
9715 Chain = DAG.getTruncStore(Chain, dl: Location, Val: OpInfo.CallOperand, Ptr: StackSlot,
9716 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI: SSFI),
9717 SVT: TLI.getMemValueType(DL, Ty));
9718 OpInfo.CallOperand = StackSlot;
9719
9720 return Chain;
9721}
9722
9723/// GetRegistersForValue - Assign registers (virtual or physical) for the
9724/// specified operand. We prefer to assign virtual registers, to allow the
9725/// register allocator to handle the assignment process. However, if the asm
9726/// uses features that we can't model on machineinstrs, we have SDISel do the
9727/// allocation. This produces generally horrible, but correct, code.
9728///
9729/// OpInfo describes the operand
9730/// RefOpInfo describes the matching operand if any, the operand otherwise
9731static std::optional<unsigned>
9732getRegistersForValue(SelectionDAG &DAG, const SDLoc &DL,
9733 SDISelAsmOperandInfo &OpInfo,
9734 SDISelAsmOperandInfo &RefOpInfo) {
9735 LLVMContext &Context = *DAG.getContext();
9736 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9737
9738 MachineFunction &MF = DAG.getMachineFunction();
9739 SmallVector<Register, 4> Regs;
9740 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
9741
9742 // No work to do for memory/address operands.
9743 if (OpInfo.ConstraintType == TargetLowering::C_Memory ||
9744 OpInfo.ConstraintType == TargetLowering::C_Address)
9745 return std::nullopt;
9746
9747 // If this is a constraint for a single physreg, or a constraint for a
9748 // register class, find it.
9749 unsigned AssignedReg;
9750 const TargetRegisterClass *RC;
9751 std::tie(args&: AssignedReg, args&: RC) = TLI.getRegForInlineAsmConstraint(
9752 TRI: &TRI, Constraint: RefOpInfo.ConstraintCode, VT: RefOpInfo.ConstraintVT);
9753 // RC is unset only on failure. Return immediately.
9754 if (!RC)
9755 return std::nullopt;
9756
9757 // Get the actual register value type. This is important, because the user
9758 // may have asked for (e.g.) the AX register in i32 type. We need to
9759 // remember that AX is actually i16 to get the right extension.
9760 const MVT RegVT = *TRI.legalclasstypes_begin(RC: *RC);
9761
9762 if (OpInfo.ConstraintVT != MVT::Other && RegVT != MVT::Untyped) {
9763 // If this is an FP operand in an integer register (or visa versa), or more
9764 // generally if the operand value disagrees with the register class we plan
9765 // to stick it in, fix the operand type.
9766 //
9767 // If this is an input value, the bitcast to the new type is done now.
9768 // Bitcast for output value is done at the end of visitInlineAsm().
9769 if ((OpInfo.Type == InlineAsm::isOutput ||
9770 OpInfo.Type == InlineAsm::isInput) &&
9771 !TRI.isTypeLegalForClass(RC: *RC, T: OpInfo.ConstraintVT)) {
9772 // Try to convert to the first EVT that the reg class contains. If the
9773 // types are identical size, use a bitcast to convert (e.g. two differing
9774 // vector types). Note: output bitcast is done at the end of
9775 // visitInlineAsm().
9776 if (RegVT.getSizeInBits() == OpInfo.ConstraintVT.getSizeInBits()) {
9777 // Exclude indirect inputs while they are unsupported because the code
9778 // to perform the load is missing and thus OpInfo.CallOperand still
9779 // refers to the input address rather than the pointed-to value.
9780 if (OpInfo.Type == InlineAsm::isInput && !OpInfo.isIndirect)
9781 OpInfo.CallOperand =
9782 DAG.getNode(Opcode: ISD::BITCAST, DL, VT: RegVT, Operand: OpInfo.CallOperand);
9783 OpInfo.ConstraintVT = RegVT;
9784 // If the operand is an FP value and we want it in integer registers,
9785 // use the corresponding integer type. This turns an f64 value into
9786 // i64, which can be passed with two i32 values on a 32-bit machine.
9787 } else if (RegVT.isInteger() && OpInfo.ConstraintVT.isFloatingPoint()) {
9788 MVT VT = MVT::getIntegerVT(BitWidth: OpInfo.ConstraintVT.getSizeInBits());
9789 if (OpInfo.Type == InlineAsm::isInput)
9790 OpInfo.CallOperand =
9791 DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: OpInfo.CallOperand);
9792 OpInfo.ConstraintVT = VT;
9793 }
9794 }
9795 }
9796
9797 // No need to allocate a matching input constraint since the constraint it's
9798 // matching to has already been allocated.
9799 if (OpInfo.isMatchingInputConstraint())
9800 return std::nullopt;
9801
9802 EVT ValueVT = OpInfo.ConstraintVT;
9803 if (OpInfo.ConstraintVT == MVT::Other)
9804 ValueVT = RegVT;
9805
9806 // Initialize NumRegs.
9807 unsigned NumRegs = 1;
9808 if (OpInfo.ConstraintVT != MVT::Other)
9809 NumRegs = TLI.getNumRegisters(Context, VT: OpInfo.ConstraintVT, RegisterVT: RegVT);
9810
9811 // If this is a constraint for a specific physical register, like {r17},
9812 // assign it now.
9813
9814 // If this associated to a specific register, initialize iterator to correct
9815 // place. If virtual, make sure we have enough registers
9816
9817 // Initialize iterator if necessary
9818 TargetRegisterClass::iterator I = RC->begin();
9819 MachineRegisterInfo &RegInfo = MF.getRegInfo();
9820
9821 // Do not check for single registers.
9822 if (AssignedReg) {
9823 I = std::find(first: I, last: RC->end(), val: AssignedReg);
9824 if (I == RC->end()) {
9825 // RC does not contain the selected register, which indicates a
9826 // mismatch between the register and the required type/bitwidth.
9827 return {AssignedReg};
9828 }
9829 }
9830
9831 for (; NumRegs; --NumRegs, ++I) {
9832 assert(I != RC->end() && "Ran out of registers to allocate!");
9833 Register R = AssignedReg ? Register(*I) : RegInfo.createVirtualRegister(RegClass: RC);
9834 Regs.push_back(Elt: R);
9835 }
9836
9837 OpInfo.AssignedRegs = RegsForValue(Regs, RegVT, ValueVT);
9838 return std::nullopt;
9839}
9840
9841static unsigned
9842findMatchingInlineAsmOperand(unsigned OperandNo,
9843 const std::vector<SDValue> &AsmNodeOperands) {
9844 // Scan until we find the definition we already emitted of this operand.
9845 unsigned CurOp = InlineAsm::Op_FirstOperand;
9846 for (; OperandNo; --OperandNo) {
9847 // Advance to the next operand.
9848 unsigned OpFlag = AsmNodeOperands[CurOp]->getAsZExtVal();
9849 const InlineAsm::Flag F(OpFlag);
9850 assert(
9851 (F.isRegDefKind() || F.isRegDefEarlyClobberKind() || F.isMemKind()) &&
9852 "Skipped past definitions?");
9853 CurOp += F.getNumOperandRegisters() + 1;
9854 }
9855 return CurOp;
9856}
9857
9858namespace {
9859
9860class ExtraFlags {
9861 unsigned Flags = 0;
9862
9863public:
9864 explicit ExtraFlags(const CallBase &Call) {
9865 const InlineAsm *IA = cast<InlineAsm>(Val: Call.getCalledOperand());
9866 if (IA->hasSideEffects())
9867 Flags |= InlineAsm::Extra_HasSideEffects;
9868 if (IA->isAlignStack())
9869 Flags |= InlineAsm::Extra_IsAlignStack;
9870 if (Call.isConvergent())
9871 Flags |= InlineAsm::Extra_IsConvergent;
9872 Flags |= IA->getDialect() * InlineAsm::Extra_AsmDialect;
9873 }
9874
9875 void update(const TargetLowering::AsmOperandInfo &OpInfo) {
9876 // Ideally, we would only check against memory constraints. However, the
9877 // meaning of an Other constraint can be target-specific and we can't easily
9878 // reason about it. Therefore, be conservative and set MayLoad/MayStore
9879 // for Other constraints as well.
9880 if (OpInfo.ConstraintType == TargetLowering::C_Memory ||
9881 OpInfo.ConstraintType == TargetLowering::C_Other) {
9882 if (OpInfo.Type == InlineAsm::isInput)
9883 Flags |= InlineAsm::Extra_MayLoad;
9884 else if (OpInfo.Type == InlineAsm::isOutput)
9885 Flags |= InlineAsm::Extra_MayStore;
9886 else if (OpInfo.Type == InlineAsm::isClobber)
9887 Flags |= (InlineAsm::Extra_MayLoad | InlineAsm::Extra_MayStore);
9888 }
9889 }
9890
9891 unsigned get() const { return Flags; }
9892};
9893
9894} // end anonymous namespace
9895
9896static bool isFunction(SDValue Op) {
9897 if (Op && Op.getOpcode() == ISD::GlobalAddress) {
9898 if (auto *GA = dyn_cast<GlobalAddressSDNode>(Val&: Op)) {
9899 auto Fn = dyn_cast_or_null<Function>(Val: GA->getGlobal());
9900
9901 // In normal "call dllimport func" instruction (non-inlineasm) it force
9902 // indirect access by specifing call opcode. And usually specially print
9903 // asm with indirect symbol (i.g: "*") according to opcode. Inline asm can
9904 // not do in this way now. (In fact, this is similar with "Data Access"
9905 // action). So here we ignore dllimport function.
9906 if (Fn && !Fn->hasDLLImportStorageClass())
9907 return true;
9908 }
9909 }
9910 return false;
9911}
9912
9913/// visitInlineAsm - Handle a call to an InlineAsm object.
9914void SelectionDAGBuilder::visitInlineAsm(const CallBase &Call,
9915 const BasicBlock *EHPadBB) {
9916 const InlineAsm *IA = cast<InlineAsm>(Val: Call.getCalledOperand());
9917
9918 /// ConstraintOperands - Information about all of the constraints.
9919 SmallVector<SDISelAsmOperandInfo, 16> ConstraintOperands;
9920
9921 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9922 TargetLowering::AsmOperandInfoVector TargetConstraints = TLI.ParseConstraints(
9923 DL: DAG.getDataLayout(), TRI: DAG.getSubtarget().getRegisterInfo(), Call);
9924
9925 // First Pass: Calculate HasSideEffects and ExtraFlags (AlignStack,
9926 // AsmDialect, MayLoad, MayStore).
9927 bool HasSideEffect = IA->hasSideEffects();
9928 ExtraFlags ExtraInfo(Call);
9929
9930 for (auto &T : TargetConstraints) {
9931 ConstraintOperands.push_back(Elt: SDISelAsmOperandInfo(T));
9932 SDISelAsmOperandInfo &OpInfo = ConstraintOperands.back();
9933
9934 if (OpInfo.CallOperandVal)
9935 OpInfo.CallOperand = getValue(V: OpInfo.CallOperandVal);
9936
9937 if (!HasSideEffect)
9938 HasSideEffect = OpInfo.hasMemory(TLI);
9939
9940 // Determine if this InlineAsm MayLoad or MayStore based on the constraints.
9941 // FIXME: Could we compute this on OpInfo rather than T?
9942
9943 // Compute the constraint code and ConstraintType to use.
9944 TLI.ComputeConstraintToUse(OpInfo&: T, Op: SDValue());
9945
9946 if (T.ConstraintType == TargetLowering::C_Immediate &&
9947 OpInfo.CallOperand && !isa<ConstantSDNode>(Val: OpInfo.CallOperand))
9948 // We've delayed emitting a diagnostic like the "n" constraint because
9949 // inlining could cause an integer showing up.
9950 return emitInlineAsmError(Call, Message: "constraint '" + Twine(T.ConstraintCode) +
9951 "' expects an integer constant "
9952 "expression");
9953
9954 ExtraInfo.update(OpInfo: T);
9955 }
9956
9957 // We won't need to flush pending loads if this asm doesn't touch
9958 // memory and is nonvolatile.
9959 SDValue Glue, Chain = (HasSideEffect) ? getRoot() : DAG.getRoot();
9960
9961 bool EmitEHLabels = isa<InvokeInst>(Val: Call);
9962 if (EmitEHLabels) {
9963 assert(EHPadBB && "InvokeInst must have an EHPadBB");
9964 }
9965 bool IsCallBr = isa<CallBrInst>(Val: Call);
9966
9967 if (IsCallBr || EmitEHLabels) {
9968 // If this is a callbr or invoke we need to flush pending exports since
9969 // inlineasm_br and invoke are terminators.
9970 // We need to do this before nodes are glued to the inlineasm_br node.
9971 Chain = getControlRoot();
9972 }
9973
9974 MCSymbol *BeginLabel = nullptr;
9975 if (EmitEHLabels) {
9976 Chain = lowerStartEH(Chain, EHPadBB, BeginLabel);
9977 }
9978
9979 int OpNo = -1;
9980 SmallVector<StringRef> AsmStrs;
9981 IA->collectAsmStrs(AsmStrs);
9982
9983 // Second pass over the constraints: compute which constraint option to use.
9984 for (SDISelAsmOperandInfo &OpInfo : ConstraintOperands) {
9985 if (OpInfo.hasArg() || OpInfo.Type == InlineAsm::isOutput)
9986 OpNo++;
9987
9988 // If this is an output operand with a matching input operand, look up the
9989 // matching input. If their types mismatch, e.g. one is an integer, the
9990 // other is floating point, or their sizes are different, flag it as an
9991 // error.
9992 if (OpInfo.hasMatchingInput()) {
9993 SDISelAsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput];
9994 patchMatchingInput(OpInfo, MatchingOpInfo&: Input, DAG);
9995 }
9996
9997 // Compute the constraint code and ConstraintType to use.
9998 TLI.ComputeConstraintToUse(OpInfo, Op: OpInfo.CallOperand, DAG: &DAG);
9999
10000 if ((OpInfo.ConstraintType == TargetLowering::C_Memory &&
10001 OpInfo.Type == InlineAsm::isClobber) ||
10002 OpInfo.ConstraintType == TargetLowering::C_Address)
10003 continue;
10004
10005 // In Linux PIC model, there are 4 cases about value/label addressing:
10006 //
10007 // 1: Function call or Label jmp inside the module.
10008 // 2: Data access (such as global variable, static variable) inside module.
10009 // 3: Function call or Label jmp outside the module.
10010 // 4: Data access (such as global variable) outside the module.
10011 //
10012 // Due to current llvm inline asm architecture designed to not "recognize"
10013 // the asm code, there are quite troubles for us to treat mem addressing
10014 // differently for same value/adress used in different instuctions.
10015 // For example, in pic model, call a func may in plt way or direclty
10016 // pc-related, but lea/mov a function adress may use got.
10017 //
10018 // Here we try to "recognize" function call for the case 1 and case 3 in
10019 // inline asm. And try to adjust the constraint for them.
10020 //
10021 // TODO: Due to current inline asm didn't encourage to jmp to the outsider
10022 // label, so here we don't handle jmp function label now, but we need to
10023 // enhance it (especilly in PIC model) if we meet meaningful requirements.
10024 if (OpInfo.isIndirect && isFunction(Op: OpInfo.CallOperand) &&
10025 TLI.isInlineAsmTargetBranch(AsmStrs, OpNo) &&
10026 TM.getCodeModel() != CodeModel::Large) {
10027 OpInfo.isIndirect = false;
10028 OpInfo.ConstraintType = TargetLowering::C_Address;
10029 }
10030
10031 // If this is a memory input, and if the operand is not indirect, do what we
10032 // need to provide an address for the memory input.
10033 if (OpInfo.ConstraintType == TargetLowering::C_Memory &&
10034 !OpInfo.isIndirect) {
10035 assert((OpInfo.isMultipleAlternative ||
10036 (OpInfo.Type == InlineAsm::isInput)) &&
10037 "Can only indirectify direct input operands!");
10038
10039 // Memory operands really want the address of the value.
10040 Chain = getAddressForMemoryInput(Chain, Location: getCurSDLoc(), OpInfo, DAG);
10041
10042 // There is no longer a Value* corresponding to this operand.
10043 OpInfo.CallOperandVal = nullptr;
10044
10045 // It is now an indirect operand.
10046 OpInfo.isIndirect = true;
10047 }
10048
10049 }
10050
10051 // AsmNodeOperands - The operands for the ISD::INLINEASM node.
10052 std::vector<SDValue> AsmNodeOperands;
10053 AsmNodeOperands.push_back(x: SDValue()); // reserve space for input chain
10054 AsmNodeOperands.push_back(x: DAG.getTargetExternalSymbol(
10055 Sym: IA->getAsmString().data(), VT: TLI.getProgramPointerTy(DL: DAG.getDataLayout())));
10056
10057 // If we have a !srcloc metadata node associated with it, we want to attach
10058 // this to the ultimately generated inline asm machineinstr. To do this, we
10059 // pass in the third operand as this (potentially null) inline asm MDNode.
10060 const MDNode *SrcLoc = Call.getMetadata(Kind: "srcloc");
10061 AsmNodeOperands.push_back(x: DAG.getMDNode(MD: SrcLoc));
10062
10063 // Remember the HasSideEffect, AlignStack, AsmDialect, MayLoad and MayStore
10064 // bits as operand 3.
10065 AsmNodeOperands.push_back(x: DAG.getTargetConstant(
10066 Val: ExtraInfo.get(), DL: getCurSDLoc(), VT: TLI.getPointerTy(DL: DAG.getDataLayout())));
10067
10068 // Third pass: Loop over operands to prepare DAG-level operands.. As part of
10069 // this, assign virtual and physical registers for inputs and otput.
10070 for (SDISelAsmOperandInfo &OpInfo : ConstraintOperands) {
10071 // Assign Registers.
10072 SDISelAsmOperandInfo &RefOpInfo =
10073 OpInfo.isMatchingInputConstraint()
10074 ? ConstraintOperands[OpInfo.getMatchedOperand()]
10075 : OpInfo;
10076 const auto RegError =
10077 getRegistersForValue(DAG, DL: getCurSDLoc(), OpInfo, RefOpInfo);
10078 if (RegError) {
10079 const MachineFunction &MF = DAG.getMachineFunction();
10080 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
10081 const char *RegName = TRI.getName(RegNo: *RegError);
10082 emitInlineAsmError(Call, Message: "register '" + Twine(RegName) +
10083 "' allocated for constraint '" +
10084 Twine(OpInfo.ConstraintCode) +
10085 "' does not match required type");
10086 return;
10087 }
10088
10089 auto DetectWriteToReservedRegister = [&]() {
10090 const MachineFunction &MF = DAG.getMachineFunction();
10091 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
10092 for (Register Reg : OpInfo.AssignedRegs.Regs) {
10093 if (Reg.isPhysical() && TRI.isInlineAsmReadOnlyReg(MF, PhysReg: Reg)) {
10094 const char *RegName = TRI.getName(RegNo: Reg);
10095 emitInlineAsmError(Call, Message: "write to reserved register '" +
10096 Twine(RegName) + "'");
10097 return true;
10098 }
10099 }
10100 return false;
10101 };
10102 assert((OpInfo.ConstraintType != TargetLowering::C_Address ||
10103 (OpInfo.Type == InlineAsm::isInput &&
10104 !OpInfo.isMatchingInputConstraint())) &&
10105 "Only address as input operand is allowed.");
10106
10107 switch (OpInfo.Type) {
10108 case InlineAsm::isOutput:
10109 if (OpInfo.ConstraintType == TargetLowering::C_Memory) {
10110 const InlineAsm::ConstraintCode ConstraintID =
10111 TLI.getInlineAsmMemConstraint(ConstraintCode: OpInfo.ConstraintCode);
10112 assert(ConstraintID != InlineAsm::ConstraintCode::Unknown &&
10113 "Failed to convert memory constraint code to constraint id.");
10114
10115 // Add information to the INLINEASM node to know about this output.
10116 InlineAsm::Flag OpFlags(InlineAsm::Kind::Mem, 1);
10117 OpFlags.setMemConstraint(ConstraintID);
10118 AsmNodeOperands.push_back(DAG.getTargetConstant(OpFlags, getCurSDLoc(),
10119 MVT::i32));
10120 AsmNodeOperands.push_back(x: OpInfo.CallOperand);
10121 } else {
10122 // Otherwise, this outputs to a register (directly for C_Register /
10123 // C_RegisterClass, and a target-defined fashion for
10124 // C_Immediate/C_Other). Find a register that we can use.
10125 if (OpInfo.AssignedRegs.Regs.empty()) {
10126 emitInlineAsmError(
10127 Call, Message: "couldn't allocate output register for constraint '" +
10128 Twine(OpInfo.ConstraintCode) + "'");
10129 return;
10130 }
10131
10132 if (DetectWriteToReservedRegister())
10133 return;
10134
10135 // Add information to the INLINEASM node to know that this register is
10136 // set.
10137 OpInfo.AssignedRegs.AddInlineAsmOperands(
10138 Code: OpInfo.isEarlyClobber ? InlineAsm::Kind::RegDefEarlyClobber
10139 : InlineAsm::Kind::RegDef,
10140 HasMatching: false, MatchingIdx: 0, dl: getCurSDLoc(), DAG, Ops&: AsmNodeOperands);
10141 }
10142 break;
10143
10144 case InlineAsm::isInput:
10145 case InlineAsm::isLabel: {
10146 SDValue InOperandVal = OpInfo.CallOperand;
10147
10148 if (OpInfo.isMatchingInputConstraint()) {
10149 // If this is required to match an output register we have already set,
10150 // just use its register.
10151 auto CurOp = findMatchingInlineAsmOperand(OperandNo: OpInfo.getMatchedOperand(),
10152 AsmNodeOperands);
10153 InlineAsm::Flag Flag(AsmNodeOperands[CurOp]->getAsZExtVal());
10154 if (Flag.isRegDefKind() || Flag.isRegDefEarlyClobberKind()) {
10155 if (OpInfo.isIndirect) {
10156 // This happens on gcc/testsuite/gcc.dg/pr8788-1.c
10157 emitInlineAsmError(Call, Message: "inline asm not supported yet: "
10158 "don't know how to handle tied "
10159 "indirect register inputs");
10160 return;
10161 }
10162
10163 SmallVector<Register, 4> Regs;
10164 MachineFunction &MF = DAG.getMachineFunction();
10165 MachineRegisterInfo &MRI = MF.getRegInfo();
10166 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
10167 auto *R = cast<RegisterSDNode>(Val&: AsmNodeOperands[CurOp+1]);
10168 Register TiedReg = R->getReg();
10169 MVT RegVT = R->getSimpleValueType(ResNo: 0);
10170 const TargetRegisterClass *RC =
10171 TiedReg.isVirtual() ? MRI.getRegClass(TiedReg)
10172 : RegVT != MVT::Untyped ? TLI.getRegClassFor(RegVT)
10173 : TRI.getMinimalPhysRegClass(TiedReg);
10174 for (unsigned i = 0, e = Flag.getNumOperandRegisters(); i != e; ++i)
10175 Regs.push_back(Elt: MRI.createVirtualRegister(RegClass: RC));
10176
10177 RegsForValue MatchedRegs(Regs, RegVT, InOperandVal.getValueType());
10178
10179 SDLoc dl = getCurSDLoc();
10180 // Use the produced MatchedRegs object to
10181 MatchedRegs.getCopyToRegs(Val: InOperandVal, DAG, dl, Chain, Glue: &Glue, V: &Call);
10182 MatchedRegs.AddInlineAsmOperands(Code: InlineAsm::Kind::RegUse, HasMatching: true,
10183 MatchingIdx: OpInfo.getMatchedOperand(), dl, DAG,
10184 Ops&: AsmNodeOperands);
10185 break;
10186 }
10187
10188 assert(Flag.isMemKind() && "Unknown matching constraint!");
10189 assert(Flag.getNumOperandRegisters() == 1 &&
10190 "Unexpected number of operands");
10191 // Add information to the INLINEASM node to know about this input.
10192 // See InlineAsm.h isUseOperandTiedToDef.
10193 Flag.clearMemConstraint();
10194 Flag.setMatchingOp(OpInfo.getMatchedOperand());
10195 AsmNodeOperands.push_back(x: DAG.getTargetConstant(
10196 Val: Flag, DL: getCurSDLoc(), VT: TLI.getPointerTy(DL: DAG.getDataLayout())));
10197 AsmNodeOperands.push_back(x: AsmNodeOperands[CurOp+1]);
10198 break;
10199 }
10200
10201 // Treat indirect 'X' constraint as memory.
10202 if (OpInfo.ConstraintType == TargetLowering::C_Other &&
10203 OpInfo.isIndirect)
10204 OpInfo.ConstraintType = TargetLowering::C_Memory;
10205
10206 if (OpInfo.ConstraintType == TargetLowering::C_Immediate ||
10207 OpInfo.ConstraintType == TargetLowering::C_Other) {
10208 std::vector<SDValue> Ops;
10209 TLI.LowerAsmOperandForConstraint(Op: InOperandVal, Constraint: OpInfo.ConstraintCode,
10210 Ops, DAG);
10211 if (Ops.empty()) {
10212 if (OpInfo.ConstraintType == TargetLowering::C_Immediate)
10213 if (isa<ConstantSDNode>(Val: InOperandVal)) {
10214 emitInlineAsmError(Call, Message: "value out of range for constraint '" +
10215 Twine(OpInfo.ConstraintCode) + "'");
10216 return;
10217 }
10218
10219 emitInlineAsmError(Call,
10220 Message: "invalid operand for inline asm constraint '" +
10221 Twine(OpInfo.ConstraintCode) + "'");
10222 return;
10223 }
10224
10225 // Add information to the INLINEASM node to know about this input.
10226 InlineAsm::Flag ResOpType(InlineAsm::Kind::Imm, Ops.size());
10227 AsmNodeOperands.push_back(x: DAG.getTargetConstant(
10228 Val: ResOpType, DL: getCurSDLoc(), VT: TLI.getPointerTy(DL: DAG.getDataLayout())));
10229 llvm::append_range(C&: AsmNodeOperands, R&: Ops);
10230 break;
10231 }
10232
10233 if (OpInfo.ConstraintType == TargetLowering::C_Memory) {
10234 assert((OpInfo.isIndirect ||
10235 OpInfo.ConstraintType != TargetLowering::C_Memory) &&
10236 "Operand must be indirect to be a mem!");
10237 assert(InOperandVal.getValueType() ==
10238 TLI.getPointerTy(DAG.getDataLayout()) &&
10239 "Memory operands expect pointer values");
10240
10241 const InlineAsm::ConstraintCode ConstraintID =
10242 TLI.getInlineAsmMemConstraint(ConstraintCode: OpInfo.ConstraintCode);
10243 assert(ConstraintID != InlineAsm::ConstraintCode::Unknown &&
10244 "Failed to convert memory constraint code to constraint id.");
10245
10246 // Add information to the INLINEASM node to know about this input.
10247 InlineAsm::Flag ResOpType(InlineAsm::Kind::Mem, 1);
10248 ResOpType.setMemConstraint(ConstraintID);
10249 AsmNodeOperands.push_back(DAG.getTargetConstant(ResOpType,
10250 getCurSDLoc(),
10251 MVT::i32));
10252 AsmNodeOperands.push_back(x: InOperandVal);
10253 break;
10254 }
10255
10256 if (OpInfo.ConstraintType == TargetLowering::C_Address) {
10257 const InlineAsm::ConstraintCode ConstraintID =
10258 TLI.getInlineAsmMemConstraint(ConstraintCode: OpInfo.ConstraintCode);
10259 assert(ConstraintID != InlineAsm::ConstraintCode::Unknown &&
10260 "Failed to convert memory constraint code to constraint id.");
10261
10262 InlineAsm::Flag ResOpType(InlineAsm::Kind::Mem, 1);
10263
10264 SDValue AsmOp = InOperandVal;
10265 if (isFunction(Op: InOperandVal)) {
10266 auto *GA = cast<GlobalAddressSDNode>(Val&: InOperandVal);
10267 ResOpType = InlineAsm::Flag(InlineAsm::Kind::Func, 1);
10268 AsmOp = DAG.getTargetGlobalAddress(GV: GA->getGlobal(), DL: getCurSDLoc(),
10269 VT: InOperandVal.getValueType(),
10270 offset: GA->getOffset());
10271 }
10272
10273 // Add information to the INLINEASM node to know about this input.
10274 ResOpType.setMemConstraint(ConstraintID);
10275
10276 AsmNodeOperands.push_back(
10277 DAG.getTargetConstant(ResOpType, getCurSDLoc(), MVT::i32));
10278
10279 AsmNodeOperands.push_back(x: AsmOp);
10280 break;
10281 }
10282
10283 if (OpInfo.ConstraintType != TargetLowering::C_RegisterClass &&
10284 OpInfo.ConstraintType != TargetLowering::C_Register) {
10285 emitInlineAsmError(Call, Message: "unknown asm constraint '" +
10286 Twine(OpInfo.ConstraintCode) + "'");
10287 return;
10288 }
10289
10290 // TODO: Support this.
10291 if (OpInfo.isIndirect) {
10292 emitInlineAsmError(
10293 Call, Message: "Don't know how to handle indirect register inputs yet "
10294 "for constraint '" +
10295 Twine(OpInfo.ConstraintCode) + "'");
10296 return;
10297 }
10298
10299 // Copy the input into the appropriate registers.
10300 if (OpInfo.AssignedRegs.Regs.empty()) {
10301 emitInlineAsmError(Call,
10302 Message: "couldn't allocate input reg for constraint '" +
10303 Twine(OpInfo.ConstraintCode) + "'");
10304 return;
10305 }
10306
10307 if (DetectWriteToReservedRegister())
10308 return;
10309
10310 SDLoc dl = getCurSDLoc();
10311
10312 OpInfo.AssignedRegs.getCopyToRegs(Val: InOperandVal, DAG, dl, Chain, Glue: &Glue,
10313 V: &Call);
10314
10315 OpInfo.AssignedRegs.AddInlineAsmOperands(Code: InlineAsm::Kind::RegUse, HasMatching: false,
10316 MatchingIdx: 0, dl, DAG, Ops&: AsmNodeOperands);
10317 break;
10318 }
10319 case InlineAsm::isClobber:
10320 // Add the clobbered value to the operand list, so that the register
10321 // allocator is aware that the physreg got clobbered.
10322 if (!OpInfo.AssignedRegs.Regs.empty())
10323 OpInfo.AssignedRegs.AddInlineAsmOperands(Code: InlineAsm::Kind::Clobber,
10324 HasMatching: false, MatchingIdx: 0, dl: getCurSDLoc(), DAG,
10325 Ops&: AsmNodeOperands);
10326 break;
10327 }
10328 }
10329
10330 // Finish up input operands. Set the input chain and add the flag last.
10331 AsmNodeOperands[InlineAsm::Op_InputChain] = Chain;
10332 if (Glue.getNode()) AsmNodeOperands.push_back(x: Glue);
10333
10334 unsigned ISDOpc = IsCallBr ? ISD::INLINEASM_BR : ISD::INLINEASM;
10335 Chain = DAG.getNode(ISDOpc, getCurSDLoc(),
10336 DAG.getVTList(MVT::Other, MVT::Glue), AsmNodeOperands);
10337 Glue = Chain.getValue(R: 1);
10338
10339 // Do additional work to generate outputs.
10340
10341 SmallVector<EVT, 1> ResultVTs;
10342 SmallVector<SDValue, 1> ResultValues;
10343 SmallVector<SDValue, 8> OutChains;
10344
10345 llvm::Type *CallResultType = Call.getType();
10346 ArrayRef<Type *> ResultTypes;
10347 if (StructType *StructResult = dyn_cast<StructType>(Val: CallResultType))
10348 ResultTypes = StructResult->elements();
10349 else if (!CallResultType->isVoidTy())
10350 ResultTypes = ArrayRef(CallResultType);
10351
10352 auto CurResultType = ResultTypes.begin();
10353 auto handleRegAssign = [&](SDValue V) {
10354 assert(CurResultType != ResultTypes.end() && "Unexpected value");
10355 assert((*CurResultType)->isSized() && "Unexpected unsized type");
10356 EVT ResultVT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: *CurResultType);
10357 ++CurResultType;
10358 // If the type of the inline asm call site return value is different but has
10359 // same size as the type of the asm output bitcast it. One example of this
10360 // is for vectors with different width / number of elements. This can
10361 // happen for register classes that can contain multiple different value
10362 // types. The preg or vreg allocated may not have the same VT as was
10363 // expected.
10364 //
10365 // This can also happen for a return value that disagrees with the register
10366 // class it is put in, eg. a double in a general-purpose register on a
10367 // 32-bit machine.
10368 if (ResultVT != V.getValueType() &&
10369 ResultVT.getSizeInBits() == V.getValueSizeInBits())
10370 V = DAG.getNode(Opcode: ISD::BITCAST, DL: getCurSDLoc(), VT: ResultVT, Operand: V);
10371 else if (ResultVT != V.getValueType() && ResultVT.isInteger() &&
10372 V.getValueType().isInteger()) {
10373 // If a result value was tied to an input value, the computed result
10374 // may have a wider width than the expected result. Extract the
10375 // relevant portion.
10376 V = DAG.getNode(Opcode: ISD::TRUNCATE, DL: getCurSDLoc(), VT: ResultVT, Operand: V);
10377 }
10378 assert(ResultVT == V.getValueType() && "Asm result value mismatch!");
10379 ResultVTs.push_back(Elt: ResultVT);
10380 ResultValues.push_back(Elt: V);
10381 };
10382
10383 // Deal with output operands.
10384 for (SDISelAsmOperandInfo &OpInfo : ConstraintOperands) {
10385 if (OpInfo.Type == InlineAsm::isOutput) {
10386 SDValue Val;
10387 // Skip trivial output operands.
10388 if (OpInfo.AssignedRegs.Regs.empty())
10389 continue;
10390
10391 switch (OpInfo.ConstraintType) {
10392 case TargetLowering::C_Register:
10393 case TargetLowering::C_RegisterClass:
10394 Val = OpInfo.AssignedRegs.getCopyFromRegs(DAG, FuncInfo, dl: getCurSDLoc(),
10395 Chain, Glue: &Glue, V: &Call);
10396 break;
10397 case TargetLowering::C_Immediate:
10398 case TargetLowering::C_Other:
10399 Val = TLI.LowerAsmOutputForConstraint(Chain, Glue, DL: getCurSDLoc(),
10400 OpInfo, DAG);
10401 break;
10402 case TargetLowering::C_Memory:
10403 break; // Already handled.
10404 case TargetLowering::C_Address:
10405 break; // Silence warning.
10406 case TargetLowering::C_Unknown:
10407 assert(false && "Unexpected unknown constraint");
10408 }
10409
10410 // Indirect output manifest as stores. Record output chains.
10411 if (OpInfo.isIndirect) {
10412 const Value *Ptr = OpInfo.CallOperandVal;
10413 assert(Ptr && "Expected value CallOperandVal for indirect asm operand");
10414 SDValue Store = DAG.getStore(Chain, dl: getCurSDLoc(), Val, Ptr: getValue(V: Ptr),
10415 PtrInfo: MachinePointerInfo(Ptr));
10416 OutChains.push_back(Elt: Store);
10417 } else {
10418 // generate CopyFromRegs to associated registers.
10419 assert(!Call.getType()->isVoidTy() && "Bad inline asm!");
10420 if (Val.getOpcode() == ISD::MERGE_VALUES) {
10421 for (const SDValue &V : Val->op_values())
10422 handleRegAssign(V);
10423 } else
10424 handleRegAssign(Val);
10425 }
10426 }
10427 }
10428
10429 // Set results.
10430 if (!ResultValues.empty()) {
10431 assert(CurResultType == ResultTypes.end() &&
10432 "Mismatch in number of ResultTypes");
10433 assert(ResultValues.size() == ResultTypes.size() &&
10434 "Mismatch in number of output operands in asm result");
10435
10436 SDValue V = DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: getCurSDLoc(),
10437 VTList: DAG.getVTList(VTs: ResultVTs), Ops: ResultValues);
10438 setValue(V: &Call, NewN: V);
10439 }
10440
10441 // Collect store chains.
10442 if (!OutChains.empty())
10443 Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other, OutChains);
10444
10445 if (EmitEHLabels) {
10446 Chain = lowerEndEH(Chain, II: cast<InvokeInst>(Val: &Call), EHPadBB, BeginLabel);
10447 }
10448
10449 // Only Update Root if inline assembly has a memory effect.
10450 if (ResultValues.empty() || HasSideEffect || !OutChains.empty() || IsCallBr ||
10451 EmitEHLabels)
10452 DAG.setRoot(Chain);
10453}
10454
10455void SelectionDAGBuilder::emitInlineAsmError(const CallBase &Call,
10456 const Twine &Message) {
10457 LLVMContext &Ctx = *DAG.getContext();
10458 Ctx.diagnose(DI: DiagnosticInfoInlineAsm(Call, Message));
10459
10460 // Make sure we leave the DAG in a valid state
10461 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
10462 SmallVector<EVT, 1> ValueVTs;
10463 ComputeValueVTs(TLI, DL: DAG.getDataLayout(), Ty: Call.getType(), ValueVTs);
10464
10465 if (ValueVTs.empty())
10466 return;
10467
10468 SmallVector<SDValue, 1> Ops;
10469 for (const EVT &VT : ValueVTs)
10470 Ops.push_back(Elt: DAG.getUNDEF(VT));
10471
10472 setValue(V: &Call, NewN: DAG.getMergeValues(Ops, dl: getCurSDLoc()));
10473}
10474
10475void SelectionDAGBuilder::visitVAStart(const CallInst &I) {
10476 DAG.setRoot(DAG.getNode(ISD::VASTART, getCurSDLoc(),
10477 MVT::Other, getRoot(),
10478 getValue(I.getArgOperand(0)),
10479 DAG.getSrcValue(I.getArgOperand(0))));
10480}
10481
10482void SelectionDAGBuilder::visitVAArg(const VAArgInst &I) {
10483 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
10484 const DataLayout &DL = DAG.getDataLayout();
10485 SDValue V = DAG.getVAArg(
10486 VT: TLI.getMemValueType(DL: DAG.getDataLayout(), Ty: I.getType()), dl: getCurSDLoc(),
10487 Chain: getRoot(), Ptr: getValue(V: I.getOperand(i_nocapture: 0)), SV: DAG.getSrcValue(v: I.getOperand(i_nocapture: 0)),
10488 Align: DL.getABITypeAlign(Ty: I.getType()).value());
10489 DAG.setRoot(V.getValue(R: 1));
10490
10491 if (I.getType()->isPointerTy())
10492 V = DAG.getPtrExtOrTrunc(
10493 Op: V, DL: getCurSDLoc(), VT: TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType()));
10494 setValue(V: &I, NewN: V);
10495}
10496
10497void SelectionDAGBuilder::visitVAEnd(const CallInst &I) {
10498 DAG.setRoot(DAG.getNode(ISD::VAEND, getCurSDLoc(),
10499 MVT::Other, getRoot(),
10500 getValue(I.getArgOperand(0)),
10501 DAG.getSrcValue(I.getArgOperand(0))));
10502}
10503
10504void SelectionDAGBuilder::visitVACopy(const CallInst &I) {
10505 DAG.setRoot(DAG.getNode(ISD::VACOPY, getCurSDLoc(),
10506 MVT::Other, getRoot(),
10507 getValue(I.getArgOperand(0)),
10508 getValue(I.getArgOperand(1)),
10509 DAG.getSrcValue(I.getArgOperand(0)),
10510 DAG.getSrcValue(I.getArgOperand(1))));
10511}
10512
10513SDValue SelectionDAGBuilder::lowerRangeToAssertZExt(SelectionDAG &DAG,
10514 const Instruction &I,
10515 SDValue Op) {
10516 std::optional<ConstantRange> CR = getRange(I);
10517
10518 if (!CR || CR->isFullSet() || CR->isEmptySet() || CR->isUpperWrapped())
10519 return Op;
10520
10521 APInt Lo = CR->getUnsignedMin();
10522 if (!Lo.isMinValue())
10523 return Op;
10524
10525 APInt Hi = CR->getUnsignedMax();
10526 unsigned Bits = std::max(a: Hi.getActiveBits(),
10527 b: static_cast<unsigned>(IntegerType::MIN_INT_BITS));
10528
10529 EVT SmallVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: Bits);
10530
10531 SDLoc SL = getCurSDLoc();
10532
10533 SDValue ZExt = DAG.getNode(Opcode: ISD::AssertZext, DL: SL, VT: Op.getValueType(), N1: Op,
10534 N2: DAG.getValueType(SmallVT));
10535 unsigned NumVals = Op.getNode()->getNumValues();
10536 if (NumVals == 1)
10537 return ZExt;
10538
10539 SmallVector<SDValue, 4> Ops;
10540
10541 Ops.push_back(Elt: ZExt);
10542 for (unsigned I = 1; I != NumVals; ++I)
10543 Ops.push_back(Elt: Op.getValue(R: I));
10544
10545 return DAG.getMergeValues(Ops, dl: SL);
10546}
10547
10548/// Populate a CallLowerinInfo (into \p CLI) based on the properties of
10549/// the call being lowered.
10550///
10551/// This is a helper for lowering intrinsics that follow a target calling
10552/// convention or require stack pointer adjustment. Only a subset of the
10553/// intrinsic's operands need to participate in the calling convention.
10554void SelectionDAGBuilder::populateCallLoweringInfo(
10555 TargetLowering::CallLoweringInfo &CLI, const CallBase *Call,
10556 unsigned ArgIdx, unsigned NumArgs, SDValue Callee, Type *ReturnTy,
10557 AttributeSet RetAttrs, bool IsPatchPoint) {
10558 TargetLowering::ArgListTy Args;
10559 Args.reserve(n: NumArgs);
10560
10561 // Populate the argument list.
10562 // Attributes for args start at offset 1, after the return attribute.
10563 for (unsigned ArgI = ArgIdx, ArgE = ArgIdx + NumArgs;
10564 ArgI != ArgE; ++ArgI) {
10565 const Value *V = Call->getOperand(i_nocapture: ArgI);
10566
10567 assert(!V->getType()->isEmptyTy() && "Empty type passed to intrinsic.");
10568
10569 TargetLowering::ArgListEntry Entry;
10570 Entry.Node = getValue(V);
10571 Entry.Ty = V->getType();
10572 Entry.setAttributes(Call, ArgIdx: ArgI);
10573 Args.push_back(x: Entry);
10574 }
10575
10576 CLI.setDebugLoc(getCurSDLoc())
10577 .setChain(getRoot())
10578 .setCallee(CC: Call->getCallingConv(), ResultType: ReturnTy, Target: Callee, ArgsList: std::move(Args),
10579 ResultAttrs: RetAttrs)
10580 .setDiscardResult(Call->use_empty())
10581 .setIsPatchPoint(IsPatchPoint)
10582 .setIsPreallocated(
10583 Call->countOperandBundlesOfType(ID: LLVMContext::OB_preallocated) != 0);
10584}
10585
10586/// Add a stack map intrinsic call's live variable operands to a stackmap
10587/// or patchpoint target node's operand list.
10588///
10589/// Constants are converted to TargetConstants purely as an optimization to
10590/// avoid constant materialization and register allocation.
10591///
10592/// FrameIndex operands are converted to TargetFrameIndex so that ISEL does not
10593/// generate addess computation nodes, and so FinalizeISel can convert the
10594/// TargetFrameIndex into a DirectMemRefOp StackMap location. This avoids
10595/// address materialization and register allocation, but may also be required
10596/// for correctness. If a StackMap (or PatchPoint) intrinsic directly uses an
10597/// alloca in the entry block, then the runtime may assume that the alloca's
10598/// StackMap location can be read immediately after compilation and that the
10599/// location is valid at any point during execution (this is similar to the
10600/// assumption made by the llvm.gcroot intrinsic). If the alloca's location were
10601/// only available in a register, then the runtime would need to trap when
10602/// execution reaches the StackMap in order to read the alloca's location.
10603static void addStackMapLiveVars(const CallBase &Call, unsigned StartIdx,
10604 const SDLoc &DL, SmallVectorImpl<SDValue> &Ops,
10605 SelectionDAGBuilder &Builder) {
10606 SelectionDAG &DAG = Builder.DAG;
10607 for (unsigned I = StartIdx; I < Call.arg_size(); I++) {
10608 SDValue Op = Builder.getValue(V: Call.getArgOperand(i: I));
10609
10610 // Things on the stack are pointer-typed, meaning that they are already
10611 // legal and can be emitted directly to target nodes.
10612 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: Op)) {
10613 Ops.push_back(Elt: DAG.getTargetFrameIndex(FI: FI->getIndex(), VT: Op.getValueType()));
10614 } else {
10615 // Otherwise emit a target independent node to be legalised.
10616 Ops.push_back(Elt: Builder.getValue(V: Call.getArgOperand(i: I)));
10617 }
10618 }
10619}
10620
10621/// Lower llvm.experimental.stackmap.
10622void SelectionDAGBuilder::visitStackmap(const CallInst &CI) {
10623 // void @llvm.experimental.stackmap(i64 <id>, i32 <numShadowBytes>,
10624 // [live variables...])
10625
10626 assert(CI.getType()->isVoidTy() && "Stackmap cannot return a value.");
10627
10628 SDValue Chain, InGlue, Callee;
10629 SmallVector<SDValue, 32> Ops;
10630
10631 SDLoc DL = getCurSDLoc();
10632 Callee = getValue(V: CI.getCalledOperand());
10633
10634 // The stackmap intrinsic only records the live variables (the arguments
10635 // passed to it) and emits NOPS (if requested). Unlike the patchpoint
10636 // intrinsic, this won't be lowered to a function call. This means we don't
10637 // have to worry about calling conventions and target specific lowering code.
10638 // Instead we perform the call lowering right here.
10639 //
10640 // chain, flag = CALLSEQ_START(chain, 0, 0)
10641 // chain, flag = STACKMAP(id, nbytes, ..., chain, flag)
10642 // chain, flag = CALLSEQ_END(chain, 0, 0, flag)
10643 //
10644 Chain = DAG.getCALLSEQ_START(Chain: getRoot(), InSize: 0, OutSize: 0, DL);
10645 InGlue = Chain.getValue(R: 1);
10646
10647 // Add the STACKMAP operands, starting with DAG house-keeping.
10648 Ops.push_back(Elt: Chain);
10649 Ops.push_back(Elt: InGlue);
10650
10651 // Add the <id>, <numShadowBytes> operands.
10652 //
10653 // These do not require legalisation, and can be emitted directly to target
10654 // constant nodes.
10655 SDValue ID = getValue(V: CI.getArgOperand(i: 0));
10656 assert(ID.getValueType() == MVT::i64);
10657 SDValue IDConst =
10658 DAG.getTargetConstant(Val: ID->getAsZExtVal(), DL, VT: ID.getValueType());
10659 Ops.push_back(Elt: IDConst);
10660
10661 SDValue Shad = getValue(V: CI.getArgOperand(i: 1));
10662 assert(Shad.getValueType() == MVT::i32);
10663 SDValue ShadConst =
10664 DAG.getTargetConstant(Val: Shad->getAsZExtVal(), DL, VT: Shad.getValueType());
10665 Ops.push_back(Elt: ShadConst);
10666
10667 // Add the live variables.
10668 addStackMapLiveVars(Call: CI, StartIdx: 2, DL, Ops, Builder&: *this);
10669
10670 // Create the STACKMAP node.
10671 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
10672 Chain = DAG.getNode(Opcode: ISD::STACKMAP, DL, VTList: NodeTys, Ops);
10673 InGlue = Chain.getValue(R: 1);
10674
10675 Chain = DAG.getCALLSEQ_END(Chain, Size1: 0, Size2: 0, Glue: InGlue, DL);
10676
10677 // Stackmaps don't generate values, so nothing goes into the NodeMap.
10678
10679 // Set the root to the target-lowered call chain.
10680 DAG.setRoot(Chain);
10681
10682 // Inform the Frame Information that we have a stackmap in this function.
10683 FuncInfo.MF->getFrameInfo().setHasStackMap();
10684}
10685
10686/// Lower llvm.experimental.patchpoint directly to its target opcode.
10687void SelectionDAGBuilder::visitPatchpoint(const CallBase &CB,
10688 const BasicBlock *EHPadBB) {
10689 // <ty> @llvm.experimental.patchpoint.<ty>(i64 <id>,
10690 // i32 <numBytes>,
10691 // i8* <target>,
10692 // i32 <numArgs>,
10693 // [Args...],
10694 // [live variables...])
10695
10696 CallingConv::ID CC = CB.getCallingConv();
10697 bool IsAnyRegCC = CC == CallingConv::AnyReg;
10698 bool HasDef = !CB.getType()->isVoidTy();
10699 SDLoc dl = getCurSDLoc();
10700 SDValue Callee = getValue(V: CB.getArgOperand(i: PatchPointOpers::TargetPos));
10701
10702 // Handle immediate and symbolic callees.
10703 if (auto* ConstCallee = dyn_cast<ConstantSDNode>(Val&: Callee))
10704 Callee = DAG.getIntPtrConstant(Val: ConstCallee->getZExtValue(), DL: dl,
10705 /*isTarget=*/true);
10706 else if (auto* SymbolicCallee = dyn_cast<GlobalAddressSDNode>(Val&: Callee))
10707 Callee = DAG.getTargetGlobalAddress(GV: SymbolicCallee->getGlobal(),
10708 DL: SDLoc(SymbolicCallee),
10709 VT: SymbolicCallee->getValueType(ResNo: 0));
10710
10711 // Get the real number of arguments participating in the call <numArgs>
10712 SDValue NArgVal = getValue(V: CB.getArgOperand(i: PatchPointOpers::NArgPos));
10713 unsigned NumArgs = NArgVal->getAsZExtVal();
10714
10715 // Skip the four meta args: <id>, <numNopBytes>, <target>, <numArgs>
10716 // Intrinsics include all meta-operands up to but not including CC.
10717 unsigned NumMetaOpers = PatchPointOpers::CCPos;
10718 assert(CB.arg_size() >= NumMetaOpers + NumArgs &&
10719 "Not enough arguments provided to the patchpoint intrinsic");
10720
10721 // For AnyRegCC the arguments are lowered later on manually.
10722 unsigned NumCallArgs = IsAnyRegCC ? 0 : NumArgs;
10723 Type *ReturnTy =
10724 IsAnyRegCC ? Type::getVoidTy(C&: *DAG.getContext()) : CB.getType();
10725
10726 TargetLowering::CallLoweringInfo CLI(DAG);
10727 populateCallLoweringInfo(CLI, Call: &CB, ArgIdx: NumMetaOpers, NumArgs: NumCallArgs, Callee,
10728 ReturnTy, RetAttrs: CB.getAttributes().getRetAttrs(), IsPatchPoint: true);
10729 std::pair<SDValue, SDValue> Result = lowerInvokable(CLI, EHPadBB);
10730
10731 SDNode *CallEnd = Result.second.getNode();
10732 if (CallEnd->getOpcode() == ISD::EH_LABEL)
10733 CallEnd = CallEnd->getOperand(Num: 0).getNode();
10734 if (HasDef && (CallEnd->getOpcode() == ISD::CopyFromReg))
10735 CallEnd = CallEnd->getOperand(Num: 0).getNode();
10736
10737 /// Get a call instruction from the call sequence chain.
10738 /// Tail calls are not allowed.
10739 assert(CallEnd->getOpcode() == ISD::CALLSEQ_END &&
10740 "Expected a callseq node.");
10741 SDNode *Call = CallEnd->getOperand(Num: 0).getNode();
10742 bool HasGlue = Call->getGluedNode();
10743
10744 // Replace the target specific call node with the patchable intrinsic.
10745 SmallVector<SDValue, 8> Ops;
10746
10747 // Push the chain.
10748 Ops.push_back(Elt: *(Call->op_begin()));
10749
10750 // Optionally, push the glue (if any).
10751 if (HasGlue)
10752 Ops.push_back(Elt: *(Call->op_end() - 1));
10753
10754 // Push the register mask info.
10755 if (HasGlue)
10756 Ops.push_back(Elt: *(Call->op_end() - 2));
10757 else
10758 Ops.push_back(Elt: *(Call->op_end() - 1));
10759
10760 // Add the <id> and <numBytes> constants.
10761 SDValue IDVal = getValue(V: CB.getArgOperand(i: PatchPointOpers::IDPos));
10762 Ops.push_back(DAG.getTargetConstant(IDVal->getAsZExtVal(), dl, MVT::i64));
10763 SDValue NBytesVal = getValue(V: CB.getArgOperand(i: PatchPointOpers::NBytesPos));
10764 Ops.push_back(DAG.getTargetConstant(NBytesVal->getAsZExtVal(), dl, MVT::i32));
10765
10766 // Add the callee.
10767 Ops.push_back(Elt: Callee);
10768
10769 // Adjust <numArgs> to account for any arguments that have been passed on the
10770 // stack instead.
10771 // Call Node: Chain, Target, {Args}, RegMask, [Glue]
10772 unsigned NumCallRegArgs = Call->getNumOperands() - (HasGlue ? 4 : 3);
10773 NumCallRegArgs = IsAnyRegCC ? NumArgs : NumCallRegArgs;
10774 Ops.push_back(DAG.getTargetConstant(NumCallRegArgs, dl, MVT::i32));
10775
10776 // Add the calling convention
10777 Ops.push_back(DAG.getTargetConstant((unsigned)CC, dl, MVT::i32));
10778
10779 // Add the arguments we omitted previously. The register allocator should
10780 // place these in any free register.
10781 if (IsAnyRegCC)
10782 for (unsigned i = NumMetaOpers, e = NumMetaOpers + NumArgs; i != e; ++i)
10783 Ops.push_back(Elt: getValue(V: CB.getArgOperand(i)));
10784
10785 // Push the arguments from the call instruction.
10786 SDNode::op_iterator e = HasGlue ? Call->op_end()-2 : Call->op_end()-1;
10787 Ops.append(in_start: Call->op_begin() + 2, in_end: e);
10788
10789 // Push live variables for the stack map.
10790 addStackMapLiveVars(Call: CB, StartIdx: NumMetaOpers + NumArgs, DL: dl, Ops, Builder&: *this);
10791
10792 SDVTList NodeTys;
10793 if (IsAnyRegCC && HasDef) {
10794 // Create the return types based on the intrinsic definition
10795 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
10796 SmallVector<EVT, 3> ValueVTs;
10797 ComputeValueVTs(TLI, DL: DAG.getDataLayout(), Ty: CB.getType(), ValueVTs);
10798 assert(ValueVTs.size() == 1 && "Expected only one return value type.");
10799
10800 // There is always a chain and a glue type at the end
10801 ValueVTs.push_back(MVT::Other);
10802 ValueVTs.push_back(MVT::Glue);
10803 NodeTys = DAG.getVTList(VTs: ValueVTs);
10804 } else
10805 NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
10806
10807 // Replace the target specific call node with a PATCHPOINT node.
10808 SDValue PPV = DAG.getNode(Opcode: ISD::PATCHPOINT, DL: dl, VTList: NodeTys, Ops);
10809
10810 // Update the NodeMap.
10811 if (HasDef) {
10812 if (IsAnyRegCC)
10813 setValue(V: &CB, NewN: SDValue(PPV.getNode(), 0));
10814 else
10815 setValue(V: &CB, NewN: Result.first);
10816 }
10817
10818 // Fixup the consumers of the intrinsic. The chain and glue may be used in the
10819 // call sequence. Furthermore the location of the chain and glue can change
10820 // when the AnyReg calling convention is used and the intrinsic returns a
10821 // value.
10822 if (IsAnyRegCC && HasDef) {
10823 SDValue From[] = {SDValue(Call, 0), SDValue(Call, 1)};
10824 SDValue To[] = {PPV.getValue(R: 1), PPV.getValue(R: 2)};
10825 DAG.ReplaceAllUsesOfValuesWith(From, To, Num: 2);
10826 } else
10827 DAG.ReplaceAllUsesWith(From: Call, To: PPV.getNode());
10828 DAG.DeleteNode(N: Call);
10829
10830 // Inform the Frame Information that we have a patchpoint in this function.
10831 FuncInfo.MF->getFrameInfo().setHasPatchPoint();
10832}
10833
10834void SelectionDAGBuilder::visitVectorReduce(const CallInst &I,
10835 unsigned Intrinsic) {
10836 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
10837 SDValue Op1 = getValue(V: I.getArgOperand(i: 0));
10838 SDValue Op2;
10839 if (I.arg_size() > 1)
10840 Op2 = getValue(V: I.getArgOperand(i: 1));
10841 SDLoc dl = getCurSDLoc();
10842 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
10843 SDValue Res;
10844 SDNodeFlags SDFlags;
10845 if (auto *FPMO = dyn_cast<FPMathOperator>(Val: &I))
10846 SDFlags.copyFMF(FPMO: *FPMO);
10847
10848 switch (Intrinsic) {
10849 case Intrinsic::vector_reduce_fadd:
10850 if (SDFlags.hasAllowReassociation())
10851 Res = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT, N1: Op1,
10852 N2: DAG.getNode(Opcode: ISD::VECREDUCE_FADD, DL: dl, VT, Operand: Op2, Flags: SDFlags),
10853 Flags: SDFlags);
10854 else
10855 Res = DAG.getNode(Opcode: ISD::VECREDUCE_SEQ_FADD, DL: dl, VT, N1: Op1, N2: Op2, Flags: SDFlags);
10856 break;
10857 case Intrinsic::vector_reduce_fmul:
10858 if (SDFlags.hasAllowReassociation())
10859 Res = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT, N1: Op1,
10860 N2: DAG.getNode(Opcode: ISD::VECREDUCE_FMUL, DL: dl, VT, Operand: Op2, Flags: SDFlags),
10861 Flags: SDFlags);
10862 else
10863 Res = DAG.getNode(Opcode: ISD::VECREDUCE_SEQ_FMUL, DL: dl, VT, N1: Op1, N2: Op2, Flags: SDFlags);
10864 break;
10865 case Intrinsic::vector_reduce_add:
10866 Res = DAG.getNode(Opcode: ISD::VECREDUCE_ADD, DL: dl, VT, Operand: Op1);
10867 break;
10868 case Intrinsic::vector_reduce_mul:
10869 Res = DAG.getNode(Opcode: ISD::VECREDUCE_MUL, DL: dl, VT, Operand: Op1);
10870 break;
10871 case Intrinsic::vector_reduce_and:
10872 Res = DAG.getNode(Opcode: ISD::VECREDUCE_AND, DL: dl, VT, Operand: Op1);
10873 break;
10874 case Intrinsic::vector_reduce_or:
10875 Res = DAG.getNode(Opcode: ISD::VECREDUCE_OR, DL: dl, VT, Operand: Op1);
10876 break;
10877 case Intrinsic::vector_reduce_xor:
10878 Res = DAG.getNode(Opcode: ISD::VECREDUCE_XOR, DL: dl, VT, Operand: Op1);
10879 break;
10880 case Intrinsic::vector_reduce_smax:
10881 Res = DAG.getNode(Opcode: ISD::VECREDUCE_SMAX, DL: dl, VT, Operand: Op1);
10882 break;
10883 case Intrinsic::vector_reduce_smin:
10884 Res = DAG.getNode(Opcode: ISD::VECREDUCE_SMIN, DL: dl, VT, Operand: Op1);
10885 break;
10886 case Intrinsic::vector_reduce_umax:
10887 Res = DAG.getNode(Opcode: ISD::VECREDUCE_UMAX, DL: dl, VT, Operand: Op1);
10888 break;
10889 case Intrinsic::vector_reduce_umin:
10890 Res = DAG.getNode(Opcode: ISD::VECREDUCE_UMIN, DL: dl, VT, Operand: Op1);
10891 break;
10892 case Intrinsic::vector_reduce_fmax:
10893 Res = DAG.getNode(Opcode: ISD::VECREDUCE_FMAX, DL: dl, VT, Operand: Op1, Flags: SDFlags);
10894 break;
10895 case Intrinsic::vector_reduce_fmin:
10896 Res = DAG.getNode(Opcode: ISD::VECREDUCE_FMIN, DL: dl, VT, Operand: Op1, Flags: SDFlags);
10897 break;
10898 case Intrinsic::vector_reduce_fmaximum:
10899 Res = DAG.getNode(Opcode: ISD::VECREDUCE_FMAXIMUM, DL: dl, VT, Operand: Op1, Flags: SDFlags);
10900 break;
10901 case Intrinsic::vector_reduce_fminimum:
10902 Res = DAG.getNode(Opcode: ISD::VECREDUCE_FMINIMUM, DL: dl, VT, Operand: Op1, Flags: SDFlags);
10903 break;
10904 default:
10905 llvm_unreachable("Unhandled vector reduce intrinsic");
10906 }
10907 setValue(V: &I, NewN: Res);
10908}
10909
10910/// Returns an AttributeList representing the attributes applied to the return
10911/// value of the given call.
10912static AttributeList getReturnAttrs(TargetLowering::CallLoweringInfo &CLI) {
10913 SmallVector<Attribute::AttrKind, 2> Attrs;
10914 if (CLI.RetSExt)
10915 Attrs.push_back(Attribute::SExt);
10916 if (CLI.RetZExt)
10917 Attrs.push_back(Attribute::ZExt);
10918 if (CLI.IsInReg)
10919 Attrs.push_back(Attribute::InReg);
10920
10921 return AttributeList::get(C&: CLI.RetTy->getContext(), Index: AttributeList::ReturnIndex,
10922 Kinds: Attrs);
10923}
10924
10925/// TargetLowering::LowerCallTo - This is the default LowerCallTo
10926/// implementation, which just calls LowerCall.
10927/// FIXME: When all targets are
10928/// migrated to using LowerCall, this hook should be integrated into SDISel.
10929std::pair<SDValue, SDValue>
10930TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const {
10931 // Handle the incoming return values from the call.
10932 CLI.Ins.clear();
10933 SmallVector<EVT, 4> RetTys;
10934 SmallVector<TypeSize, 4> Offsets;
10935 auto &DL = CLI.DAG.getDataLayout();
10936 ComputeValueVTs(TLI: *this, DL, Ty: CLI.RetTy, ValueVTs&: RetTys, Offsets: &Offsets);
10937
10938 if (CLI.IsPostTypeLegalization) {
10939 // If we are lowering a libcall after legalization, split the return type.
10940 SmallVector<EVT, 4> OldRetTys;
10941 SmallVector<TypeSize, 4> OldOffsets;
10942 RetTys.swap(RHS&: OldRetTys);
10943 Offsets.swap(RHS&: OldOffsets);
10944
10945 for (size_t i = 0, e = OldRetTys.size(); i != e; ++i) {
10946 EVT RetVT = OldRetTys[i];
10947 uint64_t Offset = OldOffsets[i];
10948 MVT RegisterVT = getRegisterType(Context&: CLI.RetTy->getContext(), VT: RetVT);
10949 unsigned NumRegs = getNumRegisters(Context&: CLI.RetTy->getContext(), VT: RetVT);
10950 unsigned RegisterVTByteSZ = RegisterVT.getSizeInBits() / 8;
10951 RetTys.append(NumInputs: NumRegs, Elt: RegisterVT);
10952 for (unsigned j = 0; j != NumRegs; ++j)
10953 Offsets.push_back(Elt: TypeSize::getFixed(ExactSize: Offset + j * RegisterVTByteSZ));
10954 }
10955 }
10956
10957 SmallVector<ISD::OutputArg, 4> Outs;
10958 GetReturnInfo(CC: CLI.CallConv, ReturnType: CLI.RetTy, attr: getReturnAttrs(CLI), Outs, TLI: *this, DL);
10959
10960 bool CanLowerReturn =
10961 this->CanLowerReturn(CLI.CallConv, CLI.DAG.getMachineFunction(),
10962 CLI.IsVarArg, Outs, CLI.RetTy->getContext(), RetTy: CLI.RetTy);
10963
10964 SDValue DemoteStackSlot;
10965 int DemoteStackIdx = -100;
10966 if (!CanLowerReturn) {
10967 // FIXME: equivalent assert?
10968 // assert(!CS.hasInAllocaArgument() &&
10969 // "sret demotion is incompatible with inalloca");
10970 uint64_t TySize = DL.getTypeAllocSize(Ty: CLI.RetTy);
10971 Align Alignment = DL.getPrefTypeAlign(Ty: CLI.RetTy);
10972 MachineFunction &MF = CLI.DAG.getMachineFunction();
10973 DemoteStackIdx =
10974 MF.getFrameInfo().CreateStackObject(Size: TySize, Alignment, isSpillSlot: false);
10975 Type *StackSlotPtrType =
10976 PointerType::get(C&: CLI.RetTy->getContext(), AddressSpace: DL.getAllocaAddrSpace());
10977
10978 DemoteStackSlot = CLI.DAG.getFrameIndex(FI: DemoteStackIdx, VT: getFrameIndexTy(DL));
10979 ArgListEntry Entry;
10980 Entry.Node = DemoteStackSlot;
10981 Entry.Ty = StackSlotPtrType;
10982 Entry.IsSExt = false;
10983 Entry.IsZExt = false;
10984 Entry.IsInReg = false;
10985 Entry.IsSRet = true;
10986 Entry.IsNest = false;
10987 Entry.IsByVal = false;
10988 Entry.IsByRef = false;
10989 Entry.IsReturned = false;
10990 Entry.IsSwiftSelf = false;
10991 Entry.IsSwiftAsync = false;
10992 Entry.IsSwiftError = false;
10993 Entry.IsCFGuardTarget = false;
10994 Entry.Alignment = Alignment;
10995 CLI.getArgs().insert(position: CLI.getArgs().begin(), x: Entry);
10996 CLI.NumFixedArgs += 1;
10997 CLI.getArgs()[0].IndirectType = CLI.RetTy;
10998 CLI.RetTy = Type::getVoidTy(C&: CLI.RetTy->getContext());
10999
11000 // sret demotion isn't compatible with tail-calls, since the sret argument
11001 // points into the callers stack frame.
11002 CLI.IsTailCall = false;
11003 } else {
11004 bool NeedsRegBlock = functionArgumentNeedsConsecutiveRegisters(
11005 Ty: CLI.RetTy, CallConv: CLI.CallConv, isVarArg: CLI.IsVarArg, DL);
11006 for (unsigned I = 0, E = RetTys.size(); I != E; ++I) {
11007 ISD::ArgFlagsTy Flags;
11008 if (NeedsRegBlock) {
11009 Flags.setInConsecutiveRegs();
11010 if (I == RetTys.size() - 1)
11011 Flags.setInConsecutiveRegsLast();
11012 }
11013 EVT VT = RetTys[I];
11014 MVT RegisterVT = getRegisterTypeForCallingConv(Context&: CLI.RetTy->getContext(),
11015 CC: CLI.CallConv, VT);
11016 unsigned NumRegs = getNumRegistersForCallingConv(Context&: CLI.RetTy->getContext(),
11017 CC: CLI.CallConv, VT);
11018 for (unsigned i = 0; i != NumRegs; ++i) {
11019 ISD::InputArg MyFlags;
11020 MyFlags.Flags = Flags;
11021 MyFlags.VT = RegisterVT;
11022 MyFlags.ArgVT = VT;
11023 MyFlags.Used = CLI.IsReturnValueUsed;
11024 if (CLI.RetTy->isPointerTy()) {
11025 MyFlags.Flags.setPointer();
11026 MyFlags.Flags.setPointerAddrSpace(
11027 cast<PointerType>(Val: CLI.RetTy)->getAddressSpace());
11028 }
11029 if (CLI.RetSExt)
11030 MyFlags.Flags.setSExt();
11031 if (CLI.RetZExt)
11032 MyFlags.Flags.setZExt();
11033 if (CLI.IsInReg)
11034 MyFlags.Flags.setInReg();
11035 CLI.Ins.push_back(Elt: MyFlags);
11036 }
11037 }
11038 }
11039
11040 // We push in swifterror return as the last element of CLI.Ins.
11041 ArgListTy &Args = CLI.getArgs();
11042 if (supportSwiftError()) {
11043 for (const ArgListEntry &Arg : Args) {
11044 if (Arg.IsSwiftError) {
11045 ISD::InputArg MyFlags;
11046 MyFlags.VT = getPointerTy(DL);
11047 MyFlags.ArgVT = EVT(getPointerTy(DL));
11048 MyFlags.Flags.setSwiftError();
11049 CLI.Ins.push_back(Elt: MyFlags);
11050 }
11051 }
11052 }
11053
11054 // Handle all of the outgoing arguments.
11055 CLI.Outs.clear();
11056 CLI.OutVals.clear();
11057 for (unsigned i = 0, e = Args.size(); i != e; ++i) {
11058 SmallVector<EVT, 4> ValueVTs;
11059 ComputeValueVTs(TLI: *this, DL, Ty: Args[i].Ty, ValueVTs);
11060 // FIXME: Split arguments if CLI.IsPostTypeLegalization
11061 Type *FinalType = Args[i].Ty;
11062 if (Args[i].IsByVal)
11063 FinalType = Args[i].IndirectType;
11064 bool NeedsRegBlock = functionArgumentNeedsConsecutiveRegisters(
11065 Ty: FinalType, CallConv: CLI.CallConv, isVarArg: CLI.IsVarArg, DL);
11066 for (unsigned Value = 0, NumValues = ValueVTs.size(); Value != NumValues;
11067 ++Value) {
11068 EVT VT = ValueVTs[Value];
11069 Type *ArgTy = VT.getTypeForEVT(Context&: CLI.RetTy->getContext());
11070 SDValue Op = SDValue(Args[i].Node.getNode(),
11071 Args[i].Node.getResNo() + Value);
11072 ISD::ArgFlagsTy Flags;
11073
11074 // Certain targets (such as MIPS), may have a different ABI alignment
11075 // for a type depending on the context. Give the target a chance to
11076 // specify the alignment it wants.
11077 const Align OriginalAlignment(getABIAlignmentForCallingConv(ArgTy, DL));
11078 Flags.setOrigAlign(OriginalAlignment);
11079
11080 if (Args[i].Ty->isPointerTy()) {
11081 Flags.setPointer();
11082 Flags.setPointerAddrSpace(
11083 cast<PointerType>(Val: Args[i].Ty)->getAddressSpace());
11084 }
11085 if (Args[i].IsZExt)
11086 Flags.setZExt();
11087 if (Args[i].IsSExt)
11088 Flags.setSExt();
11089 if (Args[i].IsNoExt)
11090 Flags.setNoExt();
11091 if (Args[i].IsInReg) {
11092 // If we are using vectorcall calling convention, a structure that is
11093 // passed InReg - is surely an HVA
11094 if (CLI.CallConv == CallingConv::X86_VectorCall &&
11095 isa<StructType>(Val: FinalType)) {
11096 // The first value of a structure is marked
11097 if (0 == Value)
11098 Flags.setHvaStart();
11099 Flags.setHva();
11100 }
11101 // Set InReg Flag
11102 Flags.setInReg();
11103 }
11104 if (Args[i].IsSRet)
11105 Flags.setSRet();
11106 if (Args[i].IsSwiftSelf)
11107 Flags.setSwiftSelf();
11108 if (Args[i].IsSwiftAsync)
11109 Flags.setSwiftAsync();
11110 if (Args[i].IsSwiftError)
11111 Flags.setSwiftError();
11112 if (Args[i].IsCFGuardTarget)
11113 Flags.setCFGuardTarget();
11114 if (Args[i].IsByVal)
11115 Flags.setByVal();
11116 if (Args[i].IsByRef)
11117 Flags.setByRef();
11118 if (Args[i].IsPreallocated) {
11119 Flags.setPreallocated();
11120 // Set the byval flag for CCAssignFn callbacks that don't know about
11121 // preallocated. This way we can know how many bytes we should've
11122 // allocated and how many bytes a callee cleanup function will pop. If
11123 // we port preallocated to more targets, we'll have to add custom
11124 // preallocated handling in the various CC lowering callbacks.
11125 Flags.setByVal();
11126 }
11127 if (Args[i].IsInAlloca) {
11128 Flags.setInAlloca();
11129 // Set the byval flag for CCAssignFn callbacks that don't know about
11130 // inalloca. This way we can know how many bytes we should've allocated
11131 // and how many bytes a callee cleanup function will pop. If we port
11132 // inalloca to more targets, we'll have to add custom inalloca handling
11133 // in the various CC lowering callbacks.
11134 Flags.setByVal();
11135 }
11136 Align MemAlign;
11137 if (Args[i].IsByVal || Args[i].IsInAlloca || Args[i].IsPreallocated) {
11138 unsigned FrameSize = DL.getTypeAllocSize(Ty: Args[i].IndirectType);
11139 Flags.setByValSize(FrameSize);
11140
11141 // info is not there but there are cases it cannot get right.
11142 if (auto MA = Args[i].Alignment)
11143 MemAlign = *MA;
11144 else
11145 MemAlign = getByValTypeAlignment(Ty: Args[i].IndirectType, DL);
11146 } else if (auto MA = Args[i].Alignment) {
11147 MemAlign = *MA;
11148 } else {
11149 MemAlign = OriginalAlignment;
11150 }
11151 Flags.setMemAlign(MemAlign);
11152 if (Args[i].IsNest)
11153 Flags.setNest();
11154 if (NeedsRegBlock)
11155 Flags.setInConsecutiveRegs();
11156
11157 MVT PartVT = getRegisterTypeForCallingConv(Context&: CLI.RetTy->getContext(),
11158 CC: CLI.CallConv, VT);
11159 unsigned NumParts = getNumRegistersForCallingConv(Context&: CLI.RetTy->getContext(),
11160 CC: CLI.CallConv, VT);
11161 SmallVector<SDValue, 4> Parts(NumParts);
11162 ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
11163
11164 if (Args[i].IsSExt)
11165 ExtendKind = ISD::SIGN_EXTEND;
11166 else if (Args[i].IsZExt)
11167 ExtendKind = ISD::ZERO_EXTEND;
11168
11169 // Conservatively only handle 'returned' on non-vectors that can be lowered,
11170 // for now.
11171 if (Args[i].IsReturned && !Op.getValueType().isVector() &&
11172 CanLowerReturn) {
11173 assert((CLI.RetTy == Args[i].Ty ||
11174 (CLI.RetTy->isPointerTy() && Args[i].Ty->isPointerTy() &&
11175 CLI.RetTy->getPointerAddressSpace() ==
11176 Args[i].Ty->getPointerAddressSpace())) &&
11177 RetTys.size() == NumValues && "unexpected use of 'returned'");
11178 // Before passing 'returned' to the target lowering code, ensure that
11179 // either the register MVT and the actual EVT are the same size or that
11180 // the return value and argument are extended in the same way; in these
11181 // cases it's safe to pass the argument register value unchanged as the
11182 // return register value (although it's at the target's option whether
11183 // to do so)
11184 // TODO: allow code generation to take advantage of partially preserved
11185 // registers rather than clobbering the entire register when the
11186 // parameter extension method is not compatible with the return
11187 // extension method
11188 if ((NumParts * PartVT.getSizeInBits() == VT.getSizeInBits()) ||
11189 (ExtendKind != ISD::ANY_EXTEND && CLI.RetSExt == Args[i].IsSExt &&
11190 CLI.RetZExt == Args[i].IsZExt))
11191 Flags.setReturned();
11192 }
11193
11194 getCopyToParts(DAG&: CLI.DAG, DL: CLI.DL, Val: Op, Parts: &Parts[0], NumParts, PartVT, V: CLI.CB,
11195 CallConv: CLI.CallConv, ExtendKind);
11196
11197 for (unsigned j = 0; j != NumParts; ++j) {
11198 // if it isn't first piece, alignment must be 1
11199 // For scalable vectors the scalable part is currently handled
11200 // by individual targets, so we just use the known minimum size here.
11201 ISD::OutputArg MyFlags(
11202 Flags, Parts[j].getValueType().getSimpleVT(), VT,
11203 i < CLI.NumFixedArgs, i,
11204 j * Parts[j].getValueType().getStoreSize().getKnownMinValue());
11205 if (NumParts > 1 && j == 0)
11206 MyFlags.Flags.setSplit();
11207 else if (j != 0) {
11208 MyFlags.Flags.setOrigAlign(Align(1));
11209 if (j == NumParts - 1)
11210 MyFlags.Flags.setSplitEnd();
11211 }
11212
11213 CLI.Outs.push_back(Elt: MyFlags);
11214 CLI.OutVals.push_back(Elt: Parts[j]);
11215 }
11216
11217 if (NeedsRegBlock && Value == NumValues - 1)
11218 CLI.Outs[CLI.Outs.size() - 1].Flags.setInConsecutiveRegsLast();
11219 }
11220 }
11221
11222 SmallVector<SDValue, 4> InVals;
11223 CLI.Chain = LowerCall(CLI, InVals);
11224
11225 // Update CLI.InVals to use outside of this function.
11226 CLI.InVals = InVals;
11227
11228 // Verify that the target's LowerCall behaved as expected.
11229 assert(CLI.Chain.getNode() && CLI.Chain.getValueType() == MVT::Other &&
11230 "LowerCall didn't return a valid chain!");
11231 assert((!CLI.IsTailCall || InVals.empty()) &&
11232 "LowerCall emitted a return value for a tail call!");
11233 assert((CLI.IsTailCall || InVals.size() == CLI.Ins.size()) &&
11234 "LowerCall didn't emit the correct number of values!");
11235
11236 // For a tail call, the return value is merely live-out and there aren't
11237 // any nodes in the DAG representing it. Return a special value to
11238 // indicate that a tail call has been emitted and no more Instructions
11239 // should be processed in the current block.
11240 if (CLI.IsTailCall) {
11241 CLI.DAG.setRoot(CLI.Chain);
11242 return std::make_pair(x: SDValue(), y: SDValue());
11243 }
11244
11245#ifndef NDEBUG
11246 for (unsigned i = 0, e = CLI.Ins.size(); i != e; ++i) {
11247 assert(InVals[i].getNode() && "LowerCall emitted a null value!");
11248 assert(EVT(CLI.Ins[i].VT) == InVals[i].getValueType() &&
11249 "LowerCall emitted a value with the wrong type!");
11250 }
11251#endif
11252
11253 SmallVector<SDValue, 4> ReturnValues;
11254 if (!CanLowerReturn) {
11255 // The instruction result is the result of loading from the
11256 // hidden sret parameter.
11257 MVT PtrVT = getPointerTy(DL, AS: DL.getAllocaAddrSpace());
11258
11259 unsigned NumValues = RetTys.size();
11260 ReturnValues.resize(N: NumValues);
11261 SmallVector<SDValue, 4> Chains(NumValues);
11262
11263 // An aggregate return value cannot wrap around the address space, so
11264 // offsets to its parts don't wrap either.
11265 MachineFunction &MF = CLI.DAG.getMachineFunction();
11266 Align HiddenSRetAlign = MF.getFrameInfo().getObjectAlign(ObjectIdx: DemoteStackIdx);
11267 for (unsigned i = 0; i < NumValues; ++i) {
11268 SDValue Add = CLI.DAG.getMemBasePlusOffset(
11269 Base: DemoteStackSlot, Offset: CLI.DAG.getConstant(Val: Offsets[i], DL: CLI.DL, VT: PtrVT),
11270 DL: CLI.DL, Flags: SDNodeFlags::NoUnsignedWrap);
11271 SDValue L = CLI.DAG.getLoad(
11272 VT: RetTys[i], dl: CLI.DL, Chain: CLI.Chain, Ptr: Add,
11273 PtrInfo: MachinePointerInfo::getFixedStack(MF&: CLI.DAG.getMachineFunction(),
11274 FI: DemoteStackIdx, Offset: Offsets[i]),
11275 Alignment: HiddenSRetAlign);
11276 ReturnValues[i] = L;
11277 Chains[i] = L.getValue(R: 1);
11278 }
11279
11280 CLI.Chain = CLI.DAG.getNode(ISD::TokenFactor, CLI.DL, MVT::Other, Chains);
11281 } else {
11282 // Collect the legal value parts into potentially illegal values
11283 // that correspond to the original function's return values.
11284 std::optional<ISD::NodeType> AssertOp;
11285 if (CLI.RetSExt)
11286 AssertOp = ISD::AssertSext;
11287 else if (CLI.RetZExt)
11288 AssertOp = ISD::AssertZext;
11289 unsigned CurReg = 0;
11290 for (EVT VT : RetTys) {
11291 MVT RegisterVT = getRegisterTypeForCallingConv(Context&: CLI.RetTy->getContext(),
11292 CC: CLI.CallConv, VT);
11293 unsigned NumRegs = getNumRegistersForCallingConv(Context&: CLI.RetTy->getContext(),
11294 CC: CLI.CallConv, VT);
11295
11296 ReturnValues.push_back(Elt: getCopyFromParts(
11297 DAG&: CLI.DAG, DL: CLI.DL, Parts: &InVals[CurReg], NumParts: NumRegs, PartVT: RegisterVT, ValueVT: VT, V: nullptr,
11298 InChain: CLI.Chain, CC: CLI.CallConv, AssertOp));
11299 CurReg += NumRegs;
11300 }
11301
11302 // For a function returning void, there is no return value. We can't create
11303 // such a node, so we just return a null return value in that case. In
11304 // that case, nothing will actually look at the value.
11305 if (ReturnValues.empty())
11306 return std::make_pair(x: SDValue(), y&: CLI.Chain);
11307 }
11308
11309 SDValue Res = CLI.DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: CLI.DL,
11310 VTList: CLI.DAG.getVTList(VTs: RetTys), Ops: ReturnValues);
11311 return std::make_pair(x&: Res, y&: CLI.Chain);
11312}
11313
11314/// Places new result values for the node in Results (their number
11315/// and types must exactly match those of the original return values of
11316/// the node), or leaves Results empty, which indicates that the node is not
11317/// to be custom lowered after all.
11318void TargetLowering::LowerOperationWrapper(SDNode *N,
11319 SmallVectorImpl<SDValue> &Results,
11320 SelectionDAG &DAG) const {
11321 SDValue Res = LowerOperation(Op: SDValue(N, 0), DAG);
11322
11323 if (!Res.getNode())
11324 return;
11325
11326 // If the original node has one result, take the return value from
11327 // LowerOperation as is. It might not be result number 0.
11328 if (N->getNumValues() == 1) {
11329 Results.push_back(Elt: Res);
11330 return;
11331 }
11332
11333 // If the original node has multiple results, then the return node should
11334 // have the same number of results.
11335 assert((N->getNumValues() == Res->getNumValues()) &&
11336 "Lowering returned the wrong number of results!");
11337
11338 // Places new result values base on N result number.
11339 for (unsigned I = 0, E = N->getNumValues(); I != E; ++I)
11340 Results.push_back(Elt: Res.getValue(R: I));
11341}
11342
11343SDValue TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
11344 llvm_unreachable("LowerOperation not implemented for this target!");
11345}
11346
11347void SelectionDAGBuilder::CopyValueToVirtualRegister(const Value *V,
11348 Register Reg,
11349 ISD::NodeType ExtendType) {
11350 SDValue Op = getNonRegisterValue(V);
11351 assert((Op.getOpcode() != ISD::CopyFromReg ||
11352 cast<RegisterSDNode>(Op.getOperand(1))->getReg() != Reg) &&
11353 "Copy from a reg to the same reg!");
11354 assert(!Reg.isPhysical() && "Is a physreg");
11355
11356 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
11357 // If this is an InlineAsm we have to match the registers required, not the
11358 // notional registers required by the type.
11359
11360 RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), Reg, V->getType(),
11361 std::nullopt); // This is not an ABI copy.
11362 SDValue Chain = DAG.getEntryNode();
11363
11364 if (ExtendType == ISD::ANY_EXTEND) {
11365 auto PreferredExtendIt = FuncInfo.PreferredExtendType.find(Val: V);
11366 if (PreferredExtendIt != FuncInfo.PreferredExtendType.end())
11367 ExtendType = PreferredExtendIt->second;
11368 }
11369 RFV.getCopyToRegs(Val: Op, DAG, dl: getCurSDLoc(), Chain, Glue: nullptr, V, PreferredExtendType: ExtendType);
11370 PendingExports.push_back(Elt: Chain);
11371}
11372
11373#include "llvm/CodeGen/SelectionDAGISel.h"
11374
11375/// isOnlyUsedInEntryBlock - If the specified argument is only used in the
11376/// entry block, return true. This includes arguments used by switches, since
11377/// the switch may expand into multiple basic blocks.
11378static bool isOnlyUsedInEntryBlock(const Argument *A, bool FastISel) {
11379 // With FastISel active, we may be splitting blocks, so force creation
11380 // of virtual registers for all non-dead arguments.
11381 if (FastISel)
11382 return A->use_empty();
11383
11384 const BasicBlock &Entry = A->getParent()->front();
11385 for (const User *U : A->users())
11386 if (cast<Instruction>(Val: U)->getParent() != &Entry || isa<SwitchInst>(Val: U))
11387 return false; // Use not in entry block.
11388
11389 return true;
11390}
11391
11392using ArgCopyElisionMapTy =
11393 DenseMap<const Argument *,
11394 std::pair<const AllocaInst *, const StoreInst *>>;
11395
11396/// Scan the entry block of the function in FuncInfo for arguments that look
11397/// like copies into a local alloca. Record any copied arguments in
11398/// ArgCopyElisionCandidates.
11399static void
11400findArgumentCopyElisionCandidates(const DataLayout &DL,
11401 FunctionLoweringInfo *FuncInfo,
11402 ArgCopyElisionMapTy &ArgCopyElisionCandidates) {
11403 // Record the state of every static alloca used in the entry block. Argument
11404 // allocas are all used in the entry block, so we need approximately as many
11405 // entries as we have arguments.
11406 enum StaticAllocaInfo { Unknown, Clobbered, Elidable };
11407 SmallDenseMap<const AllocaInst *, StaticAllocaInfo, 8> StaticAllocas;
11408 unsigned NumArgs = FuncInfo->Fn->arg_size();
11409 StaticAllocas.reserve(NumEntries: NumArgs * 2);
11410
11411 auto GetInfoIfStaticAlloca = [&](const Value *V) -> StaticAllocaInfo * {
11412 if (!V)
11413 return nullptr;
11414 V = V->stripPointerCasts();
11415 const auto *AI = dyn_cast<AllocaInst>(Val: V);
11416 if (!AI || !AI->isStaticAlloca() || !FuncInfo->StaticAllocaMap.count(Val: AI))
11417 return nullptr;
11418 auto Iter = StaticAllocas.insert(KV: {AI, Unknown});
11419 return &Iter.first->second;
11420 };
11421
11422 // Look for stores of arguments to static allocas. Look through bitcasts and
11423 // GEPs to handle type coercions, as long as the alloca is fully initialized
11424 // by the store. Any non-store use of an alloca escapes it and any subsequent
11425 // unanalyzed store might write it.
11426 // FIXME: Handle structs initialized with multiple stores.
11427 for (const Instruction &I : FuncInfo->Fn->getEntryBlock()) {
11428 // Look for stores, and handle non-store uses conservatively.
11429 const auto *SI = dyn_cast<StoreInst>(Val: &I);
11430 if (!SI) {
11431 // We will look through cast uses, so ignore them completely.
11432 if (I.isCast())
11433 continue;
11434 // Ignore debug info and pseudo op intrinsics, they don't escape or store
11435 // to allocas.
11436 if (I.isDebugOrPseudoInst())
11437 continue;
11438 // This is an unknown instruction. Assume it escapes or writes to all
11439 // static alloca operands.
11440 for (const Use &U : I.operands()) {
11441 if (StaticAllocaInfo *Info = GetInfoIfStaticAlloca(U))
11442 *Info = StaticAllocaInfo::Clobbered;
11443 }
11444 continue;
11445 }
11446
11447 // If the stored value is a static alloca, mark it as escaped.
11448 if (StaticAllocaInfo *Info = GetInfoIfStaticAlloca(SI->getValueOperand()))
11449 *Info = StaticAllocaInfo::Clobbered;
11450
11451 // Check if the destination is a static alloca.
11452 const Value *Dst = SI->getPointerOperand()->stripPointerCasts();
11453 StaticAllocaInfo *Info = GetInfoIfStaticAlloca(Dst);
11454 if (!Info)
11455 continue;
11456 const AllocaInst *AI = cast<AllocaInst>(Val: Dst);
11457
11458 // Skip allocas that have been initialized or clobbered.
11459 if (*Info != StaticAllocaInfo::Unknown)
11460 continue;
11461
11462 // Check if the stored value is an argument, and that this store fully
11463 // initializes the alloca.
11464 // If the argument type has padding bits we can't directly forward a pointer
11465 // as the upper bits may contain garbage.
11466 // Don't elide copies from the same argument twice.
11467 const Value *Val = SI->getValueOperand()->stripPointerCasts();
11468 const auto *Arg = dyn_cast<Argument>(Val);
11469 if (!Arg || Arg->hasPassPointeeByValueCopyAttr() ||
11470 Arg->getType()->isEmptyTy() ||
11471 DL.getTypeStoreSize(Ty: Arg->getType()) !=
11472 DL.getTypeAllocSize(Ty: AI->getAllocatedType()) ||
11473 !DL.typeSizeEqualsStoreSize(Ty: Arg->getType()) ||
11474 ArgCopyElisionCandidates.count(Val: Arg)) {
11475 *Info = StaticAllocaInfo::Clobbered;
11476 continue;
11477 }
11478
11479 LLVM_DEBUG(dbgs() << "Found argument copy elision candidate: " << *AI
11480 << '\n');
11481
11482 // Mark this alloca and store for argument copy elision.
11483 *Info = StaticAllocaInfo::Elidable;
11484 ArgCopyElisionCandidates.insert(KV: {Arg, {AI, SI}});
11485
11486 // Stop scanning if we've seen all arguments. This will happen early in -O0
11487 // builds, which is useful, because -O0 builds have large entry blocks and
11488 // many allocas.
11489 if (ArgCopyElisionCandidates.size() == NumArgs)
11490 break;
11491 }
11492}
11493
11494/// Try to elide argument copies from memory into a local alloca. Succeeds if
11495/// ArgVal is a load from a suitable fixed stack object.
11496static void tryToElideArgumentCopy(
11497 FunctionLoweringInfo &FuncInfo, SmallVectorImpl<SDValue> &Chains,
11498 DenseMap<int, int> &ArgCopyElisionFrameIndexMap,
11499 SmallPtrSetImpl<const Instruction *> &ElidedArgCopyInstrs,
11500 ArgCopyElisionMapTy &ArgCopyElisionCandidates, const Argument &Arg,
11501 ArrayRef<SDValue> ArgVals, bool &ArgHasUses) {
11502 // Check if this is a load from a fixed stack object.
11503 auto *LNode = dyn_cast<LoadSDNode>(Val: ArgVals[0]);
11504 if (!LNode)
11505 return;
11506 auto *FINode = dyn_cast<FrameIndexSDNode>(Val: LNode->getBasePtr().getNode());
11507 if (!FINode)
11508 return;
11509
11510 // Check that the fixed stack object is the right size and alignment.
11511 // Look at the alignment that the user wrote on the alloca instead of looking
11512 // at the stack object.
11513 auto ArgCopyIter = ArgCopyElisionCandidates.find(Val: &Arg);
11514 assert(ArgCopyIter != ArgCopyElisionCandidates.end());
11515 const AllocaInst *AI = ArgCopyIter->second.first;
11516 int FixedIndex = FINode->getIndex();
11517 int &AllocaIndex = FuncInfo.StaticAllocaMap[AI];
11518 int OldIndex = AllocaIndex;
11519 MachineFrameInfo &MFI = FuncInfo.MF->getFrameInfo();
11520 if (MFI.getObjectSize(ObjectIdx: FixedIndex) != MFI.getObjectSize(ObjectIdx: OldIndex)) {
11521 LLVM_DEBUG(
11522 dbgs() << " argument copy elision failed due to bad fixed stack "
11523 "object size\n");
11524 return;
11525 }
11526 Align RequiredAlignment = AI->getAlign();
11527 if (MFI.getObjectAlign(ObjectIdx: FixedIndex) < RequiredAlignment) {
11528 LLVM_DEBUG(dbgs() << " argument copy elision failed: alignment of alloca "
11529 "greater than stack argument alignment ("
11530 << DebugStr(RequiredAlignment) << " vs "
11531 << DebugStr(MFI.getObjectAlign(FixedIndex)) << ")\n");
11532 return;
11533 }
11534
11535 // Perform the elision. Delete the old stack object and replace its only use
11536 // in the variable info map. Mark the stack object as mutable and aliased.
11537 LLVM_DEBUG({
11538 dbgs() << "Eliding argument copy from " << Arg << " to " << *AI << '\n'
11539 << " Replacing frame index " << OldIndex << " with " << FixedIndex
11540 << '\n';
11541 });
11542 MFI.RemoveStackObject(ObjectIdx: OldIndex);
11543 MFI.setIsImmutableObjectIndex(ObjectIdx: FixedIndex, IsImmutable: false);
11544 MFI.setIsAliasedObjectIndex(ObjectIdx: FixedIndex, IsAliased: true);
11545 AllocaIndex = FixedIndex;
11546 ArgCopyElisionFrameIndexMap.insert(KV: {OldIndex, FixedIndex});
11547 for (SDValue ArgVal : ArgVals)
11548 Chains.push_back(Elt: ArgVal.getValue(R: 1));
11549
11550 // Avoid emitting code for the store implementing the copy.
11551 const StoreInst *SI = ArgCopyIter->second.second;
11552 ElidedArgCopyInstrs.insert(Ptr: SI);
11553
11554 // Check for uses of the argument again so that we can avoid exporting ArgVal
11555 // if it is't used by anything other than the store.
11556 for (const Value *U : Arg.users()) {
11557 if (U != SI) {
11558 ArgHasUses = true;
11559 break;
11560 }
11561 }
11562}
11563
11564void SelectionDAGISel::LowerArguments(const Function &F) {
11565 SelectionDAG &DAG = SDB->DAG;
11566 SDLoc dl = SDB->getCurSDLoc();
11567 const DataLayout &DL = DAG.getDataLayout();
11568 SmallVector<ISD::InputArg, 16> Ins;
11569
11570 // In Naked functions we aren't going to save any registers.
11571 if (F.hasFnAttribute(Attribute::Naked))
11572 return;
11573
11574 if (!FuncInfo->CanLowerReturn) {
11575 // Put in an sret pointer parameter before all the other parameters.
11576 MVT ValueVT = TLI->getPointerTy(DL, AS: DL.getAllocaAddrSpace());
11577
11578 ISD::ArgFlagsTy Flags;
11579 Flags.setSRet();
11580 MVT RegisterVT = TLI->getRegisterType(Context&: *DAG.getContext(), VT: ValueVT);
11581 ISD::InputArg RetArg(Flags, RegisterVT, ValueVT, true,
11582 ISD::InputArg::NoArgIndex, 0);
11583 Ins.push_back(Elt: RetArg);
11584 }
11585
11586 // Look for stores of arguments to static allocas. Mark such arguments with a
11587 // flag to ask the target to give us the memory location of that argument if
11588 // available.
11589 ArgCopyElisionMapTy ArgCopyElisionCandidates;
11590 findArgumentCopyElisionCandidates(DL, FuncInfo: FuncInfo.get(),
11591 ArgCopyElisionCandidates);
11592
11593 // Set up the incoming argument description vector.
11594 for (const Argument &Arg : F.args()) {
11595 unsigned ArgNo = Arg.getArgNo();
11596 SmallVector<EVT, 4> ValueVTs;
11597 ComputeValueVTs(TLI: *TLI, DL: DAG.getDataLayout(), Ty: Arg.getType(), ValueVTs);
11598 bool isArgValueUsed = !Arg.use_empty();
11599 unsigned PartBase = 0;
11600 Type *FinalType = Arg.getType();
11601 if (Arg.hasAttribute(Attribute::ByVal))
11602 FinalType = Arg.getParamByValType();
11603 bool NeedsRegBlock = TLI->functionArgumentNeedsConsecutiveRegisters(
11604 Ty: FinalType, CallConv: F.getCallingConv(), isVarArg: F.isVarArg(), DL);
11605 for (unsigned Value = 0, NumValues = ValueVTs.size();
11606 Value != NumValues; ++Value) {
11607 EVT VT = ValueVTs[Value];
11608 Type *ArgTy = VT.getTypeForEVT(Context&: *DAG.getContext());
11609 ISD::ArgFlagsTy Flags;
11610
11611
11612 if (Arg.getType()->isPointerTy()) {
11613 Flags.setPointer();
11614 Flags.setPointerAddrSpace(
11615 cast<PointerType>(Val: Arg.getType())->getAddressSpace());
11616 }
11617 if (Arg.hasAttribute(Attribute::ZExt))
11618 Flags.setZExt();
11619 if (Arg.hasAttribute(Attribute::SExt))
11620 Flags.setSExt();
11621 if (Arg.hasAttribute(Attribute::InReg)) {
11622 // If we are using vectorcall calling convention, a structure that is
11623 // passed InReg - is surely an HVA
11624 if (F.getCallingConv() == CallingConv::X86_VectorCall &&
11625 isa<StructType>(Val: Arg.getType())) {
11626 // The first value of a structure is marked
11627 if (0 == Value)
11628 Flags.setHvaStart();
11629 Flags.setHva();
11630 }
11631 // Set InReg Flag
11632 Flags.setInReg();
11633 }
11634 if (Arg.hasAttribute(Attribute::StructRet))
11635 Flags.setSRet();
11636 if (Arg.hasAttribute(Attribute::SwiftSelf))
11637 Flags.setSwiftSelf();
11638 if (Arg.hasAttribute(Attribute::SwiftAsync))
11639 Flags.setSwiftAsync();
11640 if (Arg.hasAttribute(Attribute::SwiftError))
11641 Flags.setSwiftError();
11642 if (Arg.hasAttribute(Attribute::ByVal))
11643 Flags.setByVal();
11644 if (Arg.hasAttribute(Attribute::ByRef))
11645 Flags.setByRef();
11646 if (Arg.hasAttribute(Attribute::InAlloca)) {
11647 Flags.setInAlloca();
11648 // Set the byval flag for CCAssignFn callbacks that don't know about
11649 // inalloca. This way we can know how many bytes we should've allocated
11650 // and how many bytes a callee cleanup function will pop. If we port
11651 // inalloca to more targets, we'll have to add custom inalloca handling
11652 // in the various CC lowering callbacks.
11653 Flags.setByVal();
11654 }
11655 if (Arg.hasAttribute(Attribute::Preallocated)) {
11656 Flags.setPreallocated();
11657 // Set the byval flag for CCAssignFn callbacks that don't know about
11658 // preallocated. This way we can know how many bytes we should've
11659 // allocated and how many bytes a callee cleanup function will pop. If
11660 // we port preallocated to more targets, we'll have to add custom
11661 // preallocated handling in the various CC lowering callbacks.
11662 Flags.setByVal();
11663 }
11664
11665 // Certain targets (such as MIPS), may have a different ABI alignment
11666 // for a type depending on the context. Give the target a chance to
11667 // specify the alignment it wants.
11668 const Align OriginalAlignment(
11669 TLI->getABIAlignmentForCallingConv(ArgTy, DL));
11670 Flags.setOrigAlign(OriginalAlignment);
11671
11672 Align MemAlign;
11673 Type *ArgMemTy = nullptr;
11674 if (Flags.isByVal() || Flags.isInAlloca() || Flags.isPreallocated() ||
11675 Flags.isByRef()) {
11676 if (!ArgMemTy)
11677 ArgMemTy = Arg.getPointeeInMemoryValueType();
11678
11679 uint64_t MemSize = DL.getTypeAllocSize(Ty: ArgMemTy);
11680
11681 // For in-memory arguments, size and alignment should be passed from FE.
11682 // BE will guess if this info is not there but there are cases it cannot
11683 // get right.
11684 if (auto ParamAlign = Arg.getParamStackAlign())
11685 MemAlign = *ParamAlign;
11686 else if ((ParamAlign = Arg.getParamAlign()))
11687 MemAlign = *ParamAlign;
11688 else
11689 MemAlign = TLI->getByValTypeAlignment(Ty: ArgMemTy, DL);
11690 if (Flags.isByRef())
11691 Flags.setByRefSize(MemSize);
11692 else
11693 Flags.setByValSize(MemSize);
11694 } else if (auto ParamAlign = Arg.getParamStackAlign()) {
11695 MemAlign = *ParamAlign;
11696 } else {
11697 MemAlign = OriginalAlignment;
11698 }
11699 Flags.setMemAlign(MemAlign);
11700
11701 if (Arg.hasAttribute(Attribute::Nest))
11702 Flags.setNest();
11703 if (NeedsRegBlock)
11704 Flags.setInConsecutiveRegs();
11705 if (ArgCopyElisionCandidates.count(Val: &Arg))
11706 Flags.setCopyElisionCandidate();
11707 if (Arg.hasAttribute(Attribute::Returned))
11708 Flags.setReturned();
11709
11710 MVT RegisterVT = TLI->getRegisterTypeForCallingConv(
11711 Context&: *CurDAG->getContext(), CC: F.getCallingConv(), VT);
11712 unsigned NumRegs = TLI->getNumRegistersForCallingConv(
11713 Context&: *CurDAG->getContext(), CC: F.getCallingConv(), VT);
11714 for (unsigned i = 0; i != NumRegs; ++i) {
11715 // For scalable vectors, use the minimum size; individual targets
11716 // are responsible for handling scalable vector arguments and
11717 // return values.
11718 ISD::InputArg MyFlags(
11719 Flags, RegisterVT, VT, isArgValueUsed, ArgNo,
11720 PartBase + i * RegisterVT.getStoreSize().getKnownMinValue());
11721 if (NumRegs > 1 && i == 0)
11722 MyFlags.Flags.setSplit();
11723 // if it isn't first piece, alignment must be 1
11724 else if (i > 0) {
11725 MyFlags.Flags.setOrigAlign(Align(1));
11726 if (i == NumRegs - 1)
11727 MyFlags.Flags.setSplitEnd();
11728 }
11729 Ins.push_back(Elt: MyFlags);
11730 }
11731 if (NeedsRegBlock && Value == NumValues - 1)
11732 Ins[Ins.size() - 1].Flags.setInConsecutiveRegsLast();
11733 PartBase += VT.getStoreSize().getKnownMinValue();
11734 }
11735 }
11736
11737 // Call the target to set up the argument values.
11738 SmallVector<SDValue, 8> InVals;
11739 SDValue NewRoot = TLI->LowerFormalArguments(
11740 DAG.getRoot(), F.getCallingConv(), F.isVarArg(), Ins, dl, DAG, InVals);
11741
11742 // Verify that the target's LowerFormalArguments behaved as expected.
11743 assert(NewRoot.getNode() && NewRoot.getValueType() == MVT::Other &&
11744 "LowerFormalArguments didn't return a valid chain!");
11745 assert(InVals.size() == Ins.size() &&
11746 "LowerFormalArguments didn't emit the correct number of values!");
11747 LLVM_DEBUG({
11748 for (unsigned i = 0, e = Ins.size(); i != e; ++i) {
11749 assert(InVals[i].getNode() &&
11750 "LowerFormalArguments emitted a null value!");
11751 assert(EVT(Ins[i].VT) == InVals[i].getValueType() &&
11752 "LowerFormalArguments emitted a value with the wrong type!");
11753 }
11754 });
11755
11756 // Update the DAG with the new chain value resulting from argument lowering.
11757 DAG.setRoot(NewRoot);
11758
11759 // Set up the argument values.
11760 unsigned i = 0;
11761 if (!FuncInfo->CanLowerReturn) {
11762 // Create a virtual register for the sret pointer, and put in a copy
11763 // from the sret argument into it.
11764 MVT VT = TLI->getPointerTy(DL, AS: DL.getAllocaAddrSpace());
11765 MVT RegVT = TLI->getRegisterType(Context&: *CurDAG->getContext(), VT);
11766 std::optional<ISD::NodeType> AssertOp;
11767 SDValue ArgValue =
11768 getCopyFromParts(DAG, DL: dl, Parts: &InVals[0], NumParts: 1, PartVT: RegVT, ValueVT: VT, V: nullptr, InChain: NewRoot,
11769 CC: F.getCallingConv(), AssertOp);
11770
11771 MachineFunction& MF = SDB->DAG.getMachineFunction();
11772 MachineRegisterInfo& RegInfo = MF.getRegInfo();
11773 Register SRetReg =
11774 RegInfo.createVirtualRegister(RegClass: TLI->getRegClassFor(VT: RegVT));
11775 FuncInfo->DemoteRegister = SRetReg;
11776 NewRoot =
11777 SDB->DAG.getCopyToReg(Chain: NewRoot, dl: SDB->getCurSDLoc(), Reg: SRetReg, N: ArgValue);
11778 DAG.setRoot(NewRoot);
11779
11780 // i indexes lowered arguments. Bump it past the hidden sret argument.
11781 ++i;
11782 }
11783
11784 SmallVector<SDValue, 4> Chains;
11785 DenseMap<int, int> ArgCopyElisionFrameIndexMap;
11786 for (const Argument &Arg : F.args()) {
11787 SmallVector<SDValue, 4> ArgValues;
11788 SmallVector<EVT, 4> ValueVTs;
11789 ComputeValueVTs(TLI: *TLI, DL: DAG.getDataLayout(), Ty: Arg.getType(), ValueVTs);
11790 unsigned NumValues = ValueVTs.size();
11791 if (NumValues == 0)
11792 continue;
11793
11794 bool ArgHasUses = !Arg.use_empty();
11795
11796 // Elide the copying store if the target loaded this argument from a
11797 // suitable fixed stack object.
11798 if (Ins[i].Flags.isCopyElisionCandidate()) {
11799 unsigned NumParts = 0;
11800 for (EVT VT : ValueVTs)
11801 NumParts += TLI->getNumRegistersForCallingConv(Context&: *CurDAG->getContext(),
11802 CC: F.getCallingConv(), VT);
11803
11804 tryToElideArgumentCopy(FuncInfo&: *FuncInfo, Chains, ArgCopyElisionFrameIndexMap,
11805 ElidedArgCopyInstrs, ArgCopyElisionCandidates, Arg,
11806 ArgVals: ArrayRef(&InVals[i], NumParts), ArgHasUses);
11807 }
11808
11809 // If this argument is unused then remember its value. It is used to generate
11810 // debugging information.
11811 bool isSwiftErrorArg =
11812 TLI->supportSwiftError() &&
11813 Arg.hasAttribute(Attribute::SwiftError);
11814 if (!ArgHasUses && !isSwiftErrorArg) {
11815 SDB->setUnusedArgValue(V: &Arg, NewN: InVals[i]);
11816
11817 // Also remember any frame index for use in FastISel.
11818 if (FrameIndexSDNode *FI =
11819 dyn_cast<FrameIndexSDNode>(Val: InVals[i].getNode()))
11820 FuncInfo->setArgumentFrameIndex(A: &Arg, FI: FI->getIndex());
11821 }
11822
11823 for (unsigned Val = 0; Val != NumValues; ++Val) {
11824 EVT VT = ValueVTs[Val];
11825 MVT PartVT = TLI->getRegisterTypeForCallingConv(Context&: *CurDAG->getContext(),
11826 CC: F.getCallingConv(), VT);
11827 unsigned NumParts = TLI->getNumRegistersForCallingConv(
11828 Context&: *CurDAG->getContext(), CC: F.getCallingConv(), VT);
11829
11830 // Even an apparent 'unused' swifterror argument needs to be returned. So
11831 // we do generate a copy for it that can be used on return from the
11832 // function.
11833 if (ArgHasUses || isSwiftErrorArg) {
11834 std::optional<ISD::NodeType> AssertOp;
11835 if (Arg.hasAttribute(Attribute::SExt))
11836 AssertOp = ISD::AssertSext;
11837 else if (Arg.hasAttribute(Attribute::ZExt))
11838 AssertOp = ISD::AssertZext;
11839
11840 SDValue OutVal =
11841 getCopyFromParts(DAG, DL: dl, Parts: &InVals[i], NumParts, PartVT, ValueVT: VT, V: nullptr,
11842 InChain: NewRoot, CC: F.getCallingConv(), AssertOp);
11843
11844 FPClassTest NoFPClass = Arg.getNoFPClass();
11845 if (NoFPClass != fcNone) {
11846 SDValue SDNoFPClass = DAG.getTargetConstant(
11847 static_cast<uint64_t>(NoFPClass), dl, MVT::i32);
11848 OutVal = DAG.getNode(Opcode: ISD::AssertNoFPClass, DL: dl, VT: OutVal.getValueType(),
11849 N1: OutVal, N2: SDNoFPClass);
11850 }
11851 ArgValues.push_back(Elt: OutVal);
11852 }
11853
11854 i += NumParts;
11855 }
11856
11857 // We don't need to do anything else for unused arguments.
11858 if (ArgValues.empty())
11859 continue;
11860
11861 // Note down frame index.
11862 if (FrameIndexSDNode *FI =
11863 dyn_cast<FrameIndexSDNode>(Val: ArgValues[0].getNode()))
11864 FuncInfo->setArgumentFrameIndex(A: &Arg, FI: FI->getIndex());
11865
11866 SDValue Res = DAG.getMergeValues(Ops: ArrayRef(ArgValues.data(), NumValues),
11867 dl: SDB->getCurSDLoc());
11868
11869 SDB->setValue(V: &Arg, NewN: Res);
11870 if (!TM.Options.EnableFastISel && Res.getOpcode() == ISD::BUILD_PAIR) {
11871 // We want to associate the argument with the frame index, among
11872 // involved operands, that correspond to the lowest address. The
11873 // getCopyFromParts function, called earlier, is swapping the order of
11874 // the operands to BUILD_PAIR depending on endianness. The result of
11875 // that swapping is that the least significant bits of the argument will
11876 // be in the first operand of the BUILD_PAIR node, and the most
11877 // significant bits will be in the second operand.
11878 unsigned LowAddressOp = DAG.getDataLayout().isBigEndian() ? 1 : 0;
11879 if (LoadSDNode *LNode =
11880 dyn_cast<LoadSDNode>(Val: Res.getOperand(i: LowAddressOp).getNode()))
11881 if (FrameIndexSDNode *FI =
11882 dyn_cast<FrameIndexSDNode>(Val: LNode->getBasePtr().getNode()))
11883 FuncInfo->setArgumentFrameIndex(A: &Arg, FI: FI->getIndex());
11884 }
11885
11886 // Analyses past this point are naive and don't expect an assertion.
11887 if (Res.getOpcode() == ISD::AssertZext)
11888 Res = Res.getOperand(i: 0);
11889
11890 // Update the SwiftErrorVRegDefMap.
11891 if (Res.getOpcode() == ISD::CopyFromReg && isSwiftErrorArg) {
11892 Register Reg = cast<RegisterSDNode>(Val: Res.getOperand(i: 1))->getReg();
11893 if (Reg.isVirtual())
11894 SwiftError->setCurrentVReg(MBB: FuncInfo->MBB, SwiftError->getFunctionArg(),
11895 Reg);
11896 }
11897
11898 // If this argument is live outside of the entry block, insert a copy from
11899 // wherever we got it to the vreg that other BB's will reference it as.
11900 if (Res.getOpcode() == ISD::CopyFromReg) {
11901 // If we can, though, try to skip creating an unnecessary vreg.
11902 // FIXME: This isn't very clean... it would be nice to make this more
11903 // general.
11904 Register Reg = cast<RegisterSDNode>(Val: Res.getOperand(i: 1))->getReg();
11905 if (Reg.isVirtual()) {
11906 FuncInfo->ValueMap[&Arg] = Reg;
11907 continue;
11908 }
11909 }
11910 if (!isOnlyUsedInEntryBlock(A: &Arg, FastISel: TM.Options.EnableFastISel)) {
11911 FuncInfo->InitializeRegForValue(V: &Arg);
11912 SDB->CopyToExportRegsIfNeeded(V: &Arg);
11913 }
11914 }
11915
11916 if (!Chains.empty()) {
11917 Chains.push_back(Elt: NewRoot);
11918 NewRoot = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
11919 }
11920
11921 DAG.setRoot(NewRoot);
11922
11923 assert(i == InVals.size() && "Argument register count mismatch!");
11924
11925 // If any argument copy elisions occurred and we have debug info, update the
11926 // stale frame indices used in the dbg.declare variable info table.
11927 if (!ArgCopyElisionFrameIndexMap.empty()) {
11928 for (MachineFunction::VariableDbgInfo &VI :
11929 MF->getInStackSlotVariableDbgInfo()) {
11930 auto I = ArgCopyElisionFrameIndexMap.find(Val: VI.getStackSlot());
11931 if (I != ArgCopyElisionFrameIndexMap.end())
11932 VI.updateStackSlot(NewSlot: I->second);
11933 }
11934 }
11935
11936 // Finally, if the target has anything special to do, allow it to do so.
11937 emitFunctionEntryCode();
11938}
11939
11940/// Handle PHI nodes in successor blocks. Emit code into the SelectionDAG to
11941/// ensure constants are generated when needed. Remember the virtual registers
11942/// that need to be added to the Machine PHI nodes as input. We cannot just
11943/// directly add them, because expansion might result in multiple MBB's for one
11944/// BB. As such, the start of the BB might correspond to a different MBB than
11945/// the end.
11946void
11947SelectionDAGBuilder::HandlePHINodesInSuccessorBlocks(const BasicBlock *LLVMBB) {
11948 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
11949
11950 SmallPtrSet<MachineBasicBlock *, 4> SuccsHandled;
11951
11952 // Check PHI nodes in successors that expect a value to be available from this
11953 // block.
11954 for (const BasicBlock *SuccBB : successors(I: LLVMBB->getTerminator())) {
11955 if (!isa<PHINode>(Val: SuccBB->begin())) continue;
11956 MachineBasicBlock *SuccMBB = FuncInfo.getMBB(BB: SuccBB);
11957
11958 // If this terminator has multiple identical successors (common for
11959 // switches), only handle each succ once.
11960 if (!SuccsHandled.insert(Ptr: SuccMBB).second)
11961 continue;
11962
11963 MachineBasicBlock::iterator MBBI = SuccMBB->begin();
11964
11965 // At this point we know that there is a 1-1 correspondence between LLVM PHI
11966 // nodes and Machine PHI nodes, but the incoming operands have not been
11967 // emitted yet.
11968 for (const PHINode &PN : SuccBB->phis()) {
11969 // Ignore dead phi's.
11970 if (PN.use_empty())
11971 continue;
11972
11973 // Skip empty types
11974 if (PN.getType()->isEmptyTy())
11975 continue;
11976
11977 Register Reg;
11978 const Value *PHIOp = PN.getIncomingValueForBlock(BB: LLVMBB);
11979
11980 if (const auto *C = dyn_cast<Constant>(Val: PHIOp)) {
11981 Register &RegOut = ConstantsOut[C];
11982 if (!RegOut) {
11983 RegOut = FuncInfo.CreateRegs(V: &PN);
11984 // We need to zero/sign extend ConstantInt phi operands to match
11985 // assumptions in FunctionLoweringInfo::ComputePHILiveOutRegInfo.
11986 ISD::NodeType ExtendType = ISD::ANY_EXTEND;
11987 if (auto *CI = dyn_cast<ConstantInt>(Val: C))
11988 ExtendType = TLI.signExtendConstant(C: CI) ? ISD::SIGN_EXTEND
11989 : ISD::ZERO_EXTEND;
11990 CopyValueToVirtualRegister(V: C, Reg: RegOut, ExtendType);
11991 }
11992 Reg = RegOut;
11993 } else {
11994 DenseMap<const Value *, Register>::iterator I =
11995 FuncInfo.ValueMap.find(Val: PHIOp);
11996 if (I != FuncInfo.ValueMap.end())
11997 Reg = I->second;
11998 else {
11999 assert(isa<AllocaInst>(PHIOp) &&
12000 FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(PHIOp)) &&
12001 "Didn't codegen value into a register!??");
12002 Reg = FuncInfo.CreateRegs(V: &PN);
12003 CopyValueToVirtualRegister(V: PHIOp, Reg);
12004 }
12005 }
12006
12007 // Remember that this register needs to added to the machine PHI node as
12008 // the input for this MBB.
12009 SmallVector<EVT, 4> ValueVTs;
12010 ComputeValueVTs(TLI, DL: DAG.getDataLayout(), Ty: PN.getType(), ValueVTs);
12011 for (EVT VT : ValueVTs) {
12012 const unsigned NumRegisters = TLI.getNumRegisters(Context&: *DAG.getContext(), VT);
12013 for (unsigned i = 0; i != NumRegisters; ++i)
12014 FuncInfo.PHINodesToUpdate.emplace_back(args: &*MBBI++, args: Reg + i);
12015 Reg += NumRegisters;
12016 }
12017 }
12018 }
12019
12020 ConstantsOut.clear();
12021}
12022
12023MachineBasicBlock *SelectionDAGBuilder::NextBlock(MachineBasicBlock *MBB) {
12024 MachineFunction::iterator I(MBB);
12025 if (++I == FuncInfo.MF->end())
12026 return nullptr;
12027 return &*I;
12028}
12029
12030/// During lowering new call nodes can be created (such as memset, etc.).
12031/// Those will become new roots of the current DAG, but complications arise
12032/// when they are tail calls. In such cases, the call lowering will update
12033/// the root, but the builder still needs to know that a tail call has been
12034/// lowered in order to avoid generating an additional return.
12035void SelectionDAGBuilder::updateDAGForMaybeTailCall(SDValue MaybeTC) {
12036 // If the node is null, we do have a tail call.
12037 if (MaybeTC.getNode() != nullptr)
12038 DAG.setRoot(MaybeTC);
12039 else
12040 HasTailCall = true;
12041}
12042
12043void SelectionDAGBuilder::lowerWorkItem(SwitchWorkListItem W, Value *Cond,
12044 MachineBasicBlock *SwitchMBB,
12045 MachineBasicBlock *DefaultMBB) {
12046 MachineFunction *CurMF = FuncInfo.MF;
12047 MachineBasicBlock *NextMBB = nullptr;
12048 MachineFunction::iterator BBI(W.MBB);
12049 if (++BBI != FuncInfo.MF->end())
12050 NextMBB = &*BBI;
12051
12052 unsigned Size = W.LastCluster - W.FirstCluster + 1;
12053
12054 BranchProbabilityInfo *BPI = FuncInfo.BPI;
12055
12056 if (Size == 2 && W.MBB == SwitchMBB) {
12057 // If any two of the cases has the same destination, and if one value
12058 // is the same as the other, but has one bit unset that the other has set,
12059 // use bit manipulation to do two compares at once. For example:
12060 // "if (X == 6 || X == 4)" -> "if ((X|2) == 6)"
12061 // TODO: This could be extended to merge any 2 cases in switches with 3
12062 // cases.
12063 // TODO: Handle cases where W.CaseBB != SwitchBB.
12064 CaseCluster &Small = *W.FirstCluster;
12065 CaseCluster &Big = *W.LastCluster;
12066
12067 if (Small.Low == Small.High && Big.Low == Big.High &&
12068 Small.MBB == Big.MBB) {
12069 const APInt &SmallValue = Small.Low->getValue();
12070 const APInt &BigValue = Big.Low->getValue();
12071
12072 // Check that there is only one bit different.
12073 APInt CommonBit = BigValue ^ SmallValue;
12074 if (CommonBit.isPowerOf2()) {
12075 SDValue CondLHS = getValue(V: Cond);
12076 EVT VT = CondLHS.getValueType();
12077 SDLoc DL = getCurSDLoc();
12078
12079 SDValue Or = DAG.getNode(Opcode: ISD::OR, DL, VT, N1: CondLHS,
12080 N2: DAG.getConstant(Val: CommonBit, DL, VT));
12081 SDValue Cond = DAG.getSetCC(
12082 DL, MVT::i1, Or, DAG.getConstant(BigValue | SmallValue, DL, VT),
12083 ISD::SETEQ);
12084
12085 // Update successor info.
12086 // Both Small and Big will jump to Small.BB, so we sum up the
12087 // probabilities.
12088 addSuccessorWithProb(Src: SwitchMBB, Dst: Small.MBB, Prob: Small.Prob + Big.Prob);
12089 if (BPI)
12090 addSuccessorWithProb(
12091 Src: SwitchMBB, Dst: DefaultMBB,
12092 // The default destination is the first successor in IR.
12093 Prob: BPI->getEdgeProbability(Src: SwitchMBB->getBasicBlock(), IndexInSuccessors: (unsigned)0));
12094 else
12095 addSuccessorWithProb(Src: SwitchMBB, Dst: DefaultMBB);
12096
12097 // Insert the true branch.
12098 SDValue BrCond =
12099 DAG.getNode(ISD::BRCOND, DL, MVT::Other, getControlRoot(), Cond,
12100 DAG.getBasicBlock(Small.MBB));
12101 // Insert the false branch.
12102 BrCond = DAG.getNode(ISD::BR, DL, MVT::Other, BrCond,
12103 DAG.getBasicBlock(DefaultMBB));
12104
12105 DAG.setRoot(BrCond);
12106 return;
12107 }
12108 }
12109 }
12110
12111 if (TM.getOptLevel() != CodeGenOptLevel::None) {
12112 // Here, we order cases by probability so the most likely case will be
12113 // checked first. However, two clusters can have the same probability in
12114 // which case their relative ordering is non-deterministic. So we use Low
12115 // as a tie-breaker as clusters are guaranteed to never overlap.
12116 llvm::sort(Start: W.FirstCluster, End: W.LastCluster + 1,
12117 Comp: [](const CaseCluster &a, const CaseCluster &b) {
12118 return a.Prob != b.Prob ?
12119 a.Prob > b.Prob :
12120 a.Low->getValue().slt(RHS: b.Low->getValue());
12121 });
12122
12123 // Rearrange the case blocks so that the last one falls through if possible
12124 // without changing the order of probabilities.
12125 for (CaseClusterIt I = W.LastCluster; I > W.FirstCluster; ) {
12126 --I;
12127 if (I->Prob > W.LastCluster->Prob)
12128 break;
12129 if (I->Kind == CC_Range && I->MBB == NextMBB) {
12130 std::swap(a&: *I, b&: *W.LastCluster);
12131 break;
12132 }
12133 }
12134 }
12135
12136 // Compute total probability.
12137 BranchProbability DefaultProb = W.DefaultProb;
12138 BranchProbability UnhandledProbs = DefaultProb;
12139 for (CaseClusterIt I = W.FirstCluster; I <= W.LastCluster; ++I)
12140 UnhandledProbs += I->Prob;
12141
12142 MachineBasicBlock *CurMBB = W.MBB;
12143 for (CaseClusterIt I = W.FirstCluster, E = W.LastCluster; I <= E; ++I) {
12144 bool FallthroughUnreachable = false;
12145 MachineBasicBlock *Fallthrough;
12146 if (I == W.LastCluster) {
12147 // For the last cluster, fall through to the default destination.
12148 Fallthrough = DefaultMBB;
12149 FallthroughUnreachable = isa<UnreachableInst>(
12150 Val: DefaultMBB->getBasicBlock()->getFirstNonPHIOrDbg());
12151 } else {
12152 Fallthrough = CurMF->CreateMachineBasicBlock(BB: CurMBB->getBasicBlock());
12153 CurMF->insert(MBBI: BBI, MBB: Fallthrough);
12154 // Put Cond in a virtual register to make it available from the new blocks.
12155 ExportFromCurrentBlock(V: Cond);
12156 }
12157 UnhandledProbs -= I->Prob;
12158
12159 switch (I->Kind) {
12160 case CC_JumpTable: {
12161 // FIXME: Optimize away range check based on pivot comparisons.
12162 JumpTableHeader *JTH = &SL->JTCases[I->JTCasesIndex].first;
12163 SwitchCG::JumpTable *JT = &SL->JTCases[I->JTCasesIndex].second;
12164
12165 // The jump block hasn't been inserted yet; insert it here.
12166 MachineBasicBlock *JumpMBB = JT->MBB;
12167 CurMF->insert(MBBI: BBI, MBB: JumpMBB);
12168
12169 auto JumpProb = I->Prob;
12170 auto FallthroughProb = UnhandledProbs;
12171
12172 // If the default statement is a target of the jump table, we evenly
12173 // distribute the default probability to successors of CurMBB. Also
12174 // update the probability on the edge from JumpMBB to Fallthrough.
12175 for (MachineBasicBlock::succ_iterator SI = JumpMBB->succ_begin(),
12176 SE = JumpMBB->succ_end();
12177 SI != SE; ++SI) {
12178 if (*SI == DefaultMBB) {
12179 JumpProb += DefaultProb / 2;
12180 FallthroughProb -= DefaultProb / 2;
12181 JumpMBB->setSuccProbability(I: SI, Prob: DefaultProb / 2);
12182 JumpMBB->normalizeSuccProbs();
12183 break;
12184 }
12185 }
12186
12187 // If the default clause is unreachable, propagate that knowledge into
12188 // JTH->FallthroughUnreachable which will use it to suppress the range
12189 // check.
12190 //
12191 // However, don't do this if we're doing branch target enforcement,
12192 // because a table branch _without_ a range check can be a tempting JOP
12193 // gadget - out-of-bounds inputs that are impossible in correct
12194 // execution become possible again if an attacker can influence the
12195 // control flow. So if an attacker doesn't already have a BTI bypass
12196 // available, we don't want them to be able to get one out of this
12197 // table branch.
12198 if (FallthroughUnreachable) {
12199 Function &CurFunc = CurMF->getFunction();
12200 if (!CurFunc.hasFnAttribute(Kind: "branch-target-enforcement"))
12201 JTH->FallthroughUnreachable = true;
12202 }
12203
12204 if (!JTH->FallthroughUnreachable)
12205 addSuccessorWithProb(Src: CurMBB, Dst: Fallthrough, Prob: FallthroughProb);
12206 addSuccessorWithProb(Src: CurMBB, Dst: JumpMBB, Prob: JumpProb);
12207 CurMBB->normalizeSuccProbs();
12208
12209 // The jump table header will be inserted in our current block, do the
12210 // range check, and fall through to our fallthrough block.
12211 JTH->HeaderBB = CurMBB;
12212 JT->Default = Fallthrough; // FIXME: Move Default to JumpTableHeader.
12213
12214 // If we're in the right place, emit the jump table header right now.
12215 if (CurMBB == SwitchMBB) {
12216 visitJumpTableHeader(JT&: *JT, JTH&: *JTH, SwitchBB: SwitchMBB);
12217 JTH->Emitted = true;
12218 }
12219 break;
12220 }
12221 case CC_BitTests: {
12222 // FIXME: Optimize away range check based on pivot comparisons.
12223 BitTestBlock *BTB = &SL->BitTestCases[I->BTCasesIndex];
12224
12225 // The bit test blocks haven't been inserted yet; insert them here.
12226 for (BitTestCase &BTC : BTB->Cases)
12227 CurMF->insert(MBBI: BBI, MBB: BTC.ThisBB);
12228
12229 // Fill in fields of the BitTestBlock.
12230 BTB->Parent = CurMBB;
12231 BTB->Default = Fallthrough;
12232
12233 BTB->DefaultProb = UnhandledProbs;
12234 // If the cases in bit test don't form a contiguous range, we evenly
12235 // distribute the probability on the edge to Fallthrough to two
12236 // successors of CurMBB.
12237 if (!BTB->ContiguousRange) {
12238 BTB->Prob += DefaultProb / 2;
12239 BTB->DefaultProb -= DefaultProb / 2;
12240 }
12241
12242 if (FallthroughUnreachable)
12243 BTB->FallthroughUnreachable = true;
12244
12245 // If we're in the right place, emit the bit test header right now.
12246 if (CurMBB == SwitchMBB) {
12247 visitBitTestHeader(B&: *BTB, SwitchBB: SwitchMBB);
12248 BTB->Emitted = true;
12249 }
12250 break;
12251 }
12252 case CC_Range: {
12253 const Value *RHS, *LHS, *MHS;
12254 ISD::CondCode CC;
12255 if (I->Low == I->High) {
12256 // Check Cond == I->Low.
12257 CC = ISD::SETEQ;
12258 LHS = Cond;
12259 RHS=I->Low;
12260 MHS = nullptr;
12261 } else {
12262 // Check I->Low <= Cond <= I->High.
12263 CC = ISD::SETLE;
12264 LHS = I->Low;
12265 MHS = Cond;
12266 RHS = I->High;
12267 }
12268
12269 // If Fallthrough is unreachable, fold away the comparison.
12270 if (FallthroughUnreachable)
12271 CC = ISD::SETTRUE;
12272
12273 // The false probability is the sum of all unhandled cases.
12274 CaseBlock CB(CC, LHS, RHS, MHS, I->MBB, Fallthrough, CurMBB,
12275 getCurSDLoc(), I->Prob, UnhandledProbs);
12276
12277 if (CurMBB == SwitchMBB)
12278 visitSwitchCase(CB, SwitchBB: SwitchMBB);
12279 else
12280 SL->SwitchCases.push_back(x: CB);
12281
12282 break;
12283 }
12284 }
12285 CurMBB = Fallthrough;
12286 }
12287}
12288
12289void SelectionDAGBuilder::splitWorkItem(SwitchWorkList &WorkList,
12290 const SwitchWorkListItem &W,
12291 Value *Cond,
12292 MachineBasicBlock *SwitchMBB) {
12293 assert(W.FirstCluster->Low->getValue().slt(W.LastCluster->Low->getValue()) &&
12294 "Clusters not sorted?");
12295 assert(W.LastCluster - W.FirstCluster + 1 >= 2 && "Too small to split!");
12296
12297 auto [LastLeft, FirstRight, LeftProb, RightProb] =
12298 SL->computeSplitWorkItemInfo(W);
12299
12300 // Use the first element on the right as pivot since we will make less-than
12301 // comparisons against it.
12302 CaseClusterIt PivotCluster = FirstRight;
12303 assert(PivotCluster > W.FirstCluster);
12304 assert(PivotCluster <= W.LastCluster);
12305
12306 CaseClusterIt FirstLeft = W.FirstCluster;
12307 CaseClusterIt LastRight = W.LastCluster;
12308
12309 const ConstantInt *Pivot = PivotCluster->Low;
12310
12311 // New blocks will be inserted immediately after the current one.
12312 MachineFunction::iterator BBI(W.MBB);
12313 ++BBI;
12314
12315 // We will branch to the LHS if Value < Pivot. If LHS is a single cluster,
12316 // we can branch to its destination directly if it's squeezed exactly in
12317 // between the known lower bound and Pivot - 1.
12318 MachineBasicBlock *LeftMBB;
12319 if (FirstLeft == LastLeft && FirstLeft->Kind == CC_Range &&
12320 FirstLeft->Low == W.GE &&
12321 (FirstLeft->High->getValue() + 1LL) == Pivot->getValue()) {
12322 LeftMBB = FirstLeft->MBB;
12323 } else {
12324 LeftMBB = FuncInfo.MF->CreateMachineBasicBlock(BB: W.MBB->getBasicBlock());
12325 FuncInfo.MF->insert(MBBI: BBI, MBB: LeftMBB);
12326 WorkList.push_back(
12327 Elt: {.MBB: LeftMBB, .FirstCluster: FirstLeft, .LastCluster: LastLeft, .GE: W.GE, .LT: Pivot, .DefaultProb: W.DefaultProb / 2});
12328 // Put Cond in a virtual register to make it available from the new blocks.
12329 ExportFromCurrentBlock(V: Cond);
12330 }
12331
12332 // Similarly, we will branch to the RHS if Value >= Pivot. If RHS is a
12333 // single cluster, RHS.Low == Pivot, and we can branch to its destination
12334 // directly if RHS.High equals the current upper bound.
12335 MachineBasicBlock *RightMBB;
12336 if (FirstRight == LastRight && FirstRight->Kind == CC_Range &&
12337 W.LT && (FirstRight->High->getValue() + 1ULL) == W.LT->getValue()) {
12338 RightMBB = FirstRight->MBB;
12339 } else {
12340 RightMBB = FuncInfo.MF->CreateMachineBasicBlock(BB: W.MBB->getBasicBlock());
12341 FuncInfo.MF->insert(MBBI: BBI, MBB: RightMBB);
12342 WorkList.push_back(
12343 Elt: {.MBB: RightMBB, .FirstCluster: FirstRight, .LastCluster: LastRight, .GE: Pivot, .LT: W.LT, .DefaultProb: W.DefaultProb / 2});
12344 // Put Cond in a virtual register to make it available from the new blocks.
12345 ExportFromCurrentBlock(V: Cond);
12346 }
12347
12348 // Create the CaseBlock record that will be used to lower the branch.
12349 CaseBlock CB(ISD::SETLT, Cond, Pivot, nullptr, LeftMBB, RightMBB, W.MBB,
12350 getCurSDLoc(), LeftProb, RightProb);
12351
12352 if (W.MBB == SwitchMBB)
12353 visitSwitchCase(CB, SwitchBB: SwitchMBB);
12354 else
12355 SL->SwitchCases.push_back(x: CB);
12356}
12357
12358// Scale CaseProb after peeling a case with the probablity of PeeledCaseProb
12359// from the swith statement.
12360static BranchProbability scaleCaseProbality(BranchProbability CaseProb,
12361 BranchProbability PeeledCaseProb) {
12362 if (PeeledCaseProb == BranchProbability::getOne())
12363 return BranchProbability::getZero();
12364 BranchProbability SwitchProb = PeeledCaseProb.getCompl();
12365
12366 uint32_t Numerator = CaseProb.getNumerator();
12367 uint32_t Denominator = SwitchProb.scale(Num: CaseProb.getDenominator());
12368 return BranchProbability(Numerator, std::max(a: Numerator, b: Denominator));
12369}
12370
12371// Try to peel the top probability case if it exceeds the threshold.
12372// Return current MachineBasicBlock for the switch statement if the peeling
12373// does not occur.
12374// If the peeling is performed, return the newly created MachineBasicBlock
12375// for the peeled switch statement. Also update Clusters to remove the peeled
12376// case. PeeledCaseProb is the BranchProbability for the peeled case.
12377MachineBasicBlock *SelectionDAGBuilder::peelDominantCaseCluster(
12378 const SwitchInst &SI, CaseClusterVector &Clusters,
12379 BranchProbability &PeeledCaseProb) {
12380 MachineBasicBlock *SwitchMBB = FuncInfo.MBB;
12381 // Don't perform if there is only one cluster or optimizing for size.
12382 if (SwitchPeelThreshold > 100 || !FuncInfo.BPI || Clusters.size() < 2 ||
12383 TM.getOptLevel() == CodeGenOptLevel::None ||
12384 SwitchMBB->getParent()->getFunction().hasMinSize())
12385 return SwitchMBB;
12386
12387 BranchProbability TopCaseProb = BranchProbability(SwitchPeelThreshold, 100);
12388 unsigned PeeledCaseIndex = 0;
12389 bool SwitchPeeled = false;
12390 for (unsigned Index = 0; Index < Clusters.size(); ++Index) {
12391 CaseCluster &CC = Clusters[Index];
12392 if (CC.Prob < TopCaseProb)
12393 continue;
12394 TopCaseProb = CC.Prob;
12395 PeeledCaseIndex = Index;
12396 SwitchPeeled = true;
12397 }
12398 if (!SwitchPeeled)
12399 return SwitchMBB;
12400
12401 LLVM_DEBUG(dbgs() << "Peeled one top case in switch stmt, prob: "
12402 << TopCaseProb << "\n");
12403
12404 // Record the MBB for the peeled switch statement.
12405 MachineFunction::iterator BBI(SwitchMBB);
12406 ++BBI;
12407 MachineBasicBlock *PeeledSwitchMBB =
12408 FuncInfo.MF->CreateMachineBasicBlock(BB: SwitchMBB->getBasicBlock());
12409 FuncInfo.MF->insert(MBBI: BBI, MBB: PeeledSwitchMBB);
12410
12411 ExportFromCurrentBlock(V: SI.getCondition());
12412 auto PeeledCaseIt = Clusters.begin() + PeeledCaseIndex;
12413 SwitchWorkListItem W = {.MBB: SwitchMBB, .FirstCluster: PeeledCaseIt, .LastCluster: PeeledCaseIt,
12414 .GE: nullptr, .LT: nullptr, .DefaultProb: TopCaseProb.getCompl()};
12415 lowerWorkItem(W, Cond: SI.getCondition(), SwitchMBB, DefaultMBB: PeeledSwitchMBB);
12416
12417 Clusters.erase(position: PeeledCaseIt);
12418 for (CaseCluster &CC : Clusters) {
12419 LLVM_DEBUG(
12420 dbgs() << "Scale the probablity for one cluster, before scaling: "
12421 << CC.Prob << "\n");
12422 CC.Prob = scaleCaseProbality(CaseProb: CC.Prob, PeeledCaseProb: TopCaseProb);
12423 LLVM_DEBUG(dbgs() << "After scaling: " << CC.Prob << "\n");
12424 }
12425 PeeledCaseProb = TopCaseProb;
12426 return PeeledSwitchMBB;
12427}
12428
12429void SelectionDAGBuilder::visitSwitch(const SwitchInst &SI) {
12430 // Extract cases from the switch.
12431 BranchProbabilityInfo *BPI = FuncInfo.BPI;
12432 CaseClusterVector Clusters;
12433 Clusters.reserve(n: SI.getNumCases());
12434 for (auto I : SI.cases()) {
12435 MachineBasicBlock *Succ = FuncInfo.getMBB(BB: I.getCaseSuccessor());
12436 const ConstantInt *CaseVal = I.getCaseValue();
12437 BranchProbability Prob =
12438 BPI ? BPI->getEdgeProbability(Src: SI.getParent(), IndexInSuccessors: I.getSuccessorIndex())
12439 : BranchProbability(1, SI.getNumCases() + 1);
12440 Clusters.push_back(x: CaseCluster::range(Low: CaseVal, High: CaseVal, MBB: Succ, Prob));
12441 }
12442
12443 MachineBasicBlock *DefaultMBB = FuncInfo.getMBB(BB: SI.getDefaultDest());
12444
12445 // Cluster adjacent cases with the same destination. We do this at all
12446 // optimization levels because it's cheap to do and will make codegen faster
12447 // if there are many clusters.
12448 sortAndRangeify(Clusters);
12449
12450 // The branch probablity of the peeled case.
12451 BranchProbability PeeledCaseProb = BranchProbability::getZero();
12452 MachineBasicBlock *PeeledSwitchMBB =
12453 peelDominantCaseCluster(SI, Clusters, PeeledCaseProb);
12454
12455 // If there is only the default destination, jump there directly.
12456 MachineBasicBlock *SwitchMBB = FuncInfo.MBB;
12457 if (Clusters.empty()) {
12458 assert(PeeledSwitchMBB == SwitchMBB);
12459 SwitchMBB->addSuccessor(Succ: DefaultMBB);
12460 if (DefaultMBB != NextBlock(MBB: SwitchMBB)) {
12461 DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other,
12462 getControlRoot(), DAG.getBasicBlock(DefaultMBB)));
12463 }
12464 return;
12465 }
12466
12467 SL->findJumpTables(Clusters, SI: &SI, SL: getCurSDLoc(), DefaultMBB, PSI: DAG.getPSI(),
12468 BFI: DAG.getBFI());
12469 SL->findBitTestClusters(Clusters, SI: &SI);
12470
12471 LLVM_DEBUG({
12472 dbgs() << "Case clusters: ";
12473 for (const CaseCluster &C : Clusters) {
12474 if (C.Kind == CC_JumpTable)
12475 dbgs() << "JT:";
12476 if (C.Kind == CC_BitTests)
12477 dbgs() << "BT:";
12478
12479 C.Low->getValue().print(dbgs(), true);
12480 if (C.Low != C.High) {
12481 dbgs() << '-';
12482 C.High->getValue().print(dbgs(), true);
12483 }
12484 dbgs() << ' ';
12485 }
12486 dbgs() << '\n';
12487 });
12488
12489 assert(!Clusters.empty());
12490 SwitchWorkList WorkList;
12491 CaseClusterIt First = Clusters.begin();
12492 CaseClusterIt Last = Clusters.end() - 1;
12493 auto DefaultProb = getEdgeProbability(Src: PeeledSwitchMBB, Dst: DefaultMBB);
12494 // Scale the branchprobability for DefaultMBB if the peel occurs and
12495 // DefaultMBB is not replaced.
12496 if (PeeledCaseProb != BranchProbability::getZero() &&
12497 DefaultMBB == FuncInfo.getMBB(BB: SI.getDefaultDest()))
12498 DefaultProb = scaleCaseProbality(CaseProb: DefaultProb, PeeledCaseProb);
12499 WorkList.push_back(
12500 Elt: {.MBB: PeeledSwitchMBB, .FirstCluster: First, .LastCluster: Last, .GE: nullptr, .LT: nullptr, .DefaultProb: DefaultProb});
12501
12502 while (!WorkList.empty()) {
12503 SwitchWorkListItem W = WorkList.pop_back_val();
12504 unsigned NumClusters = W.LastCluster - W.FirstCluster + 1;
12505
12506 if (NumClusters > 3 && TM.getOptLevel() != CodeGenOptLevel::None &&
12507 !DefaultMBB->getParent()->getFunction().hasMinSize()) {
12508 // For optimized builds, lower large range as a balanced binary tree.
12509 splitWorkItem(WorkList, W, Cond: SI.getCondition(), SwitchMBB);
12510 continue;
12511 }
12512
12513 lowerWorkItem(W, Cond: SI.getCondition(), SwitchMBB, DefaultMBB);
12514 }
12515}
12516
12517void SelectionDAGBuilder::visitStepVector(const CallInst &I) {
12518 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
12519 auto DL = getCurSDLoc();
12520 EVT ResultVT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
12521 setValue(V: &I, NewN: DAG.getStepVector(DL, ResVT: ResultVT));
12522}
12523
12524void SelectionDAGBuilder::visitVectorReverse(const CallInst &I) {
12525 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
12526 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
12527
12528 SDLoc DL = getCurSDLoc();
12529 SDValue V = getValue(V: I.getOperand(i_nocapture: 0));
12530 assert(VT == V.getValueType() && "Malformed vector.reverse!");
12531
12532 if (VT.isScalableVector()) {
12533 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::VECTOR_REVERSE, DL, VT, Operand: V));
12534 return;
12535 }
12536
12537 // Use VECTOR_SHUFFLE for the fixed-length vector
12538 // to maintain existing behavior.
12539 SmallVector<int, 8> Mask;
12540 unsigned NumElts = VT.getVectorMinNumElements();
12541 for (unsigned i = 0; i != NumElts; ++i)
12542 Mask.push_back(Elt: NumElts - 1 - i);
12543
12544 setValue(V: &I, NewN: DAG.getVectorShuffle(VT, dl: DL, N1: V, N2: DAG.getUNDEF(VT), Mask));
12545}
12546
12547void SelectionDAGBuilder::visitVectorDeinterleave(const CallInst &I,
12548 unsigned Factor) {
12549 auto DL = getCurSDLoc();
12550 SDValue InVec = getValue(V: I.getOperand(i_nocapture: 0));
12551
12552 SmallVector<EVT, 4> ValueVTs;
12553 ComputeValueVTs(TLI: DAG.getTargetLoweringInfo(), DL: DAG.getDataLayout(), Ty: I.getType(),
12554 ValueVTs);
12555
12556 EVT OutVT = ValueVTs[0];
12557 unsigned OutNumElts = OutVT.getVectorMinNumElements();
12558
12559 SmallVector<SDValue, 4> SubVecs(Factor);
12560 for (unsigned i = 0; i != Factor; ++i) {
12561 assert(ValueVTs[i] == OutVT && "Expected VTs to be the same");
12562 SubVecs[i] = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: OutVT, N1: InVec,
12563 N2: DAG.getVectorIdxConstant(Val: OutNumElts * i, DL));
12564 }
12565
12566 // Use VECTOR_SHUFFLE for fixed-length vectors with factor of 2 to benefit
12567 // from existing legalisation and combines.
12568 if (OutVT.isFixedLengthVector() && Factor == 2) {
12569 SDValue Even = DAG.getVectorShuffle(VT: OutVT, dl: DL, N1: SubVecs[0], N2: SubVecs[1],
12570 Mask: createStrideMask(Start: 0, Stride: 2, VF: OutNumElts));
12571 SDValue Odd = DAG.getVectorShuffle(VT: OutVT, dl: DL, N1: SubVecs[0], N2: SubVecs[1],
12572 Mask: createStrideMask(Start: 1, Stride: 2, VF: OutNumElts));
12573 SDValue Res = DAG.getMergeValues(Ops: {Even, Odd}, dl: getCurSDLoc());
12574 setValue(V: &I, NewN: Res);
12575 return;
12576 }
12577
12578 SDValue Res = DAG.getNode(Opcode: ISD::VECTOR_DEINTERLEAVE, DL,
12579 VTList: DAG.getVTList(VTs: ValueVTs), Ops: SubVecs);
12580 setValue(V: &I, NewN: Res);
12581}
12582
12583void SelectionDAGBuilder::visitVectorInterleave(const CallInst &I,
12584 unsigned Factor) {
12585 auto DL = getCurSDLoc();
12586 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
12587 EVT InVT = getValue(V: I.getOperand(i_nocapture: 0)).getValueType();
12588 EVT OutVT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
12589
12590 SmallVector<SDValue, 8> InVecs(Factor);
12591 for (unsigned i = 0; i < Factor; ++i) {
12592 InVecs[i] = getValue(V: I.getOperand(i_nocapture: i));
12593 assert(InVecs[i].getValueType() == InVecs[0].getValueType() &&
12594 "Expected VTs to be the same");
12595 }
12596
12597 // Use VECTOR_SHUFFLE for fixed-length vectors with factor of 2 to benefit
12598 // from existing legalisation and combines.
12599 if (OutVT.isFixedLengthVector() && Factor == 2) {
12600 unsigned NumElts = InVT.getVectorMinNumElements();
12601 SDValue V = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: OutVT, Ops: InVecs);
12602 setValue(V: &I, NewN: DAG.getVectorShuffle(VT: OutVT, dl: DL, N1: V, N2: DAG.getUNDEF(VT: OutVT),
12603 Mask: createInterleaveMask(VF: NumElts, NumVecs: 2)));
12604 return;
12605 }
12606
12607 SmallVector<EVT, 8> ValueVTs(Factor, InVT);
12608 SDValue Res =
12609 DAG.getNode(Opcode: ISD::VECTOR_INTERLEAVE, DL, VTList: DAG.getVTList(VTs: ValueVTs), Ops: InVecs);
12610
12611 SmallVector<SDValue, 8> Results(Factor);
12612 for (unsigned i = 0; i < Factor; ++i)
12613 Results[i] = Res.getValue(R: i);
12614
12615 Res = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: OutVT, Ops: Results);
12616 setValue(V: &I, NewN: Res);
12617}
12618
12619void SelectionDAGBuilder::visitFreeze(const FreezeInst &I) {
12620 SmallVector<EVT, 4> ValueVTs;
12621 ComputeValueVTs(TLI: DAG.getTargetLoweringInfo(), DL: DAG.getDataLayout(), Ty: I.getType(),
12622 ValueVTs);
12623 unsigned NumValues = ValueVTs.size();
12624 if (NumValues == 0) return;
12625
12626 SmallVector<SDValue, 4> Values(NumValues);
12627 SDValue Op = getValue(V: I.getOperand(i_nocapture: 0));
12628
12629 for (unsigned i = 0; i != NumValues; ++i)
12630 Values[i] = DAG.getNode(Opcode: ISD::FREEZE, DL: getCurSDLoc(), VT: ValueVTs[i],
12631 Operand: SDValue(Op.getNode(), Op.getResNo() + i));
12632
12633 setValue(V: &I, NewN: DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: getCurSDLoc(),
12634 VTList: DAG.getVTList(VTs: ValueVTs), Ops: Values));
12635}
12636
12637void SelectionDAGBuilder::visitVectorSplice(const CallInst &I) {
12638 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
12639 EVT VT = TLI.getValueType(DL: DAG.getDataLayout(), Ty: I.getType());
12640
12641 SDLoc DL = getCurSDLoc();
12642 SDValue V1 = getValue(V: I.getOperand(i_nocapture: 0));
12643 SDValue V2 = getValue(V: I.getOperand(i_nocapture: 1));
12644 int64_t Imm = cast<ConstantInt>(Val: I.getOperand(i_nocapture: 2))->getSExtValue();
12645
12646 // VECTOR_SHUFFLE doesn't support a scalable mask so use a dedicated node.
12647 if (VT.isScalableVector()) {
12648 setValue(
12649 V: &I, NewN: DAG.getNode(Opcode: ISD::VECTOR_SPLICE, DL, VT, N1: V1, N2: V2,
12650 N3: DAG.getSignedConstant(
12651 Val: Imm, DL, VT: TLI.getVectorIdxTy(DL: DAG.getDataLayout()))));
12652 return;
12653 }
12654
12655 unsigned NumElts = VT.getVectorNumElements();
12656
12657 uint64_t Idx = (NumElts + Imm) % NumElts;
12658
12659 // Use VECTOR_SHUFFLE to maintain original behaviour for fixed-length vectors.
12660 SmallVector<int, 8> Mask;
12661 for (unsigned i = 0; i < NumElts; ++i)
12662 Mask.push_back(Elt: Idx + i);
12663 setValue(V: &I, NewN: DAG.getVectorShuffle(VT, dl: DL, N1: V1, N2: V2, Mask));
12664}
12665
12666// Consider the following MIR after SelectionDAG, which produces output in
12667// phyregs in the first case or virtregs in the second case.
12668//
12669// INLINEASM_BR ..., implicit-def $ebx, ..., implicit-def $edx
12670// %5:gr32 = COPY $ebx
12671// %6:gr32 = COPY $edx
12672// %1:gr32 = COPY %6:gr32
12673// %0:gr32 = COPY %5:gr32
12674//
12675// INLINEASM_BR ..., def %5:gr32, ..., def %6:gr32
12676// %1:gr32 = COPY %6:gr32
12677// %0:gr32 = COPY %5:gr32
12678//
12679// Given %0, we'd like to return $ebx in the first case and %5 in the second.
12680// Given %1, we'd like to return $edx in the first case and %6 in the second.
12681//
12682// If a callbr has outputs, it will have a single mapping in FuncInfo.ValueMap
12683// to a single virtreg (such as %0). The remaining outputs monotonically
12684// increase in virtreg number from there. If a callbr has no outputs, then it
12685// should not have a corresponding callbr landingpad; in fact, the callbr
12686// landingpad would not even be able to refer to such a callbr.
12687static Register FollowCopyChain(MachineRegisterInfo &MRI, Register Reg) {
12688 MachineInstr *MI = MRI.def_begin(RegNo: Reg)->getParent();
12689 // There is definitely at least one copy.
12690 assert(MI->getOpcode() == TargetOpcode::COPY &&
12691 "start of copy chain MUST be COPY");
12692 Reg = MI->getOperand(i: 1).getReg();
12693 MI = MRI.def_begin(RegNo: Reg)->getParent();
12694 // There may be an optional second copy.
12695 if (MI->getOpcode() == TargetOpcode::COPY) {
12696 assert(Reg.isVirtual() && "expected COPY of virtual register");
12697 Reg = MI->getOperand(i: 1).getReg();
12698 assert(Reg.isPhysical() && "expected COPY of physical register");
12699 MI = MRI.def_begin(RegNo: Reg)->getParent();
12700 }
12701 // The start of the chain must be an INLINEASM_BR.
12702 assert(MI->getOpcode() == TargetOpcode::INLINEASM_BR &&
12703 "end of copy chain MUST be INLINEASM_BR");
12704 return Reg;
12705}
12706
12707// We must do this walk rather than the simpler
12708// setValue(&I, getCopyFromRegs(CBR, CBR->getType()));
12709// otherwise we will end up with copies of virtregs only valid along direct
12710// edges.
12711void SelectionDAGBuilder::visitCallBrLandingPad(const CallInst &I) {
12712 SmallVector<EVT, 8> ResultVTs;
12713 SmallVector<SDValue, 8> ResultValues;
12714 const auto *CBR =
12715 cast<CallBrInst>(Val: I.getParent()->getUniquePredecessor()->getTerminator());
12716
12717 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
12718 const TargetRegisterInfo *TRI = DAG.getSubtarget().getRegisterInfo();
12719 MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
12720
12721 Register InitialDef = FuncInfo.ValueMap[CBR];
12722 SDValue Chain = DAG.getRoot();
12723
12724 // Re-parse the asm constraints string.
12725 TargetLowering::AsmOperandInfoVector TargetConstraints =
12726 TLI.ParseConstraints(DL: DAG.getDataLayout(), TRI, Call: *CBR);
12727 for (auto &T : TargetConstraints) {
12728 SDISelAsmOperandInfo OpInfo(T);
12729 if (OpInfo.Type != InlineAsm::isOutput)
12730 continue;
12731
12732 // Pencil in OpInfo.ConstraintType and OpInfo.ConstraintVT based on the
12733 // individual constraint.
12734 TLI.ComputeConstraintToUse(OpInfo, Op: OpInfo.CallOperand, DAG: &DAG);
12735
12736 switch (OpInfo.ConstraintType) {
12737 case TargetLowering::C_Register:
12738 case TargetLowering::C_RegisterClass: {
12739 // Fill in OpInfo.AssignedRegs.Regs.
12740 getRegistersForValue(DAG, DL: getCurSDLoc(), OpInfo, RefOpInfo&: OpInfo);
12741
12742 // getRegistersForValue may produce 1 to many registers based on whether
12743 // the OpInfo.ConstraintVT is legal on the target or not.
12744 for (Register &Reg : OpInfo.AssignedRegs.Regs) {
12745 Register OriginalDef = FollowCopyChain(MRI, Reg: InitialDef++);
12746 if (OriginalDef.isPhysical())
12747 FuncInfo.MBB->addLiveIn(PhysReg: OriginalDef);
12748 // Update the assigned registers to use the original defs.
12749 Reg = OriginalDef;
12750 }
12751
12752 SDValue V = OpInfo.AssignedRegs.getCopyFromRegs(
12753 DAG, FuncInfo, dl: getCurSDLoc(), Chain, Glue: nullptr, V: CBR);
12754 ResultValues.push_back(Elt: V);
12755 ResultVTs.push_back(Elt: OpInfo.ConstraintVT);
12756 break;
12757 }
12758 case TargetLowering::C_Other: {
12759 SDValue Flag;
12760 SDValue V = TLI.LowerAsmOutputForConstraint(Chain, Glue&: Flag, DL: getCurSDLoc(),
12761 OpInfo, DAG);
12762 ++InitialDef;
12763 ResultValues.push_back(Elt: V);
12764 ResultVTs.push_back(Elt: OpInfo.ConstraintVT);
12765 break;
12766 }
12767 default:
12768 break;
12769 }
12770 }
12771 SDValue V = DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: getCurSDLoc(),
12772 VTList: DAG.getVTList(VTs: ResultVTs), Ops: ResultValues);
12773 setValue(V: &I, NewN: V);
12774}
12775

Provided by KDAB

Privacy Policy
Improve your Profiling and Debugging skills
Find out more

source code of llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp