1//===-- lib/CodeGen/GlobalISel/CallLowering.cpp - Call lowering -----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This file implements some simple delegations needed for call lowering.
11///
12//===----------------------------------------------------------------------===//
13
14#include "llvm/CodeGen/GlobalISel/CallLowering.h"
15#include "llvm/CodeGen/Analysis.h"
16#include "llvm/CodeGen/CallingConvLower.h"
17#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
18#include "llvm/CodeGen/GlobalISel/Utils.h"
19#include "llvm/CodeGen/MachineFrameInfo.h"
20#include "llvm/CodeGen/MachineOperand.h"
21#include "llvm/CodeGen/MachineRegisterInfo.h"
22#include "llvm/CodeGen/TargetLowering.h"
23#include "llvm/IR/DataLayout.h"
24#include "llvm/IR/IntrinsicInst.h"
25#include "llvm/IR/LLVMContext.h"
26#include "llvm/IR/Module.h"
27#include "llvm/Target/TargetMachine.h"
28
29#define DEBUG_TYPE "call-lowering"
30
31using namespace llvm;
32
33void CallLowering::anchor() {}
34
35/// Helper function which updates \p Flags when \p AttrFn returns true.
36static void
37addFlagsUsingAttrFn(ISD::ArgFlagsTy &Flags,
38 const std::function<bool(Attribute::AttrKind)> &AttrFn) {
39 if (AttrFn(Attribute::SExt))
40 Flags.setSExt();
41 if (AttrFn(Attribute::ZExt))
42 Flags.setZExt();
43 if (AttrFn(Attribute::InReg))
44 Flags.setInReg();
45 if (AttrFn(Attribute::StructRet))
46 Flags.setSRet();
47 if (AttrFn(Attribute::Nest))
48 Flags.setNest();
49 if (AttrFn(Attribute::ByVal))
50 Flags.setByVal();
51 if (AttrFn(Attribute::Preallocated))
52 Flags.setPreallocated();
53 if (AttrFn(Attribute::InAlloca))
54 Flags.setInAlloca();
55 if (AttrFn(Attribute::Returned))
56 Flags.setReturned();
57 if (AttrFn(Attribute::SwiftSelf))
58 Flags.setSwiftSelf();
59 if (AttrFn(Attribute::SwiftAsync))
60 Flags.setSwiftAsync();
61 if (AttrFn(Attribute::SwiftError))
62 Flags.setSwiftError();
63}
64
65ISD::ArgFlagsTy CallLowering::getAttributesForArgIdx(const CallBase &Call,
66 unsigned ArgIdx) const {
67 ISD::ArgFlagsTy Flags;
68 addFlagsUsingAttrFn(Flags, AttrFn: [&Call, &ArgIdx](Attribute::AttrKind Attr) {
69 return Call.paramHasAttr(ArgNo: ArgIdx, Kind: Attr);
70 });
71 return Flags;
72}
73
74ISD::ArgFlagsTy
75CallLowering::getAttributesForReturn(const CallBase &Call) const {
76 ISD::ArgFlagsTy Flags;
77 addFlagsUsingAttrFn(Flags, AttrFn: [&Call](Attribute::AttrKind Attr) {
78 return Call.hasRetAttr(Kind: Attr);
79 });
80 return Flags;
81}
82
83void CallLowering::addArgFlagsFromAttributes(ISD::ArgFlagsTy &Flags,
84 const AttributeList &Attrs,
85 unsigned OpIdx) const {
86 addFlagsUsingAttrFn(Flags, AttrFn: [&Attrs, &OpIdx](Attribute::AttrKind Attr) {
87 return Attrs.hasAttributeAtIndex(Index: OpIdx, Kind: Attr);
88 });
89}
90
91bool CallLowering::lowerCall(MachineIRBuilder &MIRBuilder, const CallBase &CB,
92 ArrayRef<Register> ResRegs,
93 ArrayRef<ArrayRef<Register>> ArgRegs,
94 Register SwiftErrorVReg,
95 Register ConvergenceCtrlToken,
96 std::function<unsigned()> GetCalleeReg) const {
97 CallLoweringInfo Info;
98 const DataLayout &DL = MIRBuilder.getDataLayout();
99 MachineFunction &MF = MIRBuilder.getMF();
100 MachineRegisterInfo &MRI = MF.getRegInfo();
101 bool CanBeTailCalled = CB.isTailCall() &&
102 isInTailCallPosition(Call: CB, TM: MF.getTarget()) &&
103 (MF.getFunction()
104 .getFnAttribute(Kind: "disable-tail-calls")
105 .getValueAsString() != "true");
106
107 CallingConv::ID CallConv = CB.getCallingConv();
108 Type *RetTy = CB.getType();
109 bool IsVarArg = CB.getFunctionType()->isVarArg();
110
111 SmallVector<BaseArgInfo, 4> SplitArgs;
112 getReturnInfo(CallConv, RetTy, Attrs: CB.getAttributes(), Outs&: SplitArgs, DL);
113 Info.CanLowerReturn = canLowerReturn(MF, CallConv, Outs&: SplitArgs, IsVarArg);
114
115 Info.IsConvergent = CB.isConvergent();
116
117 if (!Info.CanLowerReturn) {
118 // Callee requires sret demotion.
119 insertSRetOutgoingArgument(MIRBuilder, CB, Info);
120
121 // The sret demotion isn't compatible with tail-calls, since the sret
122 // argument points into the caller's stack frame.
123 CanBeTailCalled = false;
124 }
125
126 // First step is to marshall all the function's parameters into the correct
127 // physregs and memory locations. Gather the sequence of argument types that
128 // we'll pass to the assigner function.
129 unsigned i = 0;
130 unsigned NumFixedArgs = CB.getFunctionType()->getNumParams();
131 for (const auto &Arg : CB.args()) {
132 ArgInfo OrigArg{ArgRegs[i], *Arg.get(), i, getAttributesForArgIdx(Call: CB, ArgIdx: i),
133 i < NumFixedArgs};
134 setArgFlags(Arg&: OrigArg, OpIdx: i + AttributeList::FirstArgIndex, DL, FuncInfo: CB);
135
136 // If we have an explicit sret argument that is an Instruction, (i.e., it
137 // might point to function-local memory), we can't meaningfully tail-call.
138 if (OrigArg.Flags[0].isSRet() && isa<Instruction>(Val: &Arg))
139 CanBeTailCalled = false;
140
141 Info.OrigArgs.push_back(Elt: OrigArg);
142 ++i;
143 }
144
145 // Try looking through a bitcast from one function type to another.
146 // Commonly happens with calls to objc_msgSend().
147 const Value *CalleeV = CB.getCalledOperand()->stripPointerCasts();
148 if (const Function *F = dyn_cast<Function>(Val: CalleeV)) {
149 if (F->hasFnAttribute(Attribute::NonLazyBind)) {
150 LLT Ty = getLLTForType(Ty&: *F->getType(), DL);
151 Register Reg = MIRBuilder.buildGlobalValue(Res: Ty, GV: F).getReg(Idx: 0);
152 Info.Callee = MachineOperand::CreateReg(Reg, isDef: false);
153 } else {
154 Info.Callee = MachineOperand::CreateGA(GV: F, Offset: 0);
155 }
156 } else if (isa<GlobalIFunc>(Val: CalleeV) || isa<GlobalAlias>(Val: CalleeV)) {
157 // IR IFuncs and Aliases can't be forward declared (only defined), so the
158 // callee must be in the same TU and therefore we can direct-call it without
159 // worrying about it being out of range.
160 Info.Callee = MachineOperand::CreateGA(GV: cast<GlobalValue>(Val: CalleeV), Offset: 0);
161 } else
162 Info.Callee = MachineOperand::CreateReg(Reg: GetCalleeReg(), isDef: false);
163
164 Register ReturnHintAlignReg;
165 Align ReturnHintAlign;
166
167 Info.OrigRet = ArgInfo{ResRegs, RetTy, 0, getAttributesForReturn(Call: CB)};
168
169 if (!Info.OrigRet.Ty->isVoidTy()) {
170 setArgFlags(Arg&: Info.OrigRet, OpIdx: AttributeList::ReturnIndex, DL, FuncInfo: CB);
171
172 if (MaybeAlign Alignment = CB.getRetAlign()) {
173 if (*Alignment > Align(1)) {
174 ReturnHintAlignReg = MRI.cloneVirtualRegister(VReg: ResRegs[0]);
175 Info.OrigRet.Regs[0] = ReturnHintAlignReg;
176 ReturnHintAlign = *Alignment;
177 }
178 }
179 }
180
181 auto Bundle = CB.getOperandBundle(ID: LLVMContext::OB_kcfi);
182 if (Bundle && CB.isIndirectCall()) {
183 Info.CFIType = cast<ConstantInt>(Val: Bundle->Inputs[0]);
184 assert(Info.CFIType->getType()->isIntegerTy(32) && "Invalid CFI type");
185 }
186
187 Info.CB = &CB;
188 Info.KnownCallees = CB.getMetadata(KindID: LLVMContext::MD_callees);
189 Info.CallConv = CallConv;
190 Info.SwiftErrorVReg = SwiftErrorVReg;
191 Info.ConvergenceCtrlToken = ConvergenceCtrlToken;
192 Info.IsMustTailCall = CB.isMustTailCall();
193 Info.IsTailCall = CanBeTailCalled;
194 Info.IsVarArg = IsVarArg;
195 if (!lowerCall(MIRBuilder, Info))
196 return false;
197
198 if (ReturnHintAlignReg && !Info.LoweredTailCall) {
199 MIRBuilder.buildAssertAlign(Res: ResRegs[0], Op: ReturnHintAlignReg,
200 AlignVal: ReturnHintAlign);
201 }
202
203 return true;
204}
205
206template <typename FuncInfoTy>
207void CallLowering::setArgFlags(CallLowering::ArgInfo &Arg, unsigned OpIdx,
208 const DataLayout &DL,
209 const FuncInfoTy &FuncInfo) const {
210 auto &Flags = Arg.Flags[0];
211 const AttributeList &Attrs = FuncInfo.getAttributes();
212 addArgFlagsFromAttributes(Flags, Attrs, OpIdx);
213
214 PointerType *PtrTy = dyn_cast<PointerType>(Val: Arg.Ty->getScalarType());
215 if (PtrTy) {
216 Flags.setPointer();
217 Flags.setPointerAddrSpace(PtrTy->getPointerAddressSpace());
218 }
219
220 Align MemAlign = DL.getABITypeAlign(Ty: Arg.Ty);
221 if (Flags.isByVal() || Flags.isInAlloca() || Flags.isPreallocated()) {
222 assert(OpIdx >= AttributeList::FirstArgIndex);
223 unsigned ParamIdx = OpIdx - AttributeList::FirstArgIndex;
224
225 Type *ElementTy = FuncInfo.getParamByValType(ParamIdx);
226 if (!ElementTy)
227 ElementTy = FuncInfo.getParamInAllocaType(ParamIdx);
228 if (!ElementTy)
229 ElementTy = FuncInfo.getParamPreallocatedType(ParamIdx);
230 assert(ElementTy && "Must have byval, inalloca or preallocated type");
231 Flags.setByValSize(DL.getTypeAllocSize(Ty: ElementTy));
232
233 // For ByVal, alignment should be passed from FE. BE will guess if
234 // this info is not there but there are cases it cannot get right.
235 if (auto ParamAlign = FuncInfo.getParamStackAlign(ParamIdx))
236 MemAlign = *ParamAlign;
237 else if ((ParamAlign = FuncInfo.getParamAlign(ParamIdx)))
238 MemAlign = *ParamAlign;
239 else
240 MemAlign = Align(getTLI()->getByValTypeAlignment(Ty: ElementTy, DL));
241 } else if (OpIdx >= AttributeList::FirstArgIndex) {
242 if (auto ParamAlign =
243 FuncInfo.getParamStackAlign(OpIdx - AttributeList::FirstArgIndex))
244 MemAlign = *ParamAlign;
245 }
246 Flags.setMemAlign(MemAlign);
247 Flags.setOrigAlign(DL.getABITypeAlign(Ty: Arg.Ty));
248
249 // Don't try to use the returned attribute if the argument is marked as
250 // swiftself, since it won't be passed in x0.
251 if (Flags.isSwiftSelf())
252 Flags.setReturned(false);
253}
254
255template void
256CallLowering::setArgFlags<Function>(CallLowering::ArgInfo &Arg, unsigned OpIdx,
257 const DataLayout &DL,
258 const Function &FuncInfo) const;
259
260template void
261CallLowering::setArgFlags<CallBase>(CallLowering::ArgInfo &Arg, unsigned OpIdx,
262 const DataLayout &DL,
263 const CallBase &FuncInfo) const;
264
265void CallLowering::splitToValueTypes(const ArgInfo &OrigArg,
266 SmallVectorImpl<ArgInfo> &SplitArgs,
267 const DataLayout &DL,
268 CallingConv::ID CallConv,
269 SmallVectorImpl<uint64_t> *Offsets) const {
270 LLVMContext &Ctx = OrigArg.Ty->getContext();
271
272 SmallVector<EVT, 4> SplitVTs;
273 ComputeValueVTs(TLI: *TLI, DL, Ty: OrigArg.Ty, ValueVTs&: SplitVTs, FixedOffsets: Offsets, StartingOffset: 0);
274
275 if (SplitVTs.size() == 0)
276 return;
277
278 if (SplitVTs.size() == 1) {
279 // No splitting to do, but we want to replace the original type (e.g. [1 x
280 // double] -> double).
281 SplitArgs.emplace_back(Args: OrigArg.Regs[0], Args: SplitVTs[0].getTypeForEVT(Context&: Ctx),
282 Args: OrigArg.OrigArgIndex, Args: OrigArg.Flags[0],
283 Args: OrigArg.IsFixed, Args: OrigArg.OrigValue);
284 return;
285 }
286
287 // Create one ArgInfo for each virtual register in the original ArgInfo.
288 assert(OrigArg.Regs.size() == SplitVTs.size() && "Regs / types mismatch");
289
290 bool NeedsRegBlock = TLI->functionArgumentNeedsConsecutiveRegisters(
291 Ty: OrigArg.Ty, CallConv, isVarArg: false, DL);
292 for (unsigned i = 0, e = SplitVTs.size(); i < e; ++i) {
293 Type *SplitTy = SplitVTs[i].getTypeForEVT(Context&: Ctx);
294 SplitArgs.emplace_back(Args: OrigArg.Regs[i], Args&: SplitTy, Args: OrigArg.OrigArgIndex,
295 Args: OrigArg.Flags[0], Args: OrigArg.IsFixed);
296 if (NeedsRegBlock)
297 SplitArgs.back().Flags[0].setInConsecutiveRegs();
298 }
299
300 SplitArgs.back().Flags[0].setInConsecutiveRegsLast();
301}
302
303/// Pack values \p SrcRegs to cover the vector type result \p DstRegs.
304static MachineInstrBuilder
305mergeVectorRegsToResultRegs(MachineIRBuilder &B, ArrayRef<Register> DstRegs,
306 ArrayRef<Register> SrcRegs) {
307 MachineRegisterInfo &MRI = *B.getMRI();
308 LLT LLTy = MRI.getType(Reg: DstRegs[0]);
309 LLT PartLLT = MRI.getType(Reg: SrcRegs[0]);
310
311 // Deal with v3s16 split into v2s16
312 LLT LCMTy = getCoverTy(OrigTy: LLTy, TargetTy: PartLLT);
313 if (LCMTy == LLTy) {
314 // Common case where no padding is needed.
315 assert(DstRegs.size() == 1);
316 return B.buildConcatVectors(Res: DstRegs[0], Ops: SrcRegs);
317 }
318
319 // We need to create an unmerge to the result registers, which may require
320 // widening the original value.
321 Register UnmergeSrcReg;
322 if (LCMTy != PartLLT) {
323 assert(DstRegs.size() == 1);
324 return B.buildDeleteTrailingVectorElements(
325 Res: DstRegs[0], Op0: B.buildMergeLikeInstr(Res: LCMTy, Ops: SrcRegs));
326 } else {
327 // We don't need to widen anything if we're extracting a scalar which was
328 // promoted to a vector e.g. s8 -> v4s8 -> s8
329 assert(SrcRegs.size() == 1);
330 UnmergeSrcReg = SrcRegs[0];
331 }
332
333 int NumDst = LCMTy.getSizeInBits() / LLTy.getSizeInBits();
334
335 SmallVector<Register, 8> PadDstRegs(NumDst);
336 std::copy(first: DstRegs.begin(), last: DstRegs.end(), result: PadDstRegs.begin());
337
338 // Create the excess dead defs for the unmerge.
339 for (int I = DstRegs.size(); I != NumDst; ++I)
340 PadDstRegs[I] = MRI.createGenericVirtualRegister(Ty: LLTy);
341
342 if (PadDstRegs.size() == 1)
343 return B.buildDeleteTrailingVectorElements(Res: DstRegs[0], Op0: UnmergeSrcReg);
344 return B.buildUnmerge(Res: PadDstRegs, Op: UnmergeSrcReg);
345}
346
347/// Create a sequence of instructions to combine pieces split into register
348/// typed values to the original IR value. \p OrigRegs contains the destination
349/// value registers of type \p LLTy, and \p Regs contains the legalized pieces
350/// with type \p PartLLT. This is used for incoming values (physregs to vregs).
351static void buildCopyFromRegs(MachineIRBuilder &B, ArrayRef<Register> OrigRegs,
352 ArrayRef<Register> Regs, LLT LLTy, LLT PartLLT,
353 const ISD::ArgFlagsTy Flags) {
354 MachineRegisterInfo &MRI = *B.getMRI();
355
356 if (PartLLT == LLTy) {
357 // We should have avoided introducing a new virtual register, and just
358 // directly assigned here.
359 assert(OrigRegs[0] == Regs[0]);
360 return;
361 }
362
363 if (PartLLT.getSizeInBits() == LLTy.getSizeInBits() && OrigRegs.size() == 1 &&
364 Regs.size() == 1) {
365 B.buildBitcast(Dst: OrigRegs[0], Src: Regs[0]);
366 return;
367 }
368
369 // A vector PartLLT needs extending to LLTy's element size.
370 // E.g. <2 x s64> = G_SEXT <2 x s32>.
371 if (PartLLT.isVector() == LLTy.isVector() &&
372 PartLLT.getScalarSizeInBits() > LLTy.getScalarSizeInBits() &&
373 (!PartLLT.isVector() ||
374 PartLLT.getElementCount() == LLTy.getElementCount()) &&
375 OrigRegs.size() == 1 && Regs.size() == 1) {
376 Register SrcReg = Regs[0];
377
378 LLT LocTy = MRI.getType(Reg: SrcReg);
379
380 if (Flags.isSExt()) {
381 SrcReg = B.buildAssertSExt(Res: LocTy, Op: SrcReg, Size: LLTy.getScalarSizeInBits())
382 .getReg(Idx: 0);
383 } else if (Flags.isZExt()) {
384 SrcReg = B.buildAssertZExt(Res: LocTy, Op: SrcReg, Size: LLTy.getScalarSizeInBits())
385 .getReg(Idx: 0);
386 }
387
388 // Sometimes pointers are passed zero extended.
389 LLT OrigTy = MRI.getType(Reg: OrigRegs[0]);
390 if (OrigTy.isPointer()) {
391 LLT IntPtrTy = LLT::scalar(SizeInBits: OrigTy.getSizeInBits());
392 B.buildIntToPtr(Dst: OrigRegs[0], Src: B.buildTrunc(Res: IntPtrTy, Op: SrcReg));
393 return;
394 }
395
396 B.buildTrunc(Res: OrigRegs[0], Op: SrcReg);
397 return;
398 }
399
400 if (!LLTy.isVector() && !PartLLT.isVector()) {
401 assert(OrigRegs.size() == 1);
402 LLT OrigTy = MRI.getType(Reg: OrigRegs[0]);
403
404 unsigned SrcSize = PartLLT.getSizeInBits().getFixedValue() * Regs.size();
405 if (SrcSize == OrigTy.getSizeInBits())
406 B.buildMergeValues(Res: OrigRegs[0], Ops: Regs);
407 else {
408 auto Widened = B.buildMergeLikeInstr(Res: LLT::scalar(SizeInBits: SrcSize), Ops: Regs);
409 B.buildTrunc(Res: OrigRegs[0], Op: Widened);
410 }
411
412 return;
413 }
414
415 if (PartLLT.isVector()) {
416 assert(OrigRegs.size() == 1);
417 SmallVector<Register> CastRegs(Regs.begin(), Regs.end());
418
419 // If PartLLT is a mismatched vector in both number of elements and element
420 // size, e.g. PartLLT == v2s64 and LLTy is v3s32, then first coerce it to
421 // have the same elt type, i.e. v4s32.
422 // TODO: Extend this coersion to element multiples other than just 2.
423 if (TypeSize::isKnownGT(LHS: PartLLT.getSizeInBits(), RHS: LLTy.getSizeInBits()) &&
424 PartLLT.getScalarSizeInBits() == LLTy.getScalarSizeInBits() * 2 &&
425 Regs.size() == 1) {
426 LLT NewTy = PartLLT.changeElementType(NewEltTy: LLTy.getElementType())
427 .changeElementCount(EC: PartLLT.getElementCount() * 2);
428 CastRegs[0] = B.buildBitcast(Dst: NewTy, Src: Regs[0]).getReg(Idx: 0);
429 PartLLT = NewTy;
430 }
431
432 if (LLTy.getScalarType() == PartLLT.getElementType()) {
433 mergeVectorRegsToResultRegs(B, DstRegs: OrigRegs, SrcRegs: CastRegs);
434 } else {
435 unsigned I = 0;
436 LLT GCDTy = getGCDType(OrigTy: LLTy, TargetTy: PartLLT);
437
438 // We are both splitting a vector, and bitcasting its element types. Cast
439 // the source pieces into the appropriate number of pieces with the result
440 // element type.
441 for (Register SrcReg : CastRegs)
442 CastRegs[I++] = B.buildBitcast(Dst: GCDTy, Src: SrcReg).getReg(Idx: 0);
443 mergeVectorRegsToResultRegs(B, DstRegs: OrigRegs, SrcRegs: CastRegs);
444 }
445
446 return;
447 }
448
449 assert(LLTy.isVector() && !PartLLT.isVector());
450
451 LLT DstEltTy = LLTy.getElementType();
452
453 // Pointer information was discarded. We'll need to coerce some register types
454 // to avoid violating type constraints.
455 LLT RealDstEltTy = MRI.getType(Reg: OrigRegs[0]).getElementType();
456
457 assert(DstEltTy.getSizeInBits() == RealDstEltTy.getSizeInBits());
458
459 if (DstEltTy == PartLLT) {
460 // Vector was trivially scalarized.
461
462 if (RealDstEltTy.isPointer()) {
463 for (Register Reg : Regs)
464 MRI.setType(VReg: Reg, Ty: RealDstEltTy);
465 }
466
467 B.buildBuildVector(Res: OrigRegs[0], Ops: Regs);
468 } else if (DstEltTy.getSizeInBits() > PartLLT.getSizeInBits()) {
469 // Deal with vector with 64-bit elements decomposed to 32-bit
470 // registers. Need to create intermediate 64-bit elements.
471 SmallVector<Register, 8> EltMerges;
472 int PartsPerElt = DstEltTy.getSizeInBits() / PartLLT.getSizeInBits();
473
474 assert(DstEltTy.getSizeInBits() % PartLLT.getSizeInBits() == 0);
475
476 for (int I = 0, NumElts = LLTy.getNumElements(); I != NumElts; ++I) {
477 auto Merge =
478 B.buildMergeLikeInstr(Res: RealDstEltTy, Ops: Regs.take_front(N: PartsPerElt));
479 // Fix the type in case this is really a vector of pointers.
480 MRI.setType(VReg: Merge.getReg(Idx: 0), Ty: RealDstEltTy);
481 EltMerges.push_back(Elt: Merge.getReg(Idx: 0));
482 Regs = Regs.drop_front(N: PartsPerElt);
483 }
484
485 B.buildBuildVector(Res: OrigRegs[0], Ops: EltMerges);
486 } else {
487 // Vector was split, and elements promoted to a wider type.
488 // FIXME: Should handle floating point promotions.
489 unsigned NumElts = LLTy.getNumElements();
490 LLT BVType = LLT::fixed_vector(NumElements: NumElts, ScalarTy: PartLLT);
491
492 Register BuildVec;
493 if (NumElts == Regs.size())
494 BuildVec = B.buildBuildVector(Res: BVType, Ops: Regs).getReg(Idx: 0);
495 else {
496 // Vector elements are packed in the inputs.
497 // e.g. we have a <4 x s16> but 2 x s32 in regs.
498 assert(NumElts > Regs.size());
499 LLT SrcEltTy = MRI.getType(Reg: Regs[0]);
500
501 LLT OriginalEltTy = MRI.getType(Reg: OrigRegs[0]).getElementType();
502
503 // Input registers contain packed elements.
504 // Determine how many elements per reg.
505 assert((SrcEltTy.getSizeInBits() % OriginalEltTy.getSizeInBits()) == 0);
506 unsigned EltPerReg =
507 (SrcEltTy.getSizeInBits() / OriginalEltTy.getSizeInBits());
508
509 SmallVector<Register, 0> BVRegs;
510 BVRegs.reserve(N: Regs.size() * EltPerReg);
511 for (Register R : Regs) {
512 auto Unmerge = B.buildUnmerge(Res: OriginalEltTy, Op: R);
513 for (unsigned K = 0; K < EltPerReg; ++K)
514 BVRegs.push_back(Elt: B.buildAnyExt(Res: PartLLT, Op: Unmerge.getReg(Idx: K)).getReg(Idx: 0));
515 }
516
517 // We may have some more elements in BVRegs, e.g. if we have 2 s32 pieces
518 // for a <3 x s16> vector. We should have less than EltPerReg extra items.
519 if (BVRegs.size() > NumElts) {
520 assert((BVRegs.size() - NumElts) < EltPerReg);
521 BVRegs.truncate(N: NumElts);
522 }
523 BuildVec = B.buildBuildVector(Res: BVType, Ops: BVRegs).getReg(Idx: 0);
524 }
525 B.buildTrunc(Res: OrigRegs[0], Op: BuildVec);
526 }
527}
528
529/// Create a sequence of instructions to expand the value in \p SrcReg (of type
530/// \p SrcTy) to the types in \p DstRegs (of type \p PartTy). \p ExtendOp should
531/// contain the type of scalar value extension if necessary.
532///
533/// This is used for outgoing values (vregs to physregs)
534static void buildCopyToRegs(MachineIRBuilder &B, ArrayRef<Register> DstRegs,
535 Register SrcReg, LLT SrcTy, LLT PartTy,
536 unsigned ExtendOp = TargetOpcode::G_ANYEXT) {
537 // We could just insert a regular copy, but this is unreachable at the moment.
538 assert(SrcTy != PartTy && "identical part types shouldn't reach here");
539
540 const TypeSize PartSize = PartTy.getSizeInBits();
541
542 if (PartTy.isVector() == SrcTy.isVector() &&
543 PartTy.getScalarSizeInBits() > SrcTy.getScalarSizeInBits()) {
544 assert(DstRegs.size() == 1);
545 B.buildInstr(Opc: ExtendOp, DstOps: {DstRegs[0]}, SrcOps: {SrcReg});
546 return;
547 }
548
549 if (SrcTy.isVector() && !PartTy.isVector() &&
550 TypeSize::isKnownGT(LHS: PartSize, RHS: SrcTy.getElementType().getSizeInBits())) {
551 // Vector was scalarized, and the elements extended.
552 auto UnmergeToEltTy = B.buildUnmerge(Res: SrcTy.getElementType(), Op: SrcReg);
553 for (int i = 0, e = DstRegs.size(); i != e; ++i)
554 B.buildAnyExt(Res: DstRegs[i], Op: UnmergeToEltTy.getReg(Idx: i));
555 return;
556 }
557
558 if (SrcTy.isVector() && PartTy.isVector() &&
559 PartTy.getSizeInBits() == SrcTy.getSizeInBits() &&
560 ElementCount::isKnownLT(LHS: SrcTy.getElementCount(),
561 RHS: PartTy.getElementCount())) {
562 // A coercion like: v2f32 -> v4f32 or nxv2f32 -> nxv4f32
563 Register DstReg = DstRegs.front();
564 B.buildPadVectorWithUndefElements(Res: DstReg, Op0: SrcReg);
565 return;
566 }
567
568 LLT GCDTy = getGCDType(OrigTy: SrcTy, TargetTy: PartTy);
569 if (GCDTy == PartTy) {
570 // If this already evenly divisible, we can create a simple unmerge.
571 B.buildUnmerge(Res: DstRegs, Op: SrcReg);
572 return;
573 }
574
575 MachineRegisterInfo &MRI = *B.getMRI();
576 LLT DstTy = MRI.getType(Reg: DstRegs[0]);
577 LLT LCMTy = getCoverTy(OrigTy: SrcTy, TargetTy: PartTy);
578
579 if (PartTy.isVector() && LCMTy == PartTy) {
580 assert(DstRegs.size() == 1);
581 B.buildPadVectorWithUndefElements(Res: DstRegs[0], Op0: SrcReg);
582 return;
583 }
584
585 const unsigned DstSize = DstTy.getSizeInBits();
586 const unsigned SrcSize = SrcTy.getSizeInBits();
587 unsigned CoveringSize = LCMTy.getSizeInBits();
588
589 Register UnmergeSrc = SrcReg;
590
591 if (!LCMTy.isVector() && CoveringSize != SrcSize) {
592 // For scalars, it's common to be able to use a simple extension.
593 if (SrcTy.isScalar() && DstTy.isScalar()) {
594 CoveringSize = alignTo(Value: SrcSize, Align: DstSize);
595 LLT CoverTy = LLT::scalar(SizeInBits: CoveringSize);
596 UnmergeSrc = B.buildInstr(Opc: ExtendOp, DstOps: {CoverTy}, SrcOps: {SrcReg}).getReg(Idx: 0);
597 } else {
598 // Widen to the common type.
599 // FIXME: This should respect the extend type
600 Register Undef = B.buildUndef(Res: SrcTy).getReg(Idx: 0);
601 SmallVector<Register, 8> MergeParts(1, SrcReg);
602 for (unsigned Size = SrcSize; Size != CoveringSize; Size += SrcSize)
603 MergeParts.push_back(Elt: Undef);
604 UnmergeSrc = B.buildMergeLikeInstr(Res: LCMTy, Ops: MergeParts).getReg(Idx: 0);
605 }
606 }
607
608 if (LCMTy.isVector() && CoveringSize != SrcSize)
609 UnmergeSrc = B.buildPadVectorWithUndefElements(Res: LCMTy, Op0: SrcReg).getReg(Idx: 0);
610
611 B.buildUnmerge(Res: DstRegs, Op: UnmergeSrc);
612}
613
614bool CallLowering::determineAndHandleAssignments(
615 ValueHandler &Handler, ValueAssigner &Assigner,
616 SmallVectorImpl<ArgInfo> &Args, MachineIRBuilder &MIRBuilder,
617 CallingConv::ID CallConv, bool IsVarArg,
618 ArrayRef<Register> ThisReturnRegs) const {
619 MachineFunction &MF = MIRBuilder.getMF();
620 const Function &F = MF.getFunction();
621 SmallVector<CCValAssign, 16> ArgLocs;
622
623 CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, F.getContext());
624 if (!determineAssignments(Assigner, Args, CCInfo))
625 return false;
626
627 return handleAssignments(Handler, Args, CCState&: CCInfo, ArgLocs, MIRBuilder,
628 ThisReturnRegs);
629}
630
631static unsigned extendOpFromFlags(llvm::ISD::ArgFlagsTy Flags) {
632 if (Flags.isSExt())
633 return TargetOpcode::G_SEXT;
634 if (Flags.isZExt())
635 return TargetOpcode::G_ZEXT;
636 return TargetOpcode::G_ANYEXT;
637}
638
639bool CallLowering::determineAssignments(ValueAssigner &Assigner,
640 SmallVectorImpl<ArgInfo> &Args,
641 CCState &CCInfo) const {
642 LLVMContext &Ctx = CCInfo.getContext();
643 const CallingConv::ID CallConv = CCInfo.getCallingConv();
644
645 unsigned NumArgs = Args.size();
646 for (unsigned i = 0; i != NumArgs; ++i) {
647 EVT CurVT = EVT::getEVT(Ty: Args[i].Ty);
648
649 MVT NewVT = TLI->getRegisterTypeForCallingConv(Context&: Ctx, CC: CallConv, VT: CurVT);
650
651 // If we need to split the type over multiple regs, check it's a scenario
652 // we currently support.
653 unsigned NumParts =
654 TLI->getNumRegistersForCallingConv(Context&: Ctx, CC: CallConv, VT: CurVT);
655
656 if (NumParts == 1) {
657 // Try to use the register type if we couldn't assign the VT.
658 if (Assigner.assignArg(ValNo: i, OrigVT: CurVT, ValVT: NewVT, LocVT: NewVT, LocInfo: CCValAssign::Full, Info: Args[i],
659 Flags: Args[i].Flags[0], State&: CCInfo))
660 return false;
661 continue;
662 }
663
664 // For incoming arguments (physregs to vregs), we could have values in
665 // physregs (or memlocs) which we want to extract and copy to vregs.
666 // During this, we might have to deal with the LLT being split across
667 // multiple regs, so we have to record this information for later.
668 //
669 // If we have outgoing args, then we have the opposite case. We have a
670 // vreg with an LLT which we want to assign to a physical location, and
671 // we might have to record that the value has to be split later.
672
673 // We're handling an incoming arg which is split over multiple regs.
674 // E.g. passing an s128 on AArch64.
675 ISD::ArgFlagsTy OrigFlags = Args[i].Flags[0];
676 Args[i].Flags.clear();
677
678 for (unsigned Part = 0; Part < NumParts; ++Part) {
679 ISD::ArgFlagsTy Flags = OrigFlags;
680 if (Part == 0) {
681 Flags.setSplit();
682 } else {
683 Flags.setOrigAlign(Align(1));
684 if (Part == NumParts - 1)
685 Flags.setSplitEnd();
686 }
687
688 Args[i].Flags.push_back(Elt: Flags);
689 if (Assigner.assignArg(ValNo: i, OrigVT: CurVT, ValVT: NewVT, LocVT: NewVT, LocInfo: CCValAssign::Full, Info: Args[i],
690 Flags: Args[i].Flags[Part], State&: CCInfo)) {
691 // Still couldn't assign this smaller part type for some reason.
692 return false;
693 }
694 }
695 }
696
697 return true;
698}
699
700bool CallLowering::handleAssignments(ValueHandler &Handler,
701 SmallVectorImpl<ArgInfo> &Args,
702 CCState &CCInfo,
703 SmallVectorImpl<CCValAssign> &ArgLocs,
704 MachineIRBuilder &MIRBuilder,
705 ArrayRef<Register> ThisReturnRegs) const {
706 MachineFunction &MF = MIRBuilder.getMF();
707 MachineRegisterInfo &MRI = MF.getRegInfo();
708 const Function &F = MF.getFunction();
709 const DataLayout &DL = F.getParent()->getDataLayout();
710
711 const unsigned NumArgs = Args.size();
712
713 // Stores thunks for outgoing register assignments. This is used so we delay
714 // generating register copies until mem loc assignments are done. We do this
715 // so that if the target is using the delayed stack protector feature, we can
716 // find the split point of the block accurately. E.g. if we have:
717 // G_STORE %val, %memloc
718 // $x0 = COPY %foo
719 // $x1 = COPY %bar
720 // CALL func
721 // ... then the split point for the block will correctly be at, and including,
722 // the copy to $x0. If instead the G_STORE instruction immediately precedes
723 // the CALL, then we'd prematurely choose the CALL as the split point, thus
724 // generating a split block with a CALL that uses undefined physregs.
725 SmallVector<std::function<void()>> DelayedOutgoingRegAssignments;
726
727 for (unsigned i = 0, j = 0; i != NumArgs; ++i, ++j) {
728 assert(j < ArgLocs.size() && "Skipped too many arg locs");
729 CCValAssign &VA = ArgLocs[j];
730 assert(VA.getValNo() == i && "Location doesn't correspond to current arg");
731
732 if (VA.needsCustom()) {
733 std::function<void()> Thunk;
734 unsigned NumArgRegs = Handler.assignCustomValue(
735 Arg&: Args[i], VAs: ArrayRef(ArgLocs).slice(N: j), Thunk: &Thunk);
736 if (Thunk)
737 DelayedOutgoingRegAssignments.emplace_back(Args&: Thunk);
738 if (!NumArgRegs)
739 return false;
740 j += (NumArgRegs - 1);
741 continue;
742 }
743
744 const MVT ValVT = VA.getValVT();
745 const MVT LocVT = VA.getLocVT();
746
747 const LLT LocTy(LocVT);
748 const LLT ValTy(ValVT);
749 const LLT NewLLT = Handler.isIncomingArgumentHandler() ? LocTy : ValTy;
750 const EVT OrigVT = EVT::getEVT(Ty: Args[i].Ty);
751 const LLT OrigTy = getLLTForType(Ty&: *Args[i].Ty, DL);
752
753 // Expected to be multiple regs for a single incoming arg.
754 // There should be Regs.size() ArgLocs per argument.
755 // This should be the same as getNumRegistersForCallingConv
756 const unsigned NumParts = Args[i].Flags.size();
757
758 // Now split the registers into the assigned types.
759 Args[i].OrigRegs.assign(in_start: Args[i].Regs.begin(), in_end: Args[i].Regs.end());
760
761 if (NumParts != 1 || NewLLT != OrigTy) {
762 // If we can't directly assign the register, we need one or more
763 // intermediate values.
764 Args[i].Regs.resize(N: NumParts);
765
766 // For each split register, create and assign a vreg that will store
767 // the incoming component of the larger value. These will later be
768 // merged to form the final vreg.
769 for (unsigned Part = 0; Part < NumParts; ++Part)
770 Args[i].Regs[Part] = MRI.createGenericVirtualRegister(Ty: NewLLT);
771 }
772
773 assert((j + (NumParts - 1)) < ArgLocs.size() &&
774 "Too many regs for number of args");
775
776 // Coerce into outgoing value types before register assignment.
777 if (!Handler.isIncomingArgumentHandler() && OrigTy != ValTy) {
778 assert(Args[i].OrigRegs.size() == 1);
779 buildCopyToRegs(B&: MIRBuilder, DstRegs: Args[i].Regs, SrcReg: Args[i].OrigRegs[0], SrcTy: OrigTy,
780 PartTy: ValTy, ExtendOp: extendOpFromFlags(Flags: Args[i].Flags[0]));
781 }
782
783 bool BigEndianPartOrdering = TLI->hasBigEndianPartOrdering(VT: OrigVT, DL);
784 for (unsigned Part = 0; Part < NumParts; ++Part) {
785 Register ArgReg = Args[i].Regs[Part];
786 // There should be Regs.size() ArgLocs per argument.
787 unsigned Idx = BigEndianPartOrdering ? NumParts - 1 - Part : Part;
788 CCValAssign &VA = ArgLocs[j + Idx];
789 const ISD::ArgFlagsTy Flags = Args[i].Flags[Part];
790
791 if (VA.isMemLoc() && !Flags.isByVal()) {
792 // Individual pieces may have been spilled to the stack and others
793 // passed in registers.
794
795 // TODO: The memory size may be larger than the value we need to
796 // store. We may need to adjust the offset for big endian targets.
797 LLT MemTy = Handler.getStackValueStoreType(DL, VA, Flags);
798
799 MachinePointerInfo MPO;
800 Register StackAddr = Handler.getStackAddress(
801 MemSize: MemTy.getSizeInBytes(), Offset: VA.getLocMemOffset(), MPO, Flags);
802
803 Handler.assignValueToAddress(Arg: Args[i], ValRegIndex: Part, Addr: StackAddr, MemTy, MPO, VA);
804 continue;
805 }
806
807 if (VA.isMemLoc() && Flags.isByVal()) {
808 assert(Args[i].Regs.size() == 1 &&
809 "didn't expect split byval pointer");
810
811 if (Handler.isIncomingArgumentHandler()) {
812 // We just need to copy the frame index value to the pointer.
813 MachinePointerInfo MPO;
814 Register StackAddr = Handler.getStackAddress(
815 MemSize: Flags.getByValSize(), Offset: VA.getLocMemOffset(), MPO, Flags);
816 MIRBuilder.buildCopy(Res: Args[i].Regs[0], Op: StackAddr);
817 } else {
818 // For outgoing byval arguments, insert the implicit copy byval
819 // implies, such that writes in the callee do not modify the caller's
820 // value.
821 uint64_t MemSize = Flags.getByValSize();
822 int64_t Offset = VA.getLocMemOffset();
823
824 MachinePointerInfo DstMPO;
825 Register StackAddr =
826 Handler.getStackAddress(MemSize, Offset, MPO&: DstMPO, Flags);
827
828 MachinePointerInfo SrcMPO(Args[i].OrigValue);
829 if (!Args[i].OrigValue) {
830 // We still need to accurately track the stack address space if we
831 // don't know the underlying value.
832 const LLT PtrTy = MRI.getType(Reg: StackAddr);
833 SrcMPO = MachinePointerInfo(PtrTy.getAddressSpace());
834 }
835
836 Align DstAlign = std::max(a: Flags.getNonZeroByValAlign(),
837 b: inferAlignFromPtrInfo(MF, MPO: DstMPO));
838
839 Align SrcAlign = std::max(a: Flags.getNonZeroByValAlign(),
840 b: inferAlignFromPtrInfo(MF, MPO: SrcMPO));
841
842 Handler.copyArgumentMemory(Arg: Args[i], DstPtr: StackAddr, SrcPtr: Args[i].Regs[0],
843 DstPtrInfo: DstMPO, DstAlign, SrcPtrInfo: SrcMPO, SrcAlign,
844 MemSize, VA);
845 }
846 continue;
847 }
848
849 assert(!VA.needsCustom() && "custom loc should have been handled already");
850
851 if (i == 0 && !ThisReturnRegs.empty() &&
852 Handler.isIncomingArgumentHandler() &&
853 isTypeIsValidForThisReturn(Ty: ValVT)) {
854 Handler.assignValueToReg(ValVReg: ArgReg, PhysReg: ThisReturnRegs[Part], VA);
855 continue;
856 }
857
858 if (Handler.isIncomingArgumentHandler())
859 Handler.assignValueToReg(ValVReg: ArgReg, PhysReg: VA.getLocReg(), VA);
860 else {
861 DelayedOutgoingRegAssignments.emplace_back(Args: [=, &Handler]() {
862 Handler.assignValueToReg(ValVReg: ArgReg, PhysReg: VA.getLocReg(), VA);
863 });
864 }
865 }
866
867 // Now that all pieces have been assigned, re-pack the register typed values
868 // into the original value typed registers.
869 if (Handler.isIncomingArgumentHandler() && OrigVT != LocVT) {
870 // Merge the split registers into the expected larger result vregs of
871 // the original call.
872 buildCopyFromRegs(B&: MIRBuilder, OrigRegs: Args[i].OrigRegs, Regs: Args[i].Regs, LLTy: OrigTy,
873 PartLLT: LocTy, Flags: Args[i].Flags[0]);
874 }
875
876 j += NumParts - 1;
877 }
878 for (auto &Fn : DelayedOutgoingRegAssignments)
879 Fn();
880
881 return true;
882}
883
884void CallLowering::insertSRetLoads(MachineIRBuilder &MIRBuilder, Type *RetTy,
885 ArrayRef<Register> VRegs, Register DemoteReg,
886 int FI) const {
887 MachineFunction &MF = MIRBuilder.getMF();
888 MachineRegisterInfo &MRI = MF.getRegInfo();
889 const DataLayout &DL = MF.getDataLayout();
890
891 SmallVector<EVT, 4> SplitVTs;
892 SmallVector<uint64_t, 4> Offsets;
893 ComputeValueVTs(TLI: *TLI, DL, Ty: RetTy, ValueVTs&: SplitVTs, FixedOffsets: &Offsets, StartingOffset: 0);
894
895 assert(VRegs.size() == SplitVTs.size());
896
897 unsigned NumValues = SplitVTs.size();
898 Align BaseAlign = DL.getPrefTypeAlign(Ty: RetTy);
899 Type *RetPtrTy =
900 PointerType::get(C&: RetTy->getContext(), AddressSpace: DL.getAllocaAddrSpace());
901 LLT OffsetLLTy = getLLTForType(Ty&: *DL.getIndexType(PtrTy: RetPtrTy), DL);
902
903 MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(MF, FI);
904
905 for (unsigned I = 0; I < NumValues; ++I) {
906 Register Addr;
907 MIRBuilder.materializePtrAdd(Res&: Addr, Op0: DemoteReg, ValueTy: OffsetLLTy, Value: Offsets[I]);
908 auto *MMO = MF.getMachineMemOperand(PtrInfo, f: MachineMemOperand::MOLoad,
909 MemTy: MRI.getType(Reg: VRegs[I]),
910 base_alignment: commonAlignment(A: BaseAlign, Offset: Offsets[I]));
911 MIRBuilder.buildLoad(Res: VRegs[I], Addr, MMO&: *MMO);
912 }
913}
914
915void CallLowering::insertSRetStores(MachineIRBuilder &MIRBuilder, Type *RetTy,
916 ArrayRef<Register> VRegs,
917 Register DemoteReg) const {
918 MachineFunction &MF = MIRBuilder.getMF();
919 MachineRegisterInfo &MRI = MF.getRegInfo();
920 const DataLayout &DL = MF.getDataLayout();
921
922 SmallVector<EVT, 4> SplitVTs;
923 SmallVector<uint64_t, 4> Offsets;
924 ComputeValueVTs(TLI: *TLI, DL, Ty: RetTy, ValueVTs&: SplitVTs, FixedOffsets: &Offsets, StartingOffset: 0);
925
926 assert(VRegs.size() == SplitVTs.size());
927
928 unsigned NumValues = SplitVTs.size();
929 Align BaseAlign = DL.getPrefTypeAlign(Ty: RetTy);
930 unsigned AS = DL.getAllocaAddrSpace();
931 LLT OffsetLLTy = getLLTForType(Ty&: *DL.getIndexType(PtrTy: RetTy->getPointerTo(AddrSpace: AS)), DL);
932
933 MachinePointerInfo PtrInfo(AS);
934
935 for (unsigned I = 0; I < NumValues; ++I) {
936 Register Addr;
937 MIRBuilder.materializePtrAdd(Res&: Addr, Op0: DemoteReg, ValueTy: OffsetLLTy, Value: Offsets[I]);
938 auto *MMO = MF.getMachineMemOperand(PtrInfo, f: MachineMemOperand::MOStore,
939 MemTy: MRI.getType(Reg: VRegs[I]),
940 base_alignment: commonAlignment(A: BaseAlign, Offset: Offsets[I]));
941 MIRBuilder.buildStore(Val: VRegs[I], Addr, MMO&: *MMO);
942 }
943}
944
945void CallLowering::insertSRetIncomingArgument(
946 const Function &F, SmallVectorImpl<ArgInfo> &SplitArgs, Register &DemoteReg,
947 MachineRegisterInfo &MRI, const DataLayout &DL) const {
948 unsigned AS = DL.getAllocaAddrSpace();
949 DemoteReg = MRI.createGenericVirtualRegister(
950 Ty: LLT::pointer(AddressSpace: AS, SizeInBits: DL.getPointerSizeInBits(AS)));
951
952 Type *PtrTy = PointerType::get(ElementType: F.getReturnType(), AddressSpace: AS);
953
954 SmallVector<EVT, 1> ValueVTs;
955 ComputeValueVTs(TLI: *TLI, DL, Ty: PtrTy, ValueVTs);
956
957 // NOTE: Assume that a pointer won't get split into more than one VT.
958 assert(ValueVTs.size() == 1);
959
960 ArgInfo DemoteArg(DemoteReg, ValueVTs[0].getTypeForEVT(Context&: PtrTy->getContext()),
961 ArgInfo::NoArgIndex);
962 setArgFlags(Arg&: DemoteArg, OpIdx: AttributeList::ReturnIndex, DL, FuncInfo: F);
963 DemoteArg.Flags[0].setSRet();
964 SplitArgs.insert(I: SplitArgs.begin(), Elt: DemoteArg);
965}
966
967void CallLowering::insertSRetOutgoingArgument(MachineIRBuilder &MIRBuilder,
968 const CallBase &CB,
969 CallLoweringInfo &Info) const {
970 const DataLayout &DL = MIRBuilder.getDataLayout();
971 Type *RetTy = CB.getType();
972 unsigned AS = DL.getAllocaAddrSpace();
973 LLT FramePtrTy = LLT::pointer(AddressSpace: AS, SizeInBits: DL.getPointerSizeInBits(AS));
974
975 int FI = MIRBuilder.getMF().getFrameInfo().CreateStackObject(
976 Size: DL.getTypeAllocSize(Ty: RetTy), Alignment: DL.getPrefTypeAlign(Ty: RetTy), isSpillSlot: false);
977
978 Register DemoteReg = MIRBuilder.buildFrameIndex(Res: FramePtrTy, Idx: FI).getReg(Idx: 0);
979 ArgInfo DemoteArg(DemoteReg, PointerType::get(ElementType: RetTy, AddressSpace: AS),
980 ArgInfo::NoArgIndex);
981 setArgFlags(Arg&: DemoteArg, OpIdx: AttributeList::ReturnIndex, DL, FuncInfo: CB);
982 DemoteArg.Flags[0].setSRet();
983
984 Info.OrigArgs.insert(I: Info.OrigArgs.begin(), Elt: DemoteArg);
985 Info.DemoteStackIndex = FI;
986 Info.DemoteRegister = DemoteReg;
987}
988
989bool CallLowering::checkReturn(CCState &CCInfo,
990 SmallVectorImpl<BaseArgInfo> &Outs,
991 CCAssignFn *Fn) const {
992 for (unsigned I = 0, E = Outs.size(); I < E; ++I) {
993 MVT VT = MVT::getVT(Ty: Outs[I].Ty);
994 if (Fn(I, VT, VT, CCValAssign::Full, Outs[I].Flags[0], CCInfo))
995 return false;
996 }
997 return true;
998}
999
1000void CallLowering::getReturnInfo(CallingConv::ID CallConv, Type *RetTy,
1001 AttributeList Attrs,
1002 SmallVectorImpl<BaseArgInfo> &Outs,
1003 const DataLayout &DL) const {
1004 LLVMContext &Context = RetTy->getContext();
1005 ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
1006
1007 SmallVector<EVT, 4> SplitVTs;
1008 ComputeValueVTs(TLI: *TLI, DL, Ty: RetTy, ValueVTs&: SplitVTs);
1009 addArgFlagsFromAttributes(Flags, Attrs, OpIdx: AttributeList::ReturnIndex);
1010
1011 for (EVT VT : SplitVTs) {
1012 unsigned NumParts =
1013 TLI->getNumRegistersForCallingConv(Context, CC: CallConv, VT);
1014 MVT RegVT = TLI->getRegisterTypeForCallingConv(Context, CC: CallConv, VT);
1015 Type *PartTy = EVT(RegVT).getTypeForEVT(Context);
1016
1017 for (unsigned I = 0; I < NumParts; ++I) {
1018 Outs.emplace_back(Args&: PartTy, Args&: Flags);
1019 }
1020 }
1021}
1022
1023bool CallLowering::checkReturnTypeForCallConv(MachineFunction &MF) const {
1024 const auto &F = MF.getFunction();
1025 Type *ReturnType = F.getReturnType();
1026 CallingConv::ID CallConv = F.getCallingConv();
1027
1028 SmallVector<BaseArgInfo, 4> SplitArgs;
1029 getReturnInfo(CallConv, RetTy: ReturnType, Attrs: F.getAttributes(), Outs&: SplitArgs,
1030 DL: MF.getDataLayout());
1031 return canLowerReturn(MF, CallConv, Outs&: SplitArgs, IsVarArg: F.isVarArg());
1032}
1033
1034bool CallLowering::parametersInCSRMatch(
1035 const MachineRegisterInfo &MRI, const uint32_t *CallerPreservedMask,
1036 const SmallVectorImpl<CCValAssign> &OutLocs,
1037 const SmallVectorImpl<ArgInfo> &OutArgs) const {
1038 for (unsigned i = 0; i < OutLocs.size(); ++i) {
1039 const auto &ArgLoc = OutLocs[i];
1040 // If it's not a register, it's fine.
1041 if (!ArgLoc.isRegLoc())
1042 continue;
1043
1044 MCRegister PhysReg = ArgLoc.getLocReg();
1045
1046 // Only look at callee-saved registers.
1047 if (MachineOperand::clobbersPhysReg(RegMask: CallerPreservedMask, PhysReg))
1048 continue;
1049
1050 LLVM_DEBUG(
1051 dbgs()
1052 << "... Call has an argument passed in a callee-saved register.\n");
1053
1054 // Check if it was copied from.
1055 const ArgInfo &OutInfo = OutArgs[i];
1056
1057 if (OutInfo.Regs.size() > 1) {
1058 LLVM_DEBUG(
1059 dbgs() << "... Cannot handle arguments in multiple registers.\n");
1060 return false;
1061 }
1062
1063 // Check if we copy the register, walking through copies from virtual
1064 // registers. Note that getDefIgnoringCopies does not ignore copies from
1065 // physical registers.
1066 MachineInstr *RegDef = getDefIgnoringCopies(Reg: OutInfo.Regs[0], MRI);
1067 if (!RegDef || RegDef->getOpcode() != TargetOpcode::COPY) {
1068 LLVM_DEBUG(
1069 dbgs()
1070 << "... Parameter was not copied into a VReg, cannot tail call.\n");
1071 return false;
1072 }
1073
1074 // Got a copy. Verify that it's the same as the register we want.
1075 Register CopyRHS = RegDef->getOperand(i: 1).getReg();
1076 if (CopyRHS != PhysReg) {
1077 LLVM_DEBUG(dbgs() << "... Callee-saved register was not copied into "
1078 "VReg, cannot tail call.\n");
1079 return false;
1080 }
1081 }
1082
1083 return true;
1084}
1085
1086bool CallLowering::resultsCompatible(CallLoweringInfo &Info,
1087 MachineFunction &MF,
1088 SmallVectorImpl<ArgInfo> &InArgs,
1089 ValueAssigner &CalleeAssigner,
1090 ValueAssigner &CallerAssigner) const {
1091 const Function &F = MF.getFunction();
1092 CallingConv::ID CalleeCC = Info.CallConv;
1093 CallingConv::ID CallerCC = F.getCallingConv();
1094
1095 if (CallerCC == CalleeCC)
1096 return true;
1097
1098 SmallVector<CCValAssign, 16> ArgLocs1;
1099 CCState CCInfo1(CalleeCC, Info.IsVarArg, MF, ArgLocs1, F.getContext());
1100 if (!determineAssignments(Assigner&: CalleeAssigner, Args&: InArgs, CCInfo&: CCInfo1))
1101 return false;
1102
1103 SmallVector<CCValAssign, 16> ArgLocs2;
1104 CCState CCInfo2(CallerCC, F.isVarArg(), MF, ArgLocs2, F.getContext());
1105 if (!determineAssignments(Assigner&: CallerAssigner, Args&: InArgs, CCInfo&: CCInfo2))
1106 return false;
1107
1108 // We need the argument locations to match up exactly. If there's more in
1109 // one than the other, then we are done.
1110 if (ArgLocs1.size() != ArgLocs2.size())
1111 return false;
1112
1113 // Make sure that each location is passed in exactly the same way.
1114 for (unsigned i = 0, e = ArgLocs1.size(); i < e; ++i) {
1115 const CCValAssign &Loc1 = ArgLocs1[i];
1116 const CCValAssign &Loc2 = ArgLocs2[i];
1117
1118 // We need both of them to be the same. So if one is a register and one
1119 // isn't, we're done.
1120 if (Loc1.isRegLoc() != Loc2.isRegLoc())
1121 return false;
1122
1123 if (Loc1.isRegLoc()) {
1124 // If they don't have the same register location, we're done.
1125 if (Loc1.getLocReg() != Loc2.getLocReg())
1126 return false;
1127
1128 // They matched, so we can move to the next ArgLoc.
1129 continue;
1130 }
1131
1132 // Loc1 wasn't a RegLoc, so they both must be MemLocs. Check if they match.
1133 if (Loc1.getLocMemOffset() != Loc2.getLocMemOffset())
1134 return false;
1135 }
1136
1137 return true;
1138}
1139
1140LLT CallLowering::ValueHandler::getStackValueStoreType(
1141 const DataLayout &DL, const CCValAssign &VA, ISD::ArgFlagsTy Flags) const {
1142 const MVT ValVT = VA.getValVT();
1143 if (ValVT != MVT::iPTR) {
1144 LLT ValTy(ValVT);
1145
1146 // We lost the pointeriness going through CCValAssign, so try to restore it
1147 // based on the flags.
1148 if (Flags.isPointer()) {
1149 LLT PtrTy = LLT::pointer(AddressSpace: Flags.getPointerAddrSpace(),
1150 SizeInBits: ValTy.getScalarSizeInBits());
1151 if (ValVT.isVector())
1152 return LLT::vector(EC: ValTy.getElementCount(), ScalarTy: PtrTy);
1153 return PtrTy;
1154 }
1155
1156 return ValTy;
1157 }
1158
1159 unsigned AddrSpace = Flags.getPointerAddrSpace();
1160 return LLT::pointer(AddressSpace: AddrSpace, SizeInBits: DL.getPointerSize(AS: AddrSpace));
1161}
1162
1163void CallLowering::ValueHandler::copyArgumentMemory(
1164 const ArgInfo &Arg, Register DstPtr, Register SrcPtr,
1165 const MachinePointerInfo &DstPtrInfo, Align DstAlign,
1166 const MachinePointerInfo &SrcPtrInfo, Align SrcAlign, uint64_t MemSize,
1167 CCValAssign &VA) const {
1168 MachineFunction &MF = MIRBuilder.getMF();
1169 MachineMemOperand *SrcMMO = MF.getMachineMemOperand(
1170 PtrInfo: SrcPtrInfo,
1171 F: MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable, Size: MemSize,
1172 BaseAlignment: SrcAlign);
1173
1174 MachineMemOperand *DstMMO = MF.getMachineMemOperand(
1175 PtrInfo: DstPtrInfo,
1176 F: MachineMemOperand::MOStore | MachineMemOperand::MODereferenceable,
1177 Size: MemSize, BaseAlignment: DstAlign);
1178
1179 const LLT PtrTy = MRI.getType(Reg: DstPtr);
1180 const LLT SizeTy = LLT::scalar(SizeInBits: PtrTy.getSizeInBits());
1181
1182 auto SizeConst = MIRBuilder.buildConstant(Res: SizeTy, Val: MemSize);
1183 MIRBuilder.buildMemCpy(DstPtr, SrcPtr, Size: SizeConst, DstMMO&: *DstMMO, SrcMMO&: *SrcMMO);
1184}
1185
1186Register CallLowering::ValueHandler::extendRegister(Register ValReg,
1187 const CCValAssign &VA,
1188 unsigned MaxSizeBits) {
1189 LLT LocTy{VA.getLocVT()};
1190 LLT ValTy{VA.getValVT()};
1191
1192 if (LocTy.getSizeInBits() == ValTy.getSizeInBits())
1193 return ValReg;
1194
1195 if (LocTy.isScalar() && MaxSizeBits && MaxSizeBits < LocTy.getSizeInBits()) {
1196 if (MaxSizeBits <= ValTy.getSizeInBits())
1197 return ValReg;
1198 LocTy = LLT::scalar(SizeInBits: MaxSizeBits);
1199 }
1200
1201 const LLT ValRegTy = MRI.getType(Reg: ValReg);
1202 if (ValRegTy.isPointer()) {
1203 // The x32 ABI wants to zero extend 32-bit pointers to 64-bit registers, so
1204 // we have to cast to do the extension.
1205 LLT IntPtrTy = LLT::scalar(SizeInBits: ValRegTy.getSizeInBits());
1206 ValReg = MIRBuilder.buildPtrToInt(Dst: IntPtrTy, Src: ValReg).getReg(Idx: 0);
1207 }
1208
1209 switch (VA.getLocInfo()) {
1210 default: break;
1211 case CCValAssign::Full:
1212 case CCValAssign::BCvt:
1213 // FIXME: bitconverting between vector types may or may not be a
1214 // nop in big-endian situations.
1215 return ValReg;
1216 case CCValAssign::AExt: {
1217 auto MIB = MIRBuilder.buildAnyExt(Res: LocTy, Op: ValReg);
1218 return MIB.getReg(Idx: 0);
1219 }
1220 case CCValAssign::SExt: {
1221 Register NewReg = MRI.createGenericVirtualRegister(Ty: LocTy);
1222 MIRBuilder.buildSExt(Res: NewReg, Op: ValReg);
1223 return NewReg;
1224 }
1225 case CCValAssign::ZExt: {
1226 Register NewReg = MRI.createGenericVirtualRegister(Ty: LocTy);
1227 MIRBuilder.buildZExt(Res: NewReg, Op: ValReg);
1228 return NewReg;
1229 }
1230 }
1231 llvm_unreachable("unable to extend register");
1232}
1233
1234void CallLowering::ValueAssigner::anchor() {}
1235
1236Register CallLowering::IncomingValueHandler::buildExtensionHint(
1237 const CCValAssign &VA, Register SrcReg, LLT NarrowTy) {
1238 switch (VA.getLocInfo()) {
1239 case CCValAssign::LocInfo::ZExt: {
1240 return MIRBuilder
1241 .buildAssertZExt(Res: MRI.cloneVirtualRegister(VReg: SrcReg), Op: SrcReg,
1242 Size: NarrowTy.getScalarSizeInBits())
1243 .getReg(Idx: 0);
1244 }
1245 case CCValAssign::LocInfo::SExt: {
1246 return MIRBuilder
1247 .buildAssertSExt(Res: MRI.cloneVirtualRegister(VReg: SrcReg), Op: SrcReg,
1248 Size: NarrowTy.getScalarSizeInBits())
1249 .getReg(Idx: 0);
1250 break;
1251 }
1252 default:
1253 return SrcReg;
1254 }
1255}
1256
1257/// Check if we can use a basic COPY instruction between the two types.
1258///
1259/// We're currently building on top of the infrastructure using MVT, which loses
1260/// pointer information in the CCValAssign. We accept copies from physical
1261/// registers that have been reported as integers if it's to an equivalent sized
1262/// pointer LLT.
1263static bool isCopyCompatibleType(LLT SrcTy, LLT DstTy) {
1264 if (SrcTy == DstTy)
1265 return true;
1266
1267 if (SrcTy.getSizeInBits() != DstTy.getSizeInBits())
1268 return false;
1269
1270 SrcTy = SrcTy.getScalarType();
1271 DstTy = DstTy.getScalarType();
1272
1273 return (SrcTy.isPointer() && DstTy.isScalar()) ||
1274 (DstTy.isPointer() && SrcTy.isScalar());
1275}
1276
1277void CallLowering::IncomingValueHandler::assignValueToReg(
1278 Register ValVReg, Register PhysReg, const CCValAssign &VA) {
1279 const MVT LocVT = VA.getLocVT();
1280 const LLT LocTy(LocVT);
1281 const LLT RegTy = MRI.getType(Reg: ValVReg);
1282
1283 if (isCopyCompatibleType(SrcTy: RegTy, DstTy: LocTy)) {
1284 MIRBuilder.buildCopy(Res: ValVReg, Op: PhysReg);
1285 return;
1286 }
1287
1288 auto Copy = MIRBuilder.buildCopy(Res: LocTy, Op: PhysReg);
1289 auto Hint = buildExtensionHint(VA, SrcReg: Copy.getReg(Idx: 0), NarrowTy: RegTy);
1290 MIRBuilder.buildTrunc(Res: ValVReg, Op: Hint);
1291}
1292

source code of llvm/lib/CodeGen/GlobalISel/CallLowering.cpp