1 | //===-- lib/CodeGen/GlobalISel/CallLowering.cpp - Call lowering -----------===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | /// |
9 | /// \file |
10 | /// This file implements some simple delegations needed for call lowering. |
11 | /// |
12 | //===----------------------------------------------------------------------===// |
13 | |
14 | #include "llvm/CodeGen/GlobalISel/CallLowering.h" |
15 | #include "llvm/CodeGen/Analysis.h" |
16 | #include "llvm/CodeGen/CallingConvLower.h" |
17 | #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" |
18 | #include "llvm/CodeGen/GlobalISel/Utils.h" |
19 | #include "llvm/CodeGen/MachineFrameInfo.h" |
20 | #include "llvm/CodeGen/MachineOperand.h" |
21 | #include "llvm/CodeGen/MachineRegisterInfo.h" |
22 | #include "llvm/CodeGen/TargetLowering.h" |
23 | #include "llvm/IR/DataLayout.h" |
24 | #include "llvm/IR/IntrinsicInst.h" |
25 | #include "llvm/IR/LLVMContext.h" |
26 | #include "llvm/IR/Module.h" |
27 | #include "llvm/Target/TargetMachine.h" |
28 | |
29 | #define DEBUG_TYPE "call-lowering" |
30 | |
31 | using namespace llvm; |
32 | |
33 | void CallLowering::anchor() {} |
34 | |
35 | /// Helper function which updates \p Flags when \p AttrFn returns true. |
36 | static void |
37 | addFlagsUsingAttrFn(ISD::ArgFlagsTy &Flags, |
38 | const std::function<bool(Attribute::AttrKind)> &AttrFn) { |
39 | if (AttrFn(Attribute::SExt)) |
40 | Flags.setSExt(); |
41 | if (AttrFn(Attribute::ZExt)) |
42 | Flags.setZExt(); |
43 | if (AttrFn(Attribute::InReg)) |
44 | Flags.setInReg(); |
45 | if (AttrFn(Attribute::StructRet)) |
46 | Flags.setSRet(); |
47 | if (AttrFn(Attribute::Nest)) |
48 | Flags.setNest(); |
49 | if (AttrFn(Attribute::ByVal)) |
50 | Flags.setByVal(); |
51 | if (AttrFn(Attribute::Preallocated)) |
52 | Flags.setPreallocated(); |
53 | if (AttrFn(Attribute::InAlloca)) |
54 | Flags.setInAlloca(); |
55 | if (AttrFn(Attribute::Returned)) |
56 | Flags.setReturned(); |
57 | if (AttrFn(Attribute::SwiftSelf)) |
58 | Flags.setSwiftSelf(); |
59 | if (AttrFn(Attribute::SwiftAsync)) |
60 | Flags.setSwiftAsync(); |
61 | if (AttrFn(Attribute::SwiftError)) |
62 | Flags.setSwiftError(); |
63 | } |
64 | |
65 | ISD::ArgFlagsTy CallLowering::getAttributesForArgIdx(const CallBase &Call, |
66 | unsigned ArgIdx) const { |
67 | ISD::ArgFlagsTy Flags; |
68 | addFlagsUsingAttrFn(Flags, AttrFn: [&Call, &ArgIdx](Attribute::AttrKind Attr) { |
69 | return Call.paramHasAttr(ArgNo: ArgIdx, Kind: Attr); |
70 | }); |
71 | return Flags; |
72 | } |
73 | |
74 | ISD::ArgFlagsTy |
75 | CallLowering::getAttributesForReturn(const CallBase &Call) const { |
76 | ISD::ArgFlagsTy Flags; |
77 | addFlagsUsingAttrFn(Flags, AttrFn: [&Call](Attribute::AttrKind Attr) { |
78 | return Call.hasRetAttr(Kind: Attr); |
79 | }); |
80 | return Flags; |
81 | } |
82 | |
83 | void CallLowering::addArgFlagsFromAttributes(ISD::ArgFlagsTy &Flags, |
84 | const AttributeList &Attrs, |
85 | unsigned OpIdx) const { |
86 | addFlagsUsingAttrFn(Flags, AttrFn: [&Attrs, &OpIdx](Attribute::AttrKind Attr) { |
87 | return Attrs.hasAttributeAtIndex(Index: OpIdx, Kind: Attr); |
88 | }); |
89 | } |
90 | |
91 | bool CallLowering::lowerCall(MachineIRBuilder &MIRBuilder, const CallBase &CB, |
92 | ArrayRef<Register> ResRegs, |
93 | ArrayRef<ArrayRef<Register>> ArgRegs, |
94 | Register SwiftErrorVReg, |
95 | Register ConvergenceCtrlToken, |
96 | std::function<unsigned()> GetCalleeReg) const { |
97 | CallLoweringInfo Info; |
98 | const DataLayout &DL = MIRBuilder.getDataLayout(); |
99 | MachineFunction &MF = MIRBuilder.getMF(); |
100 | MachineRegisterInfo &MRI = MF.getRegInfo(); |
101 | bool CanBeTailCalled = CB.isTailCall() && |
102 | isInTailCallPosition(Call: CB, TM: MF.getTarget()) && |
103 | (MF.getFunction() |
104 | .getFnAttribute(Kind: "disable-tail-calls" ) |
105 | .getValueAsString() != "true" ); |
106 | |
107 | CallingConv::ID CallConv = CB.getCallingConv(); |
108 | Type *RetTy = CB.getType(); |
109 | bool IsVarArg = CB.getFunctionType()->isVarArg(); |
110 | |
111 | SmallVector<BaseArgInfo, 4> SplitArgs; |
112 | getReturnInfo(CallConv, RetTy, Attrs: CB.getAttributes(), Outs&: SplitArgs, DL); |
113 | Info.CanLowerReturn = canLowerReturn(MF, CallConv, Outs&: SplitArgs, IsVarArg); |
114 | |
115 | Info.IsConvergent = CB.isConvergent(); |
116 | |
117 | if (!Info.CanLowerReturn) { |
118 | // Callee requires sret demotion. |
119 | insertSRetOutgoingArgument(MIRBuilder, CB, Info); |
120 | |
121 | // The sret demotion isn't compatible with tail-calls, since the sret |
122 | // argument points into the caller's stack frame. |
123 | CanBeTailCalled = false; |
124 | } |
125 | |
126 | // First step is to marshall all the function's parameters into the correct |
127 | // physregs and memory locations. Gather the sequence of argument types that |
128 | // we'll pass to the assigner function. |
129 | unsigned i = 0; |
130 | unsigned NumFixedArgs = CB.getFunctionType()->getNumParams(); |
131 | for (const auto &Arg : CB.args()) { |
132 | ArgInfo OrigArg{ArgRegs[i], *Arg.get(), i, getAttributesForArgIdx(Call: CB, ArgIdx: i), |
133 | i < NumFixedArgs}; |
134 | setArgFlags(Arg&: OrigArg, OpIdx: i + AttributeList::FirstArgIndex, DL, FuncInfo: CB); |
135 | |
136 | // If we have an explicit sret argument that is an Instruction, (i.e., it |
137 | // might point to function-local memory), we can't meaningfully tail-call. |
138 | if (OrigArg.Flags[0].isSRet() && isa<Instruction>(Val: &Arg)) |
139 | CanBeTailCalled = false; |
140 | |
141 | Info.OrigArgs.push_back(Elt: OrigArg); |
142 | ++i; |
143 | } |
144 | |
145 | // Try looking through a bitcast from one function type to another. |
146 | // Commonly happens with calls to objc_msgSend(). |
147 | const Value *CalleeV = CB.getCalledOperand()->stripPointerCasts(); |
148 | if (const Function *F = dyn_cast<Function>(Val: CalleeV)) { |
149 | if (F->hasFnAttribute(Attribute::NonLazyBind)) { |
150 | LLT Ty = getLLTForType(Ty&: *F->getType(), DL); |
151 | Register Reg = MIRBuilder.buildGlobalValue(Res: Ty, GV: F).getReg(Idx: 0); |
152 | Info.Callee = MachineOperand::CreateReg(Reg, isDef: false); |
153 | } else { |
154 | Info.Callee = MachineOperand::CreateGA(GV: F, Offset: 0); |
155 | } |
156 | } else if (isa<GlobalIFunc>(Val: CalleeV) || isa<GlobalAlias>(Val: CalleeV)) { |
157 | // IR IFuncs and Aliases can't be forward declared (only defined), so the |
158 | // callee must be in the same TU and therefore we can direct-call it without |
159 | // worrying about it being out of range. |
160 | Info.Callee = MachineOperand::CreateGA(GV: cast<GlobalValue>(Val: CalleeV), Offset: 0); |
161 | } else |
162 | Info.Callee = MachineOperand::CreateReg(Reg: GetCalleeReg(), isDef: false); |
163 | |
164 | Register ReturnHintAlignReg; |
165 | Align ReturnHintAlign; |
166 | |
167 | Info.OrigRet = ArgInfo{ResRegs, RetTy, 0, getAttributesForReturn(Call: CB)}; |
168 | |
169 | if (!Info.OrigRet.Ty->isVoidTy()) { |
170 | setArgFlags(Arg&: Info.OrigRet, OpIdx: AttributeList::ReturnIndex, DL, FuncInfo: CB); |
171 | |
172 | if (MaybeAlign Alignment = CB.getRetAlign()) { |
173 | if (*Alignment > Align(1)) { |
174 | ReturnHintAlignReg = MRI.cloneVirtualRegister(VReg: ResRegs[0]); |
175 | Info.OrigRet.Regs[0] = ReturnHintAlignReg; |
176 | ReturnHintAlign = *Alignment; |
177 | } |
178 | } |
179 | } |
180 | |
181 | auto Bundle = CB.getOperandBundle(ID: LLVMContext::OB_kcfi); |
182 | if (Bundle && CB.isIndirectCall()) { |
183 | Info.CFIType = cast<ConstantInt>(Val: Bundle->Inputs[0]); |
184 | assert(Info.CFIType->getType()->isIntegerTy(32) && "Invalid CFI type" ); |
185 | } |
186 | |
187 | Info.CB = &CB; |
188 | Info.KnownCallees = CB.getMetadata(KindID: LLVMContext::MD_callees); |
189 | Info.CallConv = CallConv; |
190 | Info.SwiftErrorVReg = SwiftErrorVReg; |
191 | Info.ConvergenceCtrlToken = ConvergenceCtrlToken; |
192 | Info.IsMustTailCall = CB.isMustTailCall(); |
193 | Info.IsTailCall = CanBeTailCalled; |
194 | Info.IsVarArg = IsVarArg; |
195 | if (!lowerCall(MIRBuilder, Info)) |
196 | return false; |
197 | |
198 | if (ReturnHintAlignReg && !Info.LoweredTailCall) { |
199 | MIRBuilder.buildAssertAlign(Res: ResRegs[0], Op: ReturnHintAlignReg, |
200 | AlignVal: ReturnHintAlign); |
201 | } |
202 | |
203 | return true; |
204 | } |
205 | |
206 | template <typename FuncInfoTy> |
207 | void CallLowering::setArgFlags(CallLowering::ArgInfo &Arg, unsigned OpIdx, |
208 | const DataLayout &DL, |
209 | const FuncInfoTy &FuncInfo) const { |
210 | auto &Flags = Arg.Flags[0]; |
211 | const AttributeList &Attrs = FuncInfo.getAttributes(); |
212 | addArgFlagsFromAttributes(Flags, Attrs, OpIdx); |
213 | |
214 | PointerType *PtrTy = dyn_cast<PointerType>(Val: Arg.Ty->getScalarType()); |
215 | if (PtrTy) { |
216 | Flags.setPointer(); |
217 | Flags.setPointerAddrSpace(PtrTy->getPointerAddressSpace()); |
218 | } |
219 | |
220 | Align MemAlign = DL.getABITypeAlign(Ty: Arg.Ty); |
221 | if (Flags.isByVal() || Flags.isInAlloca() || Flags.isPreallocated()) { |
222 | assert(OpIdx >= AttributeList::FirstArgIndex); |
223 | unsigned ParamIdx = OpIdx - AttributeList::FirstArgIndex; |
224 | |
225 | Type *ElementTy = FuncInfo.getParamByValType(ParamIdx); |
226 | if (!ElementTy) |
227 | ElementTy = FuncInfo.getParamInAllocaType(ParamIdx); |
228 | if (!ElementTy) |
229 | ElementTy = FuncInfo.getParamPreallocatedType(ParamIdx); |
230 | assert(ElementTy && "Must have byval, inalloca or preallocated type" ); |
231 | Flags.setByValSize(DL.getTypeAllocSize(Ty: ElementTy)); |
232 | |
233 | // For ByVal, alignment should be passed from FE. BE will guess if |
234 | // this info is not there but there are cases it cannot get right. |
235 | if (auto ParamAlign = FuncInfo.getParamStackAlign(ParamIdx)) |
236 | MemAlign = *ParamAlign; |
237 | else if ((ParamAlign = FuncInfo.getParamAlign(ParamIdx))) |
238 | MemAlign = *ParamAlign; |
239 | else |
240 | MemAlign = Align(getTLI()->getByValTypeAlignment(Ty: ElementTy, DL)); |
241 | } else if (OpIdx >= AttributeList::FirstArgIndex) { |
242 | if (auto ParamAlign = |
243 | FuncInfo.getParamStackAlign(OpIdx - AttributeList::FirstArgIndex)) |
244 | MemAlign = *ParamAlign; |
245 | } |
246 | Flags.setMemAlign(MemAlign); |
247 | Flags.setOrigAlign(DL.getABITypeAlign(Ty: Arg.Ty)); |
248 | |
249 | // Don't try to use the returned attribute if the argument is marked as |
250 | // swiftself, since it won't be passed in x0. |
251 | if (Flags.isSwiftSelf()) |
252 | Flags.setReturned(false); |
253 | } |
254 | |
255 | template void |
256 | CallLowering::setArgFlags<Function>(CallLowering::ArgInfo &Arg, unsigned OpIdx, |
257 | const DataLayout &DL, |
258 | const Function &FuncInfo) const; |
259 | |
260 | template void |
261 | CallLowering::setArgFlags<CallBase>(CallLowering::ArgInfo &Arg, unsigned OpIdx, |
262 | const DataLayout &DL, |
263 | const CallBase &FuncInfo) const; |
264 | |
265 | void CallLowering::splitToValueTypes(const ArgInfo &OrigArg, |
266 | SmallVectorImpl<ArgInfo> &SplitArgs, |
267 | const DataLayout &DL, |
268 | CallingConv::ID CallConv, |
269 | SmallVectorImpl<uint64_t> *Offsets) const { |
270 | LLVMContext &Ctx = OrigArg.Ty->getContext(); |
271 | |
272 | SmallVector<EVT, 4> SplitVTs; |
273 | ComputeValueVTs(TLI: *TLI, DL, Ty: OrigArg.Ty, ValueVTs&: SplitVTs, FixedOffsets: Offsets, StartingOffset: 0); |
274 | |
275 | if (SplitVTs.size() == 0) |
276 | return; |
277 | |
278 | if (SplitVTs.size() == 1) { |
279 | // No splitting to do, but we want to replace the original type (e.g. [1 x |
280 | // double] -> double). |
281 | SplitArgs.emplace_back(Args: OrigArg.Regs[0], Args: SplitVTs[0].getTypeForEVT(Context&: Ctx), |
282 | Args: OrigArg.OrigArgIndex, Args: OrigArg.Flags[0], |
283 | Args: OrigArg.IsFixed, Args: OrigArg.OrigValue); |
284 | return; |
285 | } |
286 | |
287 | // Create one ArgInfo for each virtual register in the original ArgInfo. |
288 | assert(OrigArg.Regs.size() == SplitVTs.size() && "Regs / types mismatch" ); |
289 | |
290 | bool NeedsRegBlock = TLI->functionArgumentNeedsConsecutiveRegisters( |
291 | Ty: OrigArg.Ty, CallConv, isVarArg: false, DL); |
292 | for (unsigned i = 0, e = SplitVTs.size(); i < e; ++i) { |
293 | Type *SplitTy = SplitVTs[i].getTypeForEVT(Context&: Ctx); |
294 | SplitArgs.emplace_back(Args: OrigArg.Regs[i], Args&: SplitTy, Args: OrigArg.OrigArgIndex, |
295 | Args: OrigArg.Flags[0], Args: OrigArg.IsFixed); |
296 | if (NeedsRegBlock) |
297 | SplitArgs.back().Flags[0].setInConsecutiveRegs(); |
298 | } |
299 | |
300 | SplitArgs.back().Flags[0].setInConsecutiveRegsLast(); |
301 | } |
302 | |
303 | /// Pack values \p SrcRegs to cover the vector type result \p DstRegs. |
304 | static MachineInstrBuilder |
305 | mergeVectorRegsToResultRegs(MachineIRBuilder &B, ArrayRef<Register> DstRegs, |
306 | ArrayRef<Register> SrcRegs) { |
307 | MachineRegisterInfo &MRI = *B.getMRI(); |
308 | LLT LLTy = MRI.getType(Reg: DstRegs[0]); |
309 | LLT PartLLT = MRI.getType(Reg: SrcRegs[0]); |
310 | |
311 | // Deal with v3s16 split into v2s16 |
312 | LLT LCMTy = getCoverTy(OrigTy: LLTy, TargetTy: PartLLT); |
313 | if (LCMTy == LLTy) { |
314 | // Common case where no padding is needed. |
315 | assert(DstRegs.size() == 1); |
316 | return B.buildConcatVectors(Res: DstRegs[0], Ops: SrcRegs); |
317 | } |
318 | |
319 | // We need to create an unmerge to the result registers, which may require |
320 | // widening the original value. |
321 | Register UnmergeSrcReg; |
322 | if (LCMTy != PartLLT) { |
323 | assert(DstRegs.size() == 1); |
324 | return B.buildDeleteTrailingVectorElements( |
325 | Res: DstRegs[0], Op0: B.buildMergeLikeInstr(Res: LCMTy, Ops: SrcRegs)); |
326 | } else { |
327 | // We don't need to widen anything if we're extracting a scalar which was |
328 | // promoted to a vector e.g. s8 -> v4s8 -> s8 |
329 | assert(SrcRegs.size() == 1); |
330 | UnmergeSrcReg = SrcRegs[0]; |
331 | } |
332 | |
333 | int NumDst = LCMTy.getSizeInBits() / LLTy.getSizeInBits(); |
334 | |
335 | SmallVector<Register, 8> PadDstRegs(NumDst); |
336 | std::copy(first: DstRegs.begin(), last: DstRegs.end(), result: PadDstRegs.begin()); |
337 | |
338 | // Create the excess dead defs for the unmerge. |
339 | for (int I = DstRegs.size(); I != NumDst; ++I) |
340 | PadDstRegs[I] = MRI.createGenericVirtualRegister(Ty: LLTy); |
341 | |
342 | if (PadDstRegs.size() == 1) |
343 | return B.buildDeleteTrailingVectorElements(Res: DstRegs[0], Op0: UnmergeSrcReg); |
344 | return B.buildUnmerge(Res: PadDstRegs, Op: UnmergeSrcReg); |
345 | } |
346 | |
347 | /// Create a sequence of instructions to combine pieces split into register |
348 | /// typed values to the original IR value. \p OrigRegs contains the destination |
349 | /// value registers of type \p LLTy, and \p Regs contains the legalized pieces |
350 | /// with type \p PartLLT. This is used for incoming values (physregs to vregs). |
351 | static void buildCopyFromRegs(MachineIRBuilder &B, ArrayRef<Register> OrigRegs, |
352 | ArrayRef<Register> Regs, LLT LLTy, LLT PartLLT, |
353 | const ISD::ArgFlagsTy Flags) { |
354 | MachineRegisterInfo &MRI = *B.getMRI(); |
355 | |
356 | if (PartLLT == LLTy) { |
357 | // We should have avoided introducing a new virtual register, and just |
358 | // directly assigned here. |
359 | assert(OrigRegs[0] == Regs[0]); |
360 | return; |
361 | } |
362 | |
363 | if (PartLLT.getSizeInBits() == LLTy.getSizeInBits() && OrigRegs.size() == 1 && |
364 | Regs.size() == 1) { |
365 | B.buildBitcast(Dst: OrigRegs[0], Src: Regs[0]); |
366 | return; |
367 | } |
368 | |
369 | // A vector PartLLT needs extending to LLTy's element size. |
370 | // E.g. <2 x s64> = G_SEXT <2 x s32>. |
371 | if (PartLLT.isVector() == LLTy.isVector() && |
372 | PartLLT.getScalarSizeInBits() > LLTy.getScalarSizeInBits() && |
373 | (!PartLLT.isVector() || |
374 | PartLLT.getElementCount() == LLTy.getElementCount()) && |
375 | OrigRegs.size() == 1 && Regs.size() == 1) { |
376 | Register SrcReg = Regs[0]; |
377 | |
378 | LLT LocTy = MRI.getType(Reg: SrcReg); |
379 | |
380 | if (Flags.isSExt()) { |
381 | SrcReg = B.buildAssertSExt(Res: LocTy, Op: SrcReg, Size: LLTy.getScalarSizeInBits()) |
382 | .getReg(Idx: 0); |
383 | } else if (Flags.isZExt()) { |
384 | SrcReg = B.buildAssertZExt(Res: LocTy, Op: SrcReg, Size: LLTy.getScalarSizeInBits()) |
385 | .getReg(Idx: 0); |
386 | } |
387 | |
388 | // Sometimes pointers are passed zero extended. |
389 | LLT OrigTy = MRI.getType(Reg: OrigRegs[0]); |
390 | if (OrigTy.isPointer()) { |
391 | LLT IntPtrTy = LLT::scalar(SizeInBits: OrigTy.getSizeInBits()); |
392 | B.buildIntToPtr(Dst: OrigRegs[0], Src: B.buildTrunc(Res: IntPtrTy, Op: SrcReg)); |
393 | return; |
394 | } |
395 | |
396 | B.buildTrunc(Res: OrigRegs[0], Op: SrcReg); |
397 | return; |
398 | } |
399 | |
400 | if (!LLTy.isVector() && !PartLLT.isVector()) { |
401 | assert(OrigRegs.size() == 1); |
402 | LLT OrigTy = MRI.getType(Reg: OrigRegs[0]); |
403 | |
404 | unsigned SrcSize = PartLLT.getSizeInBits().getFixedValue() * Regs.size(); |
405 | if (SrcSize == OrigTy.getSizeInBits()) |
406 | B.buildMergeValues(Res: OrigRegs[0], Ops: Regs); |
407 | else { |
408 | auto Widened = B.buildMergeLikeInstr(Res: LLT::scalar(SizeInBits: SrcSize), Ops: Regs); |
409 | B.buildTrunc(Res: OrigRegs[0], Op: Widened); |
410 | } |
411 | |
412 | return; |
413 | } |
414 | |
415 | if (PartLLT.isVector()) { |
416 | assert(OrigRegs.size() == 1); |
417 | SmallVector<Register> CastRegs(Regs.begin(), Regs.end()); |
418 | |
419 | // If PartLLT is a mismatched vector in both number of elements and element |
420 | // size, e.g. PartLLT == v2s64 and LLTy is v3s32, then first coerce it to |
421 | // have the same elt type, i.e. v4s32. |
422 | // TODO: Extend this coersion to element multiples other than just 2. |
423 | if (TypeSize::isKnownGT(LHS: PartLLT.getSizeInBits(), RHS: LLTy.getSizeInBits()) && |
424 | PartLLT.getScalarSizeInBits() == LLTy.getScalarSizeInBits() * 2 && |
425 | Regs.size() == 1) { |
426 | LLT NewTy = PartLLT.changeElementType(NewEltTy: LLTy.getElementType()) |
427 | .changeElementCount(EC: PartLLT.getElementCount() * 2); |
428 | CastRegs[0] = B.buildBitcast(Dst: NewTy, Src: Regs[0]).getReg(Idx: 0); |
429 | PartLLT = NewTy; |
430 | } |
431 | |
432 | if (LLTy.getScalarType() == PartLLT.getElementType()) { |
433 | mergeVectorRegsToResultRegs(B, DstRegs: OrigRegs, SrcRegs: CastRegs); |
434 | } else { |
435 | unsigned I = 0; |
436 | LLT GCDTy = getGCDType(OrigTy: LLTy, TargetTy: PartLLT); |
437 | |
438 | // We are both splitting a vector, and bitcasting its element types. Cast |
439 | // the source pieces into the appropriate number of pieces with the result |
440 | // element type. |
441 | for (Register SrcReg : CastRegs) |
442 | CastRegs[I++] = B.buildBitcast(Dst: GCDTy, Src: SrcReg).getReg(Idx: 0); |
443 | mergeVectorRegsToResultRegs(B, DstRegs: OrigRegs, SrcRegs: CastRegs); |
444 | } |
445 | |
446 | return; |
447 | } |
448 | |
449 | assert(LLTy.isVector() && !PartLLT.isVector()); |
450 | |
451 | LLT DstEltTy = LLTy.getElementType(); |
452 | |
453 | // Pointer information was discarded. We'll need to coerce some register types |
454 | // to avoid violating type constraints. |
455 | LLT RealDstEltTy = MRI.getType(Reg: OrigRegs[0]).getElementType(); |
456 | |
457 | assert(DstEltTy.getSizeInBits() == RealDstEltTy.getSizeInBits()); |
458 | |
459 | if (DstEltTy == PartLLT) { |
460 | // Vector was trivially scalarized. |
461 | |
462 | if (RealDstEltTy.isPointer()) { |
463 | for (Register Reg : Regs) |
464 | MRI.setType(VReg: Reg, Ty: RealDstEltTy); |
465 | } |
466 | |
467 | B.buildBuildVector(Res: OrigRegs[0], Ops: Regs); |
468 | } else if (DstEltTy.getSizeInBits() > PartLLT.getSizeInBits()) { |
469 | // Deal with vector with 64-bit elements decomposed to 32-bit |
470 | // registers. Need to create intermediate 64-bit elements. |
471 | SmallVector<Register, 8> EltMerges; |
472 | int PartsPerElt = DstEltTy.getSizeInBits() / PartLLT.getSizeInBits(); |
473 | |
474 | assert(DstEltTy.getSizeInBits() % PartLLT.getSizeInBits() == 0); |
475 | |
476 | for (int I = 0, NumElts = LLTy.getNumElements(); I != NumElts; ++I) { |
477 | auto Merge = |
478 | B.buildMergeLikeInstr(Res: RealDstEltTy, Ops: Regs.take_front(N: PartsPerElt)); |
479 | // Fix the type in case this is really a vector of pointers. |
480 | MRI.setType(VReg: Merge.getReg(Idx: 0), Ty: RealDstEltTy); |
481 | EltMerges.push_back(Elt: Merge.getReg(Idx: 0)); |
482 | Regs = Regs.drop_front(N: PartsPerElt); |
483 | } |
484 | |
485 | B.buildBuildVector(Res: OrigRegs[0], Ops: EltMerges); |
486 | } else { |
487 | // Vector was split, and elements promoted to a wider type. |
488 | // FIXME: Should handle floating point promotions. |
489 | unsigned NumElts = LLTy.getNumElements(); |
490 | LLT BVType = LLT::fixed_vector(NumElements: NumElts, ScalarTy: PartLLT); |
491 | |
492 | Register BuildVec; |
493 | if (NumElts == Regs.size()) |
494 | BuildVec = B.buildBuildVector(Res: BVType, Ops: Regs).getReg(Idx: 0); |
495 | else { |
496 | // Vector elements are packed in the inputs. |
497 | // e.g. we have a <4 x s16> but 2 x s32 in regs. |
498 | assert(NumElts > Regs.size()); |
499 | LLT SrcEltTy = MRI.getType(Reg: Regs[0]); |
500 | |
501 | LLT OriginalEltTy = MRI.getType(Reg: OrigRegs[0]).getElementType(); |
502 | |
503 | // Input registers contain packed elements. |
504 | // Determine how many elements per reg. |
505 | assert((SrcEltTy.getSizeInBits() % OriginalEltTy.getSizeInBits()) == 0); |
506 | unsigned EltPerReg = |
507 | (SrcEltTy.getSizeInBits() / OriginalEltTy.getSizeInBits()); |
508 | |
509 | SmallVector<Register, 0> BVRegs; |
510 | BVRegs.reserve(N: Regs.size() * EltPerReg); |
511 | for (Register R : Regs) { |
512 | auto Unmerge = B.buildUnmerge(Res: OriginalEltTy, Op: R); |
513 | for (unsigned K = 0; K < EltPerReg; ++K) |
514 | BVRegs.push_back(Elt: B.buildAnyExt(Res: PartLLT, Op: Unmerge.getReg(Idx: K)).getReg(Idx: 0)); |
515 | } |
516 | |
517 | // We may have some more elements in BVRegs, e.g. if we have 2 s32 pieces |
518 | // for a <3 x s16> vector. We should have less than EltPerReg extra items. |
519 | if (BVRegs.size() > NumElts) { |
520 | assert((BVRegs.size() - NumElts) < EltPerReg); |
521 | BVRegs.truncate(N: NumElts); |
522 | } |
523 | BuildVec = B.buildBuildVector(Res: BVType, Ops: BVRegs).getReg(Idx: 0); |
524 | } |
525 | B.buildTrunc(Res: OrigRegs[0], Op: BuildVec); |
526 | } |
527 | } |
528 | |
529 | /// Create a sequence of instructions to expand the value in \p SrcReg (of type |
530 | /// \p SrcTy) to the types in \p DstRegs (of type \p PartTy). \p ExtendOp should |
531 | /// contain the type of scalar value extension if necessary. |
532 | /// |
533 | /// This is used for outgoing values (vregs to physregs) |
534 | static void buildCopyToRegs(MachineIRBuilder &B, ArrayRef<Register> DstRegs, |
535 | Register SrcReg, LLT SrcTy, LLT PartTy, |
536 | unsigned ExtendOp = TargetOpcode::G_ANYEXT) { |
537 | // We could just insert a regular copy, but this is unreachable at the moment. |
538 | assert(SrcTy != PartTy && "identical part types shouldn't reach here" ); |
539 | |
540 | const TypeSize PartSize = PartTy.getSizeInBits(); |
541 | |
542 | if (PartTy.isVector() == SrcTy.isVector() && |
543 | PartTy.getScalarSizeInBits() > SrcTy.getScalarSizeInBits()) { |
544 | assert(DstRegs.size() == 1); |
545 | B.buildInstr(Opc: ExtendOp, DstOps: {DstRegs[0]}, SrcOps: {SrcReg}); |
546 | return; |
547 | } |
548 | |
549 | if (SrcTy.isVector() && !PartTy.isVector() && |
550 | TypeSize::isKnownGT(LHS: PartSize, RHS: SrcTy.getElementType().getSizeInBits())) { |
551 | // Vector was scalarized, and the elements extended. |
552 | auto UnmergeToEltTy = B.buildUnmerge(Res: SrcTy.getElementType(), Op: SrcReg); |
553 | for (int i = 0, e = DstRegs.size(); i != e; ++i) |
554 | B.buildAnyExt(Res: DstRegs[i], Op: UnmergeToEltTy.getReg(Idx: i)); |
555 | return; |
556 | } |
557 | |
558 | if (SrcTy.isVector() && PartTy.isVector() && |
559 | PartTy.getSizeInBits() == SrcTy.getSizeInBits() && |
560 | ElementCount::isKnownLT(LHS: SrcTy.getElementCount(), |
561 | RHS: PartTy.getElementCount())) { |
562 | // A coercion like: v2f32 -> v4f32 or nxv2f32 -> nxv4f32 |
563 | Register DstReg = DstRegs.front(); |
564 | B.buildPadVectorWithUndefElements(Res: DstReg, Op0: SrcReg); |
565 | return; |
566 | } |
567 | |
568 | LLT GCDTy = getGCDType(OrigTy: SrcTy, TargetTy: PartTy); |
569 | if (GCDTy == PartTy) { |
570 | // If this already evenly divisible, we can create a simple unmerge. |
571 | B.buildUnmerge(Res: DstRegs, Op: SrcReg); |
572 | return; |
573 | } |
574 | |
575 | MachineRegisterInfo &MRI = *B.getMRI(); |
576 | LLT DstTy = MRI.getType(Reg: DstRegs[0]); |
577 | LLT LCMTy = getCoverTy(OrigTy: SrcTy, TargetTy: PartTy); |
578 | |
579 | if (PartTy.isVector() && LCMTy == PartTy) { |
580 | assert(DstRegs.size() == 1); |
581 | B.buildPadVectorWithUndefElements(Res: DstRegs[0], Op0: SrcReg); |
582 | return; |
583 | } |
584 | |
585 | const unsigned DstSize = DstTy.getSizeInBits(); |
586 | const unsigned SrcSize = SrcTy.getSizeInBits(); |
587 | unsigned CoveringSize = LCMTy.getSizeInBits(); |
588 | |
589 | Register UnmergeSrc = SrcReg; |
590 | |
591 | if (!LCMTy.isVector() && CoveringSize != SrcSize) { |
592 | // For scalars, it's common to be able to use a simple extension. |
593 | if (SrcTy.isScalar() && DstTy.isScalar()) { |
594 | CoveringSize = alignTo(Value: SrcSize, Align: DstSize); |
595 | LLT CoverTy = LLT::scalar(SizeInBits: CoveringSize); |
596 | UnmergeSrc = B.buildInstr(Opc: ExtendOp, DstOps: {CoverTy}, SrcOps: {SrcReg}).getReg(Idx: 0); |
597 | } else { |
598 | // Widen to the common type. |
599 | // FIXME: This should respect the extend type |
600 | Register Undef = B.buildUndef(Res: SrcTy).getReg(Idx: 0); |
601 | SmallVector<Register, 8> MergeParts(1, SrcReg); |
602 | for (unsigned Size = SrcSize; Size != CoveringSize; Size += SrcSize) |
603 | MergeParts.push_back(Elt: Undef); |
604 | UnmergeSrc = B.buildMergeLikeInstr(Res: LCMTy, Ops: MergeParts).getReg(Idx: 0); |
605 | } |
606 | } |
607 | |
608 | if (LCMTy.isVector() && CoveringSize != SrcSize) |
609 | UnmergeSrc = B.buildPadVectorWithUndefElements(Res: LCMTy, Op0: SrcReg).getReg(Idx: 0); |
610 | |
611 | B.buildUnmerge(Res: DstRegs, Op: UnmergeSrc); |
612 | } |
613 | |
614 | bool CallLowering::determineAndHandleAssignments( |
615 | ValueHandler &Handler, ValueAssigner &Assigner, |
616 | SmallVectorImpl<ArgInfo> &Args, MachineIRBuilder &MIRBuilder, |
617 | CallingConv::ID CallConv, bool IsVarArg, |
618 | ArrayRef<Register> ThisReturnRegs) const { |
619 | MachineFunction &MF = MIRBuilder.getMF(); |
620 | const Function &F = MF.getFunction(); |
621 | SmallVector<CCValAssign, 16> ArgLocs; |
622 | |
623 | CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, F.getContext()); |
624 | if (!determineAssignments(Assigner, Args, CCInfo)) |
625 | return false; |
626 | |
627 | return handleAssignments(Handler, Args, CCState&: CCInfo, ArgLocs, MIRBuilder, |
628 | ThisReturnRegs); |
629 | } |
630 | |
631 | static unsigned extendOpFromFlags(llvm::ISD::ArgFlagsTy Flags) { |
632 | if (Flags.isSExt()) |
633 | return TargetOpcode::G_SEXT; |
634 | if (Flags.isZExt()) |
635 | return TargetOpcode::G_ZEXT; |
636 | return TargetOpcode::G_ANYEXT; |
637 | } |
638 | |
639 | bool CallLowering::determineAssignments(ValueAssigner &Assigner, |
640 | SmallVectorImpl<ArgInfo> &Args, |
641 | CCState &CCInfo) const { |
642 | LLVMContext &Ctx = CCInfo.getContext(); |
643 | const CallingConv::ID CallConv = CCInfo.getCallingConv(); |
644 | |
645 | unsigned NumArgs = Args.size(); |
646 | for (unsigned i = 0; i != NumArgs; ++i) { |
647 | EVT CurVT = EVT::getEVT(Ty: Args[i].Ty); |
648 | |
649 | MVT NewVT = TLI->getRegisterTypeForCallingConv(Context&: Ctx, CC: CallConv, VT: CurVT); |
650 | |
651 | // If we need to split the type over multiple regs, check it's a scenario |
652 | // we currently support. |
653 | unsigned NumParts = |
654 | TLI->getNumRegistersForCallingConv(Context&: Ctx, CC: CallConv, VT: CurVT); |
655 | |
656 | if (NumParts == 1) { |
657 | // Try to use the register type if we couldn't assign the VT. |
658 | if (Assigner.assignArg(ValNo: i, OrigVT: CurVT, ValVT: NewVT, LocVT: NewVT, LocInfo: CCValAssign::Full, Info: Args[i], |
659 | Flags: Args[i].Flags[0], State&: CCInfo)) |
660 | return false; |
661 | continue; |
662 | } |
663 | |
664 | // For incoming arguments (physregs to vregs), we could have values in |
665 | // physregs (or memlocs) which we want to extract and copy to vregs. |
666 | // During this, we might have to deal with the LLT being split across |
667 | // multiple regs, so we have to record this information for later. |
668 | // |
669 | // If we have outgoing args, then we have the opposite case. We have a |
670 | // vreg with an LLT which we want to assign to a physical location, and |
671 | // we might have to record that the value has to be split later. |
672 | |
673 | // We're handling an incoming arg which is split over multiple regs. |
674 | // E.g. passing an s128 on AArch64. |
675 | ISD::ArgFlagsTy OrigFlags = Args[i].Flags[0]; |
676 | Args[i].Flags.clear(); |
677 | |
678 | for (unsigned Part = 0; Part < NumParts; ++Part) { |
679 | ISD::ArgFlagsTy Flags = OrigFlags; |
680 | if (Part == 0) { |
681 | Flags.setSplit(); |
682 | } else { |
683 | Flags.setOrigAlign(Align(1)); |
684 | if (Part == NumParts - 1) |
685 | Flags.setSplitEnd(); |
686 | } |
687 | |
688 | Args[i].Flags.push_back(Elt: Flags); |
689 | if (Assigner.assignArg(ValNo: i, OrigVT: CurVT, ValVT: NewVT, LocVT: NewVT, LocInfo: CCValAssign::Full, Info: Args[i], |
690 | Flags: Args[i].Flags[Part], State&: CCInfo)) { |
691 | // Still couldn't assign this smaller part type for some reason. |
692 | return false; |
693 | } |
694 | } |
695 | } |
696 | |
697 | return true; |
698 | } |
699 | |
700 | bool CallLowering::handleAssignments(ValueHandler &Handler, |
701 | SmallVectorImpl<ArgInfo> &Args, |
702 | CCState &CCInfo, |
703 | SmallVectorImpl<CCValAssign> &ArgLocs, |
704 | MachineIRBuilder &MIRBuilder, |
705 | ArrayRef<Register> ThisReturnRegs) const { |
706 | MachineFunction &MF = MIRBuilder.getMF(); |
707 | MachineRegisterInfo &MRI = MF.getRegInfo(); |
708 | const Function &F = MF.getFunction(); |
709 | const DataLayout &DL = F.getParent()->getDataLayout(); |
710 | |
711 | const unsigned NumArgs = Args.size(); |
712 | |
713 | // Stores thunks for outgoing register assignments. This is used so we delay |
714 | // generating register copies until mem loc assignments are done. We do this |
715 | // so that if the target is using the delayed stack protector feature, we can |
716 | // find the split point of the block accurately. E.g. if we have: |
717 | // G_STORE %val, %memloc |
718 | // $x0 = COPY %foo |
719 | // $x1 = COPY %bar |
720 | // CALL func |
721 | // ... then the split point for the block will correctly be at, and including, |
722 | // the copy to $x0. If instead the G_STORE instruction immediately precedes |
723 | // the CALL, then we'd prematurely choose the CALL as the split point, thus |
724 | // generating a split block with a CALL that uses undefined physregs. |
725 | SmallVector<std::function<void()>> DelayedOutgoingRegAssignments; |
726 | |
727 | for (unsigned i = 0, j = 0; i != NumArgs; ++i, ++j) { |
728 | assert(j < ArgLocs.size() && "Skipped too many arg locs" ); |
729 | CCValAssign &VA = ArgLocs[j]; |
730 | assert(VA.getValNo() == i && "Location doesn't correspond to current arg" ); |
731 | |
732 | if (VA.needsCustom()) { |
733 | std::function<void()> Thunk; |
734 | unsigned NumArgRegs = Handler.assignCustomValue( |
735 | Arg&: Args[i], VAs: ArrayRef(ArgLocs).slice(N: j), Thunk: &Thunk); |
736 | if (Thunk) |
737 | DelayedOutgoingRegAssignments.emplace_back(Args&: Thunk); |
738 | if (!NumArgRegs) |
739 | return false; |
740 | j += (NumArgRegs - 1); |
741 | continue; |
742 | } |
743 | |
744 | const MVT ValVT = VA.getValVT(); |
745 | const MVT LocVT = VA.getLocVT(); |
746 | |
747 | const LLT LocTy(LocVT); |
748 | const LLT ValTy(ValVT); |
749 | const LLT NewLLT = Handler.isIncomingArgumentHandler() ? LocTy : ValTy; |
750 | const EVT OrigVT = EVT::getEVT(Ty: Args[i].Ty); |
751 | const LLT OrigTy = getLLTForType(Ty&: *Args[i].Ty, DL); |
752 | |
753 | // Expected to be multiple regs for a single incoming arg. |
754 | // There should be Regs.size() ArgLocs per argument. |
755 | // This should be the same as getNumRegistersForCallingConv |
756 | const unsigned NumParts = Args[i].Flags.size(); |
757 | |
758 | // Now split the registers into the assigned types. |
759 | Args[i].OrigRegs.assign(in_start: Args[i].Regs.begin(), in_end: Args[i].Regs.end()); |
760 | |
761 | if (NumParts != 1 || NewLLT != OrigTy) { |
762 | // If we can't directly assign the register, we need one or more |
763 | // intermediate values. |
764 | Args[i].Regs.resize(N: NumParts); |
765 | |
766 | // For each split register, create and assign a vreg that will store |
767 | // the incoming component of the larger value. These will later be |
768 | // merged to form the final vreg. |
769 | for (unsigned Part = 0; Part < NumParts; ++Part) |
770 | Args[i].Regs[Part] = MRI.createGenericVirtualRegister(Ty: NewLLT); |
771 | } |
772 | |
773 | assert((j + (NumParts - 1)) < ArgLocs.size() && |
774 | "Too many regs for number of args" ); |
775 | |
776 | // Coerce into outgoing value types before register assignment. |
777 | if (!Handler.isIncomingArgumentHandler() && OrigTy != ValTy) { |
778 | assert(Args[i].OrigRegs.size() == 1); |
779 | buildCopyToRegs(B&: MIRBuilder, DstRegs: Args[i].Regs, SrcReg: Args[i].OrigRegs[0], SrcTy: OrigTy, |
780 | PartTy: ValTy, ExtendOp: extendOpFromFlags(Flags: Args[i].Flags[0])); |
781 | } |
782 | |
783 | bool BigEndianPartOrdering = TLI->hasBigEndianPartOrdering(VT: OrigVT, DL); |
784 | for (unsigned Part = 0; Part < NumParts; ++Part) { |
785 | Register ArgReg = Args[i].Regs[Part]; |
786 | // There should be Regs.size() ArgLocs per argument. |
787 | unsigned Idx = BigEndianPartOrdering ? NumParts - 1 - Part : Part; |
788 | CCValAssign &VA = ArgLocs[j + Idx]; |
789 | const ISD::ArgFlagsTy Flags = Args[i].Flags[Part]; |
790 | |
791 | if (VA.isMemLoc() && !Flags.isByVal()) { |
792 | // Individual pieces may have been spilled to the stack and others |
793 | // passed in registers. |
794 | |
795 | // TODO: The memory size may be larger than the value we need to |
796 | // store. We may need to adjust the offset for big endian targets. |
797 | LLT MemTy = Handler.getStackValueStoreType(DL, VA, Flags); |
798 | |
799 | MachinePointerInfo MPO; |
800 | Register StackAddr = Handler.getStackAddress( |
801 | MemSize: MemTy.getSizeInBytes(), Offset: VA.getLocMemOffset(), MPO, Flags); |
802 | |
803 | Handler.assignValueToAddress(Arg: Args[i], ValRegIndex: Part, Addr: StackAddr, MemTy, MPO, VA); |
804 | continue; |
805 | } |
806 | |
807 | if (VA.isMemLoc() && Flags.isByVal()) { |
808 | assert(Args[i].Regs.size() == 1 && |
809 | "didn't expect split byval pointer" ); |
810 | |
811 | if (Handler.isIncomingArgumentHandler()) { |
812 | // We just need to copy the frame index value to the pointer. |
813 | MachinePointerInfo MPO; |
814 | Register StackAddr = Handler.getStackAddress( |
815 | MemSize: Flags.getByValSize(), Offset: VA.getLocMemOffset(), MPO, Flags); |
816 | MIRBuilder.buildCopy(Res: Args[i].Regs[0], Op: StackAddr); |
817 | } else { |
818 | // For outgoing byval arguments, insert the implicit copy byval |
819 | // implies, such that writes in the callee do not modify the caller's |
820 | // value. |
821 | uint64_t MemSize = Flags.getByValSize(); |
822 | int64_t Offset = VA.getLocMemOffset(); |
823 | |
824 | MachinePointerInfo DstMPO; |
825 | Register StackAddr = |
826 | Handler.getStackAddress(MemSize, Offset, MPO&: DstMPO, Flags); |
827 | |
828 | MachinePointerInfo SrcMPO(Args[i].OrigValue); |
829 | if (!Args[i].OrigValue) { |
830 | // We still need to accurately track the stack address space if we |
831 | // don't know the underlying value. |
832 | const LLT PtrTy = MRI.getType(Reg: StackAddr); |
833 | SrcMPO = MachinePointerInfo(PtrTy.getAddressSpace()); |
834 | } |
835 | |
836 | Align DstAlign = std::max(a: Flags.getNonZeroByValAlign(), |
837 | b: inferAlignFromPtrInfo(MF, MPO: DstMPO)); |
838 | |
839 | Align SrcAlign = std::max(a: Flags.getNonZeroByValAlign(), |
840 | b: inferAlignFromPtrInfo(MF, MPO: SrcMPO)); |
841 | |
842 | Handler.copyArgumentMemory(Arg: Args[i], DstPtr: StackAddr, SrcPtr: Args[i].Regs[0], |
843 | DstPtrInfo: DstMPO, DstAlign, SrcPtrInfo: SrcMPO, SrcAlign, |
844 | MemSize, VA); |
845 | } |
846 | continue; |
847 | } |
848 | |
849 | assert(!VA.needsCustom() && "custom loc should have been handled already" ); |
850 | |
851 | if (i == 0 && !ThisReturnRegs.empty() && |
852 | Handler.isIncomingArgumentHandler() && |
853 | isTypeIsValidForThisReturn(Ty: ValVT)) { |
854 | Handler.assignValueToReg(ValVReg: ArgReg, PhysReg: ThisReturnRegs[Part], VA); |
855 | continue; |
856 | } |
857 | |
858 | if (Handler.isIncomingArgumentHandler()) |
859 | Handler.assignValueToReg(ValVReg: ArgReg, PhysReg: VA.getLocReg(), VA); |
860 | else { |
861 | DelayedOutgoingRegAssignments.emplace_back(Args: [=, &Handler]() { |
862 | Handler.assignValueToReg(ValVReg: ArgReg, PhysReg: VA.getLocReg(), VA); |
863 | }); |
864 | } |
865 | } |
866 | |
867 | // Now that all pieces have been assigned, re-pack the register typed values |
868 | // into the original value typed registers. |
869 | if (Handler.isIncomingArgumentHandler() && OrigVT != LocVT) { |
870 | // Merge the split registers into the expected larger result vregs of |
871 | // the original call. |
872 | buildCopyFromRegs(B&: MIRBuilder, OrigRegs: Args[i].OrigRegs, Regs: Args[i].Regs, LLTy: OrigTy, |
873 | PartLLT: LocTy, Flags: Args[i].Flags[0]); |
874 | } |
875 | |
876 | j += NumParts - 1; |
877 | } |
878 | for (auto &Fn : DelayedOutgoingRegAssignments) |
879 | Fn(); |
880 | |
881 | return true; |
882 | } |
883 | |
884 | void CallLowering::insertSRetLoads(MachineIRBuilder &MIRBuilder, Type *RetTy, |
885 | ArrayRef<Register> VRegs, Register DemoteReg, |
886 | int FI) const { |
887 | MachineFunction &MF = MIRBuilder.getMF(); |
888 | MachineRegisterInfo &MRI = MF.getRegInfo(); |
889 | const DataLayout &DL = MF.getDataLayout(); |
890 | |
891 | SmallVector<EVT, 4> SplitVTs; |
892 | SmallVector<uint64_t, 4> Offsets; |
893 | ComputeValueVTs(TLI: *TLI, DL, Ty: RetTy, ValueVTs&: SplitVTs, FixedOffsets: &Offsets, StartingOffset: 0); |
894 | |
895 | assert(VRegs.size() == SplitVTs.size()); |
896 | |
897 | unsigned NumValues = SplitVTs.size(); |
898 | Align BaseAlign = DL.getPrefTypeAlign(Ty: RetTy); |
899 | Type *RetPtrTy = |
900 | PointerType::get(C&: RetTy->getContext(), AddressSpace: DL.getAllocaAddrSpace()); |
901 | LLT OffsetLLTy = getLLTForType(Ty&: *DL.getIndexType(PtrTy: RetPtrTy), DL); |
902 | |
903 | MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(MF, FI); |
904 | |
905 | for (unsigned I = 0; I < NumValues; ++I) { |
906 | Register Addr; |
907 | MIRBuilder.materializePtrAdd(Res&: Addr, Op0: DemoteReg, ValueTy: OffsetLLTy, Value: Offsets[I]); |
908 | auto *MMO = MF.getMachineMemOperand(PtrInfo, f: MachineMemOperand::MOLoad, |
909 | MemTy: MRI.getType(Reg: VRegs[I]), |
910 | base_alignment: commonAlignment(A: BaseAlign, Offset: Offsets[I])); |
911 | MIRBuilder.buildLoad(Res: VRegs[I], Addr, MMO&: *MMO); |
912 | } |
913 | } |
914 | |
915 | void CallLowering::insertSRetStores(MachineIRBuilder &MIRBuilder, Type *RetTy, |
916 | ArrayRef<Register> VRegs, |
917 | Register DemoteReg) const { |
918 | MachineFunction &MF = MIRBuilder.getMF(); |
919 | MachineRegisterInfo &MRI = MF.getRegInfo(); |
920 | const DataLayout &DL = MF.getDataLayout(); |
921 | |
922 | SmallVector<EVT, 4> SplitVTs; |
923 | SmallVector<uint64_t, 4> Offsets; |
924 | ComputeValueVTs(TLI: *TLI, DL, Ty: RetTy, ValueVTs&: SplitVTs, FixedOffsets: &Offsets, StartingOffset: 0); |
925 | |
926 | assert(VRegs.size() == SplitVTs.size()); |
927 | |
928 | unsigned NumValues = SplitVTs.size(); |
929 | Align BaseAlign = DL.getPrefTypeAlign(Ty: RetTy); |
930 | unsigned AS = DL.getAllocaAddrSpace(); |
931 | LLT OffsetLLTy = getLLTForType(Ty&: *DL.getIndexType(PtrTy: RetTy->getPointerTo(AddrSpace: AS)), DL); |
932 | |
933 | MachinePointerInfo PtrInfo(AS); |
934 | |
935 | for (unsigned I = 0; I < NumValues; ++I) { |
936 | Register Addr; |
937 | MIRBuilder.materializePtrAdd(Res&: Addr, Op0: DemoteReg, ValueTy: OffsetLLTy, Value: Offsets[I]); |
938 | auto *MMO = MF.getMachineMemOperand(PtrInfo, f: MachineMemOperand::MOStore, |
939 | MemTy: MRI.getType(Reg: VRegs[I]), |
940 | base_alignment: commonAlignment(A: BaseAlign, Offset: Offsets[I])); |
941 | MIRBuilder.buildStore(Val: VRegs[I], Addr, MMO&: *MMO); |
942 | } |
943 | } |
944 | |
945 | void CallLowering::insertSRetIncomingArgument( |
946 | const Function &F, SmallVectorImpl<ArgInfo> &SplitArgs, Register &DemoteReg, |
947 | MachineRegisterInfo &MRI, const DataLayout &DL) const { |
948 | unsigned AS = DL.getAllocaAddrSpace(); |
949 | DemoteReg = MRI.createGenericVirtualRegister( |
950 | Ty: LLT::pointer(AddressSpace: AS, SizeInBits: DL.getPointerSizeInBits(AS))); |
951 | |
952 | Type *PtrTy = PointerType::get(ElementType: F.getReturnType(), AddressSpace: AS); |
953 | |
954 | SmallVector<EVT, 1> ValueVTs; |
955 | ComputeValueVTs(TLI: *TLI, DL, Ty: PtrTy, ValueVTs); |
956 | |
957 | // NOTE: Assume that a pointer won't get split into more than one VT. |
958 | assert(ValueVTs.size() == 1); |
959 | |
960 | ArgInfo DemoteArg(DemoteReg, ValueVTs[0].getTypeForEVT(Context&: PtrTy->getContext()), |
961 | ArgInfo::NoArgIndex); |
962 | setArgFlags(Arg&: DemoteArg, OpIdx: AttributeList::ReturnIndex, DL, FuncInfo: F); |
963 | DemoteArg.Flags[0].setSRet(); |
964 | SplitArgs.insert(I: SplitArgs.begin(), Elt: DemoteArg); |
965 | } |
966 | |
967 | void CallLowering::insertSRetOutgoingArgument(MachineIRBuilder &MIRBuilder, |
968 | const CallBase &CB, |
969 | CallLoweringInfo &Info) const { |
970 | const DataLayout &DL = MIRBuilder.getDataLayout(); |
971 | Type *RetTy = CB.getType(); |
972 | unsigned AS = DL.getAllocaAddrSpace(); |
973 | LLT FramePtrTy = LLT::pointer(AddressSpace: AS, SizeInBits: DL.getPointerSizeInBits(AS)); |
974 | |
975 | int FI = MIRBuilder.getMF().getFrameInfo().CreateStackObject( |
976 | Size: DL.getTypeAllocSize(Ty: RetTy), Alignment: DL.getPrefTypeAlign(Ty: RetTy), isSpillSlot: false); |
977 | |
978 | Register DemoteReg = MIRBuilder.buildFrameIndex(Res: FramePtrTy, Idx: FI).getReg(Idx: 0); |
979 | ArgInfo DemoteArg(DemoteReg, PointerType::get(ElementType: RetTy, AddressSpace: AS), |
980 | ArgInfo::NoArgIndex); |
981 | setArgFlags(Arg&: DemoteArg, OpIdx: AttributeList::ReturnIndex, DL, FuncInfo: CB); |
982 | DemoteArg.Flags[0].setSRet(); |
983 | |
984 | Info.OrigArgs.insert(I: Info.OrigArgs.begin(), Elt: DemoteArg); |
985 | Info.DemoteStackIndex = FI; |
986 | Info.DemoteRegister = DemoteReg; |
987 | } |
988 | |
989 | bool CallLowering::checkReturn(CCState &CCInfo, |
990 | SmallVectorImpl<BaseArgInfo> &Outs, |
991 | CCAssignFn *Fn) const { |
992 | for (unsigned I = 0, E = Outs.size(); I < E; ++I) { |
993 | MVT VT = MVT::getVT(Ty: Outs[I].Ty); |
994 | if (Fn(I, VT, VT, CCValAssign::Full, Outs[I].Flags[0], CCInfo)) |
995 | return false; |
996 | } |
997 | return true; |
998 | } |
999 | |
1000 | void CallLowering::getReturnInfo(CallingConv::ID CallConv, Type *RetTy, |
1001 | AttributeList Attrs, |
1002 | SmallVectorImpl<BaseArgInfo> &Outs, |
1003 | const DataLayout &DL) const { |
1004 | LLVMContext &Context = RetTy->getContext(); |
1005 | ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy(); |
1006 | |
1007 | SmallVector<EVT, 4> SplitVTs; |
1008 | ComputeValueVTs(TLI: *TLI, DL, Ty: RetTy, ValueVTs&: SplitVTs); |
1009 | addArgFlagsFromAttributes(Flags, Attrs, OpIdx: AttributeList::ReturnIndex); |
1010 | |
1011 | for (EVT VT : SplitVTs) { |
1012 | unsigned NumParts = |
1013 | TLI->getNumRegistersForCallingConv(Context, CC: CallConv, VT); |
1014 | MVT RegVT = TLI->getRegisterTypeForCallingConv(Context, CC: CallConv, VT); |
1015 | Type *PartTy = EVT(RegVT).getTypeForEVT(Context); |
1016 | |
1017 | for (unsigned I = 0; I < NumParts; ++I) { |
1018 | Outs.emplace_back(Args&: PartTy, Args&: Flags); |
1019 | } |
1020 | } |
1021 | } |
1022 | |
1023 | bool CallLowering::checkReturnTypeForCallConv(MachineFunction &MF) const { |
1024 | const auto &F = MF.getFunction(); |
1025 | Type *ReturnType = F.getReturnType(); |
1026 | CallingConv::ID CallConv = F.getCallingConv(); |
1027 | |
1028 | SmallVector<BaseArgInfo, 4> SplitArgs; |
1029 | getReturnInfo(CallConv, RetTy: ReturnType, Attrs: F.getAttributes(), Outs&: SplitArgs, |
1030 | DL: MF.getDataLayout()); |
1031 | return canLowerReturn(MF, CallConv, Outs&: SplitArgs, IsVarArg: F.isVarArg()); |
1032 | } |
1033 | |
1034 | bool CallLowering::parametersInCSRMatch( |
1035 | const MachineRegisterInfo &MRI, const uint32_t *CallerPreservedMask, |
1036 | const SmallVectorImpl<CCValAssign> &OutLocs, |
1037 | const SmallVectorImpl<ArgInfo> &OutArgs) const { |
1038 | for (unsigned i = 0; i < OutLocs.size(); ++i) { |
1039 | const auto &ArgLoc = OutLocs[i]; |
1040 | // If it's not a register, it's fine. |
1041 | if (!ArgLoc.isRegLoc()) |
1042 | continue; |
1043 | |
1044 | MCRegister PhysReg = ArgLoc.getLocReg(); |
1045 | |
1046 | // Only look at callee-saved registers. |
1047 | if (MachineOperand::clobbersPhysReg(RegMask: CallerPreservedMask, PhysReg)) |
1048 | continue; |
1049 | |
1050 | LLVM_DEBUG( |
1051 | dbgs() |
1052 | << "... Call has an argument passed in a callee-saved register.\n" ); |
1053 | |
1054 | // Check if it was copied from. |
1055 | const ArgInfo &OutInfo = OutArgs[i]; |
1056 | |
1057 | if (OutInfo.Regs.size() > 1) { |
1058 | LLVM_DEBUG( |
1059 | dbgs() << "... Cannot handle arguments in multiple registers.\n" ); |
1060 | return false; |
1061 | } |
1062 | |
1063 | // Check if we copy the register, walking through copies from virtual |
1064 | // registers. Note that getDefIgnoringCopies does not ignore copies from |
1065 | // physical registers. |
1066 | MachineInstr *RegDef = getDefIgnoringCopies(Reg: OutInfo.Regs[0], MRI); |
1067 | if (!RegDef || RegDef->getOpcode() != TargetOpcode::COPY) { |
1068 | LLVM_DEBUG( |
1069 | dbgs() |
1070 | << "... Parameter was not copied into a VReg, cannot tail call.\n" ); |
1071 | return false; |
1072 | } |
1073 | |
1074 | // Got a copy. Verify that it's the same as the register we want. |
1075 | Register CopyRHS = RegDef->getOperand(i: 1).getReg(); |
1076 | if (CopyRHS != PhysReg) { |
1077 | LLVM_DEBUG(dbgs() << "... Callee-saved register was not copied into " |
1078 | "VReg, cannot tail call.\n" ); |
1079 | return false; |
1080 | } |
1081 | } |
1082 | |
1083 | return true; |
1084 | } |
1085 | |
1086 | bool CallLowering::resultsCompatible(CallLoweringInfo &Info, |
1087 | MachineFunction &MF, |
1088 | SmallVectorImpl<ArgInfo> &InArgs, |
1089 | ValueAssigner &CalleeAssigner, |
1090 | ValueAssigner &CallerAssigner) const { |
1091 | const Function &F = MF.getFunction(); |
1092 | CallingConv::ID CalleeCC = Info.CallConv; |
1093 | CallingConv::ID CallerCC = F.getCallingConv(); |
1094 | |
1095 | if (CallerCC == CalleeCC) |
1096 | return true; |
1097 | |
1098 | SmallVector<CCValAssign, 16> ArgLocs1; |
1099 | CCState CCInfo1(CalleeCC, Info.IsVarArg, MF, ArgLocs1, F.getContext()); |
1100 | if (!determineAssignments(Assigner&: CalleeAssigner, Args&: InArgs, CCInfo&: CCInfo1)) |
1101 | return false; |
1102 | |
1103 | SmallVector<CCValAssign, 16> ArgLocs2; |
1104 | CCState CCInfo2(CallerCC, F.isVarArg(), MF, ArgLocs2, F.getContext()); |
1105 | if (!determineAssignments(Assigner&: CallerAssigner, Args&: InArgs, CCInfo&: CCInfo2)) |
1106 | return false; |
1107 | |
1108 | // We need the argument locations to match up exactly. If there's more in |
1109 | // one than the other, then we are done. |
1110 | if (ArgLocs1.size() != ArgLocs2.size()) |
1111 | return false; |
1112 | |
1113 | // Make sure that each location is passed in exactly the same way. |
1114 | for (unsigned i = 0, e = ArgLocs1.size(); i < e; ++i) { |
1115 | const CCValAssign &Loc1 = ArgLocs1[i]; |
1116 | const CCValAssign &Loc2 = ArgLocs2[i]; |
1117 | |
1118 | // We need both of them to be the same. So if one is a register and one |
1119 | // isn't, we're done. |
1120 | if (Loc1.isRegLoc() != Loc2.isRegLoc()) |
1121 | return false; |
1122 | |
1123 | if (Loc1.isRegLoc()) { |
1124 | // If they don't have the same register location, we're done. |
1125 | if (Loc1.getLocReg() != Loc2.getLocReg()) |
1126 | return false; |
1127 | |
1128 | // They matched, so we can move to the next ArgLoc. |
1129 | continue; |
1130 | } |
1131 | |
1132 | // Loc1 wasn't a RegLoc, so they both must be MemLocs. Check if they match. |
1133 | if (Loc1.getLocMemOffset() != Loc2.getLocMemOffset()) |
1134 | return false; |
1135 | } |
1136 | |
1137 | return true; |
1138 | } |
1139 | |
1140 | LLT CallLowering::ValueHandler::getStackValueStoreType( |
1141 | const DataLayout &DL, const CCValAssign &VA, ISD::ArgFlagsTy Flags) const { |
1142 | const MVT ValVT = VA.getValVT(); |
1143 | if (ValVT != MVT::iPTR) { |
1144 | LLT ValTy(ValVT); |
1145 | |
1146 | // We lost the pointeriness going through CCValAssign, so try to restore it |
1147 | // based on the flags. |
1148 | if (Flags.isPointer()) { |
1149 | LLT PtrTy = LLT::pointer(AddressSpace: Flags.getPointerAddrSpace(), |
1150 | SizeInBits: ValTy.getScalarSizeInBits()); |
1151 | if (ValVT.isVector()) |
1152 | return LLT::vector(EC: ValTy.getElementCount(), ScalarTy: PtrTy); |
1153 | return PtrTy; |
1154 | } |
1155 | |
1156 | return ValTy; |
1157 | } |
1158 | |
1159 | unsigned AddrSpace = Flags.getPointerAddrSpace(); |
1160 | return LLT::pointer(AddressSpace: AddrSpace, SizeInBits: DL.getPointerSize(AS: AddrSpace)); |
1161 | } |
1162 | |
1163 | void CallLowering::ValueHandler::copyArgumentMemory( |
1164 | const ArgInfo &Arg, Register DstPtr, Register SrcPtr, |
1165 | const MachinePointerInfo &DstPtrInfo, Align DstAlign, |
1166 | const MachinePointerInfo &SrcPtrInfo, Align SrcAlign, uint64_t MemSize, |
1167 | CCValAssign &VA) const { |
1168 | MachineFunction &MF = MIRBuilder.getMF(); |
1169 | MachineMemOperand *SrcMMO = MF.getMachineMemOperand( |
1170 | PtrInfo: SrcPtrInfo, |
1171 | F: MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable, Size: MemSize, |
1172 | BaseAlignment: SrcAlign); |
1173 | |
1174 | MachineMemOperand *DstMMO = MF.getMachineMemOperand( |
1175 | PtrInfo: DstPtrInfo, |
1176 | F: MachineMemOperand::MOStore | MachineMemOperand::MODereferenceable, |
1177 | Size: MemSize, BaseAlignment: DstAlign); |
1178 | |
1179 | const LLT PtrTy = MRI.getType(Reg: DstPtr); |
1180 | const LLT SizeTy = LLT::scalar(SizeInBits: PtrTy.getSizeInBits()); |
1181 | |
1182 | auto SizeConst = MIRBuilder.buildConstant(Res: SizeTy, Val: MemSize); |
1183 | MIRBuilder.buildMemCpy(DstPtr, SrcPtr, Size: SizeConst, DstMMO&: *DstMMO, SrcMMO&: *SrcMMO); |
1184 | } |
1185 | |
1186 | Register CallLowering::ValueHandler::extendRegister(Register ValReg, |
1187 | const CCValAssign &VA, |
1188 | unsigned MaxSizeBits) { |
1189 | LLT LocTy{VA.getLocVT()}; |
1190 | LLT ValTy{VA.getValVT()}; |
1191 | |
1192 | if (LocTy.getSizeInBits() == ValTy.getSizeInBits()) |
1193 | return ValReg; |
1194 | |
1195 | if (LocTy.isScalar() && MaxSizeBits && MaxSizeBits < LocTy.getSizeInBits()) { |
1196 | if (MaxSizeBits <= ValTy.getSizeInBits()) |
1197 | return ValReg; |
1198 | LocTy = LLT::scalar(SizeInBits: MaxSizeBits); |
1199 | } |
1200 | |
1201 | const LLT ValRegTy = MRI.getType(Reg: ValReg); |
1202 | if (ValRegTy.isPointer()) { |
1203 | // The x32 ABI wants to zero extend 32-bit pointers to 64-bit registers, so |
1204 | // we have to cast to do the extension. |
1205 | LLT IntPtrTy = LLT::scalar(SizeInBits: ValRegTy.getSizeInBits()); |
1206 | ValReg = MIRBuilder.buildPtrToInt(Dst: IntPtrTy, Src: ValReg).getReg(Idx: 0); |
1207 | } |
1208 | |
1209 | switch (VA.getLocInfo()) { |
1210 | default: break; |
1211 | case CCValAssign::Full: |
1212 | case CCValAssign::BCvt: |
1213 | // FIXME: bitconverting between vector types may or may not be a |
1214 | // nop in big-endian situations. |
1215 | return ValReg; |
1216 | case CCValAssign::AExt: { |
1217 | auto MIB = MIRBuilder.buildAnyExt(Res: LocTy, Op: ValReg); |
1218 | return MIB.getReg(Idx: 0); |
1219 | } |
1220 | case CCValAssign::SExt: { |
1221 | Register NewReg = MRI.createGenericVirtualRegister(Ty: LocTy); |
1222 | MIRBuilder.buildSExt(Res: NewReg, Op: ValReg); |
1223 | return NewReg; |
1224 | } |
1225 | case CCValAssign::ZExt: { |
1226 | Register NewReg = MRI.createGenericVirtualRegister(Ty: LocTy); |
1227 | MIRBuilder.buildZExt(Res: NewReg, Op: ValReg); |
1228 | return NewReg; |
1229 | } |
1230 | } |
1231 | llvm_unreachable("unable to extend register" ); |
1232 | } |
1233 | |
1234 | void CallLowering::ValueAssigner::anchor() {} |
1235 | |
1236 | Register CallLowering::IncomingValueHandler::buildExtensionHint( |
1237 | const CCValAssign &VA, Register SrcReg, LLT NarrowTy) { |
1238 | switch (VA.getLocInfo()) { |
1239 | case CCValAssign::LocInfo::ZExt: { |
1240 | return MIRBuilder |
1241 | .buildAssertZExt(Res: MRI.cloneVirtualRegister(VReg: SrcReg), Op: SrcReg, |
1242 | Size: NarrowTy.getScalarSizeInBits()) |
1243 | .getReg(Idx: 0); |
1244 | } |
1245 | case CCValAssign::LocInfo::SExt: { |
1246 | return MIRBuilder |
1247 | .buildAssertSExt(Res: MRI.cloneVirtualRegister(VReg: SrcReg), Op: SrcReg, |
1248 | Size: NarrowTy.getScalarSizeInBits()) |
1249 | .getReg(Idx: 0); |
1250 | break; |
1251 | } |
1252 | default: |
1253 | return SrcReg; |
1254 | } |
1255 | } |
1256 | |
1257 | /// Check if we can use a basic COPY instruction between the two types. |
1258 | /// |
1259 | /// We're currently building on top of the infrastructure using MVT, which loses |
1260 | /// pointer information in the CCValAssign. We accept copies from physical |
1261 | /// registers that have been reported as integers if it's to an equivalent sized |
1262 | /// pointer LLT. |
1263 | static bool isCopyCompatibleType(LLT SrcTy, LLT DstTy) { |
1264 | if (SrcTy == DstTy) |
1265 | return true; |
1266 | |
1267 | if (SrcTy.getSizeInBits() != DstTy.getSizeInBits()) |
1268 | return false; |
1269 | |
1270 | SrcTy = SrcTy.getScalarType(); |
1271 | DstTy = DstTy.getScalarType(); |
1272 | |
1273 | return (SrcTy.isPointer() && DstTy.isScalar()) || |
1274 | (DstTy.isPointer() && SrcTy.isScalar()); |
1275 | } |
1276 | |
1277 | void CallLowering::IncomingValueHandler::assignValueToReg( |
1278 | Register ValVReg, Register PhysReg, const CCValAssign &VA) { |
1279 | const MVT LocVT = VA.getLocVT(); |
1280 | const LLT LocTy(LocVT); |
1281 | const LLT RegTy = MRI.getType(Reg: ValVReg); |
1282 | |
1283 | if (isCopyCompatibleType(SrcTy: RegTy, DstTy: LocTy)) { |
1284 | MIRBuilder.buildCopy(Res: ValVReg, Op: PhysReg); |
1285 | return; |
1286 | } |
1287 | |
1288 | auto Copy = MIRBuilder.buildCopy(Res: LocTy, Op: PhysReg); |
1289 | auto Hint = buildExtensionHint(VA, SrcReg: Copy.getReg(Idx: 0), NarrowTy: RegTy); |
1290 | MIRBuilder.buildTrunc(Res: ValVReg, Op: Hint); |
1291 | } |
1292 | |