1//===- AArch64.cpp --------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "ABIInfoImpl.h"
10#include "TargetInfo.h"
11#include "clang/AST/Decl.h"
12#include "clang/Basic/DiagnosticFrontend.h"
13#include "llvm/TargetParser/AArch64TargetParser.h"
14
15using namespace clang;
16using namespace clang::CodeGen;
17
18//===----------------------------------------------------------------------===//
19// AArch64 ABI Implementation
20//===----------------------------------------------------------------------===//
21
22namespace {
23
24class AArch64ABIInfo : public ABIInfo {
25 AArch64ABIKind Kind;
26
27public:
28 AArch64ABIInfo(CodeGenTypes &CGT, AArch64ABIKind Kind)
29 : ABIInfo(CGT), Kind(Kind) {}
30
31 bool isSoftFloat() const { return Kind == AArch64ABIKind::AAPCSSoft; }
32
33private:
34 AArch64ABIKind getABIKind() const { return Kind; }
35 bool isDarwinPCS() const { return Kind == AArch64ABIKind::DarwinPCS; }
36
37 ABIArgInfo classifyReturnType(QualType RetTy, bool IsVariadicFn) const;
38 ABIArgInfo classifyArgumentType(QualType RetTy, bool IsVariadicFn,
39 bool IsNamedArg, unsigned CallingConvention,
40 unsigned &NSRN, unsigned &NPRN) const;
41 llvm::Type *convertFixedToScalableVectorType(const VectorType *VT) const;
42 ABIArgInfo coerceIllegalVector(QualType Ty, unsigned &NSRN,
43 unsigned &NPRN) const;
44 ABIArgInfo coerceAndExpandPureScalableAggregate(
45 QualType Ty, bool IsNamedArg, unsigned NVec, unsigned NPred,
46 const SmallVectorImpl<llvm::Type *> &UnpaddedCoerceToSeq, unsigned &NSRN,
47 unsigned &NPRN) const;
48 bool isHomogeneousAggregateBaseType(QualType Ty) const override;
49 bool isHomogeneousAggregateSmallEnough(const Type *Ty,
50 uint64_t Members) const override;
51 bool isZeroLengthBitfieldPermittedInHomogeneousAggregate() const override;
52
53 bool isIllegalVectorType(QualType Ty) const;
54
55 bool passAsAggregateType(QualType Ty) const;
56 bool passAsPureScalableType(QualType Ty, unsigned &NV, unsigned &NP,
57 SmallVectorImpl<llvm::Type *> &CoerceToSeq) const;
58
59 void flattenType(llvm::Type *Ty,
60 SmallVectorImpl<llvm::Type *> &Flattened) const;
61
62 void computeInfo(CGFunctionInfo &FI) const override {
63 if (!::classifyReturnType(CXXABI: getCXXABI(), FI, Info: *this))
64 FI.getReturnInfo() =
65 classifyReturnType(RetTy: FI.getReturnType(), IsVariadicFn: FI.isVariadic());
66
67 unsigned ArgNo = 0;
68 unsigned NSRN = 0, NPRN = 0;
69 for (auto &it : FI.arguments()) {
70 const bool IsNamedArg =
71 !FI.isVariadic() || ArgNo < FI.getRequiredArgs().getNumRequiredArgs();
72 ++ArgNo;
73 it.info = classifyArgumentType(RetTy: it.type, IsVariadicFn: FI.isVariadic(), IsNamedArg,
74 CallingConvention: FI.getCallingConvention(), NSRN, NPRN);
75 }
76 }
77
78 RValue EmitDarwinVAArg(Address VAListAddr, QualType Ty, CodeGenFunction &CGF,
79 AggValueSlot Slot) const;
80
81 RValue EmitAAPCSVAArg(Address VAListAddr, QualType Ty, CodeGenFunction &CGF,
82 AArch64ABIKind Kind, AggValueSlot Slot) const;
83
84 RValue EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, QualType Ty,
85 AggValueSlot Slot) const override {
86 llvm::Type *BaseTy = CGF.ConvertType(T: Ty);
87 if (isa<llvm::ScalableVectorType>(Val: BaseTy))
88 llvm::report_fatal_error(reason: "Passing SVE types to variadic functions is "
89 "currently not supported");
90
91 return Kind == AArch64ABIKind::Win64
92 ? EmitMSVAArg(CGF, VAListAddr, Ty, Slot)
93 : isDarwinPCS() ? EmitDarwinVAArg(VAListAddr, Ty, CGF, Slot)
94 : EmitAAPCSVAArg(VAListAddr, Ty, CGF, Kind, Slot);
95 }
96
97 RValue EmitMSVAArg(CodeGenFunction &CGF, Address VAListAddr, QualType Ty,
98 AggValueSlot Slot) const override;
99
100 bool allowBFloatArgsAndRet() const override {
101 return getTarget().hasBFloat16Type();
102 }
103
104 using ABIInfo::appendAttributeMangling;
105 void appendAttributeMangling(TargetClonesAttr *Attr, unsigned Index,
106 raw_ostream &Out) const override;
107 void appendAttributeMangling(StringRef AttrStr,
108 raw_ostream &Out) const override;
109};
110
111class AArch64SwiftABIInfo : public SwiftABIInfo {
112public:
113 explicit AArch64SwiftABIInfo(CodeGenTypes &CGT)
114 : SwiftABIInfo(CGT, /*SwiftErrorInRegister=*/true) {}
115
116 bool isLegalVectorType(CharUnits VectorSize, llvm::Type *EltTy,
117 unsigned NumElts) const override;
118};
119
120class AArch64TargetCodeGenInfo : public TargetCodeGenInfo {
121public:
122 AArch64TargetCodeGenInfo(CodeGenTypes &CGT, AArch64ABIKind Kind)
123 : TargetCodeGenInfo(std::make_unique<AArch64ABIInfo>(args&: CGT, args&: Kind)) {
124 SwiftInfo = std::make_unique<AArch64SwiftABIInfo>(args&: CGT);
125 }
126
127 StringRef getARCRetainAutoreleasedReturnValueMarker() const override {
128 return "mov\tfp, fp\t\t// marker for objc_retainAutoreleaseReturnValue";
129 }
130
131 int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const override {
132 return 31;
133 }
134
135 bool doesReturnSlotInterfereWithArgs() const override { return false; }
136
137 void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
138 CodeGen::CodeGenModule &CGM) const override {
139 auto *Fn = dyn_cast<llvm::Function>(Val: GV);
140 if (!Fn)
141 return;
142
143 const auto *FD = dyn_cast_or_null<FunctionDecl>(Val: D);
144 TargetInfo::BranchProtectionInfo BPI(CGM.getLangOpts());
145
146 if (FD && FD->hasAttr<TargetAttr>()) {
147 const auto *TA = FD->getAttr<TargetAttr>();
148 ParsedTargetAttr Attr =
149 CGM.getTarget().parseTargetAttr(Str: TA->getFeaturesStr());
150 if (!Attr.BranchProtection.empty()) {
151 StringRef Error;
152 (void)CGM.getTarget().validateBranchProtection(
153 Spec: Attr.BranchProtection, Arch: Attr.CPU, BPI, LO: CGM.getLangOpts(), Err&: Error);
154 assert(Error.empty());
155 }
156 }
157 setBranchProtectionFnAttributes(BPI, F&: *Fn);
158 setPointerAuthFnAttributes(Opts: CGM.getCodeGenOpts().PointerAuth, F&: *Fn);
159 }
160
161 bool isScalarizableAsmOperand(CodeGen::CodeGenFunction &CGF,
162 llvm::Type *Ty) const override {
163 if (CGF.getTarget().hasFeature(Feature: "ls64")) {
164 auto *ST = dyn_cast<llvm::StructType>(Val: Ty);
165 if (ST && ST->getNumElements() == 1) {
166 auto *AT = dyn_cast<llvm::ArrayType>(Val: ST->getElementType(N: 0));
167 if (AT && AT->getNumElements() == 8 &&
168 AT->getElementType()->isIntegerTy(Bitwidth: 64))
169 return true;
170 }
171 }
172 return TargetCodeGenInfo::isScalarizableAsmOperand(CGF, Ty);
173 }
174
175 void checkFunctionABI(CodeGenModule &CGM,
176 const FunctionDecl *Decl) const override;
177
178 void checkFunctionCallABI(CodeGenModule &CGM, SourceLocation CallLoc,
179 const FunctionDecl *Caller,
180 const FunctionDecl *Callee, const CallArgList &Args,
181 QualType ReturnType) const override;
182
183 bool wouldInliningViolateFunctionCallABI(
184 const FunctionDecl *Caller, const FunctionDecl *Callee) const override;
185
186private:
187 // Diagnose calls between functions with incompatible Streaming SVE
188 // attributes.
189 void checkFunctionCallABIStreaming(CodeGenModule &CGM, SourceLocation CallLoc,
190 const FunctionDecl *Caller,
191 const FunctionDecl *Callee) const;
192 // Diagnose calls which must pass arguments in floating-point registers when
193 // the selected target does not have floating-point registers.
194 void checkFunctionCallABISoftFloat(CodeGenModule &CGM, SourceLocation CallLoc,
195 const FunctionDecl *Caller,
196 const FunctionDecl *Callee,
197 const CallArgList &Args,
198 QualType ReturnType) const;
199};
200
201class WindowsAArch64TargetCodeGenInfo : public AArch64TargetCodeGenInfo {
202public:
203 WindowsAArch64TargetCodeGenInfo(CodeGenTypes &CGT, AArch64ABIKind K)
204 : AArch64TargetCodeGenInfo(CGT, K) {}
205
206 void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
207 CodeGen::CodeGenModule &CGM) const override;
208
209 void getDependentLibraryOption(llvm::StringRef Lib,
210 llvm::SmallString<24> &Opt) const override {
211 Opt = "/DEFAULTLIB:" + qualifyWindowsLibrary(Lib);
212 }
213
214 void getDetectMismatchOption(llvm::StringRef Name, llvm::StringRef Value,
215 llvm::SmallString<32> &Opt) const override {
216 Opt = "/FAILIFMISMATCH:\"" + Name.str() + "=" + Value.str() + "\"";
217 }
218};
219
220void WindowsAArch64TargetCodeGenInfo::setTargetAttributes(
221 const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &CGM) const {
222 AArch64TargetCodeGenInfo::setTargetAttributes(D, GV, CGM);
223 if (GV->isDeclaration())
224 return;
225 addStackProbeTargetAttributes(D, GV, CGM);
226}
227}
228
229llvm::Type *
230AArch64ABIInfo::convertFixedToScalableVectorType(const VectorType *VT) const {
231 assert(VT->getElementType()->isBuiltinType() && "expected builtin type!");
232
233 if (VT->getVectorKind() == VectorKind::SveFixedLengthPredicate) {
234 assert(VT->getElementType()->castAs<BuiltinType>()->getKind() ==
235 BuiltinType::UChar &&
236 "unexpected builtin type for SVE predicate!");
237 return llvm::ScalableVectorType::get(ElementType: llvm::Type::getInt1Ty(C&: getVMContext()),
238 MinNumElts: 16);
239 }
240
241 if (VT->getVectorKind() == VectorKind::SveFixedLengthData) {
242 const auto *BT = VT->getElementType()->castAs<BuiltinType>();
243 switch (BT->getKind()) {
244 default:
245 llvm_unreachable("unexpected builtin type for SVE vector!");
246
247 case BuiltinType::SChar:
248 case BuiltinType::UChar:
249 case BuiltinType::MFloat8:
250 return llvm::ScalableVectorType::get(
251 ElementType: llvm::Type::getInt8Ty(C&: getVMContext()), MinNumElts: 16);
252
253 case BuiltinType::Short:
254 case BuiltinType::UShort:
255 return llvm::ScalableVectorType::get(
256 ElementType: llvm::Type::getInt16Ty(C&: getVMContext()), MinNumElts: 8);
257
258 case BuiltinType::Int:
259 case BuiltinType::UInt:
260 return llvm::ScalableVectorType::get(
261 ElementType: llvm::Type::getInt32Ty(C&: getVMContext()), MinNumElts: 4);
262
263 case BuiltinType::Long:
264 case BuiltinType::ULong:
265 return llvm::ScalableVectorType::get(
266 ElementType: llvm::Type::getInt64Ty(C&: getVMContext()), MinNumElts: 2);
267
268 case BuiltinType::Half:
269 return llvm::ScalableVectorType::get(
270 ElementType: llvm::Type::getHalfTy(C&: getVMContext()), MinNumElts: 8);
271
272 case BuiltinType::Float:
273 return llvm::ScalableVectorType::get(
274 ElementType: llvm::Type::getFloatTy(C&: getVMContext()), MinNumElts: 4);
275
276 case BuiltinType::Double:
277 return llvm::ScalableVectorType::get(
278 ElementType: llvm::Type::getDoubleTy(C&: getVMContext()), MinNumElts: 2);
279
280 case BuiltinType::BFloat16:
281 return llvm::ScalableVectorType::get(
282 ElementType: llvm::Type::getBFloatTy(C&: getVMContext()), MinNumElts: 8);
283 }
284 }
285
286 llvm_unreachable("expected fixed-length SVE vector");
287}
288
289ABIArgInfo AArch64ABIInfo::coerceIllegalVector(QualType Ty, unsigned &NSRN,
290 unsigned &NPRN) const {
291 assert(Ty->isVectorType() && "expected vector type!");
292
293 const auto *VT = Ty->castAs<VectorType>();
294 if (VT->getVectorKind() == VectorKind::SveFixedLengthPredicate) {
295 assert(VT->getElementType()->isBuiltinType() && "expected builtin type!");
296 assert(VT->getElementType()->castAs<BuiltinType>()->getKind() ==
297 BuiltinType::UChar &&
298 "unexpected builtin type for SVE predicate!");
299 NPRN = std::min(a: NPRN + 1, b: 4u);
300 return ABIArgInfo::getDirect(T: llvm::ScalableVectorType::get(
301 ElementType: llvm::Type::getInt1Ty(C&: getVMContext()), MinNumElts: 16));
302 }
303
304 if (VT->getVectorKind() == VectorKind::SveFixedLengthData) {
305 NSRN = std::min(a: NSRN + 1, b: 8u);
306 return ABIArgInfo::getDirect(T: convertFixedToScalableVectorType(VT));
307 }
308
309 uint64_t Size = getContext().getTypeSize(T: Ty);
310 // Android promotes <2 x i8> to i16, not i32
311 if ((isAndroid() || isOHOSFamily()) && (Size <= 16)) {
312 llvm::Type *ResType = llvm::Type::getInt16Ty(C&: getVMContext());
313 return ABIArgInfo::getDirect(T: ResType);
314 }
315 if (Size <= 32) {
316 llvm::Type *ResType = llvm::Type::getInt32Ty(C&: getVMContext());
317 return ABIArgInfo::getDirect(T: ResType);
318 }
319 if (Size == 64) {
320 NSRN = std::min(a: NSRN + 1, b: 8u);
321 auto *ResType =
322 llvm::FixedVectorType::get(ElementType: llvm::Type::getInt32Ty(C&: getVMContext()), NumElts: 2);
323 return ABIArgInfo::getDirect(T: ResType);
324 }
325 if (Size == 128) {
326 NSRN = std::min(a: NSRN + 1, b: 8u);
327 auto *ResType =
328 llvm::FixedVectorType::get(ElementType: llvm::Type::getInt32Ty(C&: getVMContext()), NumElts: 4);
329 return ABIArgInfo::getDirect(T: ResType);
330 }
331
332 return getNaturalAlignIndirect(Ty, AddrSpace: getDataLayout().getAllocaAddrSpace(),
333 /*ByVal=*/false);
334}
335
336ABIArgInfo AArch64ABIInfo::coerceAndExpandPureScalableAggregate(
337 QualType Ty, bool IsNamedArg, unsigned NVec, unsigned NPred,
338 const SmallVectorImpl<llvm::Type *> &UnpaddedCoerceToSeq, unsigned &NSRN,
339 unsigned &NPRN) const {
340 if (!IsNamedArg || NSRN + NVec > 8 || NPRN + NPred > 4)
341 return getNaturalAlignIndirect(Ty, AddrSpace: getDataLayout().getAllocaAddrSpace(),
342 /*ByVal=*/false);
343 NSRN += NVec;
344 NPRN += NPred;
345
346 // Handle SVE vector tuples.
347 if (Ty->isSVESizelessBuiltinType())
348 return ABIArgInfo::getDirect();
349
350 llvm::Type *UnpaddedCoerceToType =
351 UnpaddedCoerceToSeq.size() == 1
352 ? UnpaddedCoerceToSeq[0]
353 : llvm::StructType::get(Context&: CGT.getLLVMContext(), Elements: UnpaddedCoerceToSeq,
354 isPacked: true);
355
356 SmallVector<llvm::Type *> CoerceToSeq;
357 flattenType(Ty: CGT.ConvertType(T: Ty), Flattened&: CoerceToSeq);
358 auto *CoerceToType =
359 llvm::StructType::get(Context&: CGT.getLLVMContext(), Elements: CoerceToSeq, isPacked: false);
360
361 return ABIArgInfo::getCoerceAndExpand(coerceToType: CoerceToType, unpaddedCoerceToType: UnpaddedCoerceToType);
362}
363
364ABIArgInfo AArch64ABIInfo::classifyArgumentType(QualType Ty, bool IsVariadicFn,
365 bool IsNamedArg,
366 unsigned CallingConvention,
367 unsigned &NSRN,
368 unsigned &NPRN) const {
369 Ty = useFirstFieldIfTransparentUnion(Ty);
370
371 // Handle illegal vector types here.
372 if (isIllegalVectorType(Ty))
373 return coerceIllegalVector(Ty, NSRN, NPRN);
374
375 if (!passAsAggregateType(Ty)) {
376 // Treat an enum type as its underlying type.
377 if (const EnumType *EnumTy = Ty->getAs<EnumType>())
378 Ty = EnumTy->getDecl()->getIntegerType();
379
380 if (const auto *EIT = Ty->getAs<BitIntType>())
381 if (EIT->getNumBits() > 128)
382 return getNaturalAlignIndirect(Ty, AddrSpace: getDataLayout().getAllocaAddrSpace(),
383 ByVal: false);
384
385 if (Ty->isVectorType())
386 NSRN = std::min(a: NSRN + 1, b: 8u);
387 else if (const auto *BT = Ty->getAs<BuiltinType>()) {
388 if (BT->isFloatingPoint())
389 NSRN = std::min(a: NSRN + 1, b: 8u);
390 else {
391 switch (BT->getKind()) {
392 case BuiltinType::SveBool:
393 case BuiltinType::SveCount:
394 NPRN = std::min(a: NPRN + 1, b: 4u);
395 break;
396 case BuiltinType::SveBoolx2:
397 NPRN = std::min(a: NPRN + 2, b: 4u);
398 break;
399 case BuiltinType::SveBoolx4:
400 NPRN = std::min(a: NPRN + 4, b: 4u);
401 break;
402 default:
403 if (BT->isSVESizelessBuiltinType())
404 NSRN = std::min(
405 a: NSRN + getContext().getBuiltinVectorTypeInfo(VecTy: BT).NumVectors,
406 b: 8u);
407 }
408 }
409 }
410
411 return (isPromotableIntegerTypeForABI(Ty) && isDarwinPCS()
412 ? ABIArgInfo::getExtend(Ty, T: CGT.ConvertType(T: Ty))
413 : ABIArgInfo::getDirect());
414 }
415
416 // Structures with either a non-trivial destructor or a non-trivial
417 // copy constructor are always indirect.
418 if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(T: Ty, CXXABI&: getCXXABI())) {
419 return getNaturalAlignIndirect(
420 Ty, /*AddrSpace=*/getDataLayout().getAllocaAddrSpace(),
421 /*ByVal=*/RAA == CGCXXABI::RAA_DirectInMemory);
422 }
423
424 // Empty records:
425 uint64_t Size = getContext().getTypeSize(T: Ty);
426 bool IsEmpty = isEmptyRecord(Context&: getContext(), T: Ty, AllowArrays: true);
427 if (!Ty->isSVESizelessBuiltinType() && (IsEmpty || Size == 0)) {
428 // Empty records are ignored in C mode, and in C++ on Darwin.
429 if (!getContext().getLangOpts().CPlusPlus || isDarwinPCS())
430 return ABIArgInfo::getIgnore();
431
432 // In C++ mode, arguments which have sizeof() == 0 (which are non-standard
433 // C++) are ignored. This isn't defined by any standard, so we copy GCC's
434 // behaviour here.
435 if (Size == 0)
436 return ABIArgInfo::getIgnore();
437
438 // Otherwise, they are passed as if they have a size of 1 byte.
439 return ABIArgInfo::getDirect(T: llvm::Type::getInt8Ty(C&: getVMContext()));
440 }
441
442 // Homogeneous Floating-point Aggregates (HFAs) need to be expanded.
443 const Type *Base = nullptr;
444 uint64_t Members = 0;
445 bool IsWin64 = Kind == AArch64ABIKind::Win64 ||
446 CallingConvention == llvm::CallingConv::Win64;
447 bool IsWinVariadic = IsWin64 && IsVariadicFn;
448 // In variadic functions on Windows, all composite types are treated alike,
449 // no special handling of HFAs/HVAs.
450 if (!IsWinVariadic && isHomogeneousAggregate(Ty, Base, Members)) {
451 NSRN = std::min(a: NSRN + Members, b: uint64_t(8));
452 if (Kind != AArch64ABIKind::AAPCS)
453 return ABIArgInfo::getDirect(
454 T: llvm::ArrayType::get(ElementType: CGT.ConvertType(T: QualType(Base, 0)), NumElements: Members));
455
456 // For HFAs/HVAs, cap the argument alignment to 16, otherwise
457 // set it to 8 according to the AAPCS64 document.
458 unsigned Align =
459 getContext().getTypeUnadjustedAlignInChars(T: Ty).getQuantity();
460 Align = (Align >= 16) ? 16 : 8;
461 return ABIArgInfo::getDirect(
462 T: llvm::ArrayType::get(ElementType: CGT.ConvertType(T: QualType(Base, 0)), NumElements: Members), Offset: 0,
463 Padding: nullptr, CanBeFlattened: true, Align);
464 }
465
466 // In AAPCS named arguments of a Pure Scalable Type are passed expanded in
467 // registers, or indirectly if there are not enough registers.
468 if (Kind == AArch64ABIKind::AAPCS) {
469 unsigned NVec = 0, NPred = 0;
470 SmallVector<llvm::Type *> UnpaddedCoerceToSeq;
471 if (passAsPureScalableType(Ty, NV&: NVec, NP&: NPred, CoerceToSeq&: UnpaddedCoerceToSeq) &&
472 (NVec + NPred) > 0)
473 return coerceAndExpandPureScalableAggregate(
474 Ty, IsNamedArg, NVec, NPred, UnpaddedCoerceToSeq, NSRN, NPRN);
475 }
476
477 // Aggregates <= 16 bytes are passed directly in registers or on the stack.
478 if (Size <= 128) {
479 unsigned Alignment;
480 if (Kind == AArch64ABIKind::AAPCS) {
481 Alignment = getContext().getTypeUnadjustedAlign(T: Ty);
482 Alignment = Alignment < 128 ? 64 : 128;
483 } else {
484 Alignment =
485 std::max(a: getContext().getTypeAlign(T: Ty),
486 b: (unsigned)getTarget().getPointerWidth(AddrSpace: LangAS::Default));
487 }
488 Size = llvm::alignTo(Value: Size, Align: Alignment);
489
490 // If the Aggregate is made up of pointers, use an array of pointers for the
491 // coerced type. This prevents having to convert ptr2int->int2ptr through
492 // the call, allowing alias analysis to produce better code.
493 auto ContainsOnlyPointers = [&](const auto &Self, QualType Ty) {
494 if (isEmptyRecord(Context&: getContext(), T: Ty, AllowArrays: true))
495 return false;
496 const RecordType *RT = Ty->getAs<RecordType>();
497 if (!RT)
498 return false;
499 const RecordDecl *RD = RT->getDecl();
500 if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(Val: RD)) {
501 for (const auto &I : CXXRD->bases())
502 if (!Self(Self, I.getType()))
503 return false;
504 }
505 return all_of(RD->fields(), [&](FieldDecl *FD) {
506 QualType FDTy = FD->getType();
507 if (FDTy->isArrayType())
508 FDTy = getContext().getBaseElementType(QT: FDTy);
509 return (FDTy->isPointerOrReferenceType() &&
510 getContext().getTypeSize(T: FDTy) == 64) ||
511 Self(Self, FDTy);
512 });
513 };
514
515 // We use a pair of i64 for 16-byte aggregate with 8-byte alignment.
516 // For aggregates with 16-byte alignment, we use i128.
517 llvm::Type *BaseTy = llvm::Type::getIntNTy(C&: getVMContext(), N: Alignment);
518 if ((Size == 64 || Size == 128) && Alignment == 64 &&
519 ContainsOnlyPointers(ContainsOnlyPointers, Ty))
520 BaseTy = llvm::PointerType::getUnqual(C&: getVMContext());
521 return ABIArgInfo::getDirect(
522 T: Size == Alignment ? BaseTy
523 : llvm::ArrayType::get(ElementType: BaseTy, NumElements: Size / Alignment));
524 }
525
526 return getNaturalAlignIndirect(Ty, AddrSpace: getDataLayout().getAllocaAddrSpace(),
527 /*ByVal=*/false);
528}
529
530ABIArgInfo AArch64ABIInfo::classifyReturnType(QualType RetTy,
531 bool IsVariadicFn) const {
532 if (RetTy->isVoidType())
533 return ABIArgInfo::getIgnore();
534
535 if (const auto *VT = RetTy->getAs<VectorType>()) {
536 if (VT->getVectorKind() == VectorKind::SveFixedLengthData ||
537 VT->getVectorKind() == VectorKind::SveFixedLengthPredicate) {
538 unsigned NSRN = 0, NPRN = 0;
539 return coerceIllegalVector(Ty: RetTy, NSRN, NPRN);
540 }
541 }
542
543 // Large vector types should be returned via memory.
544 if (RetTy->isVectorType() && getContext().getTypeSize(T: RetTy) > 128)
545 return getNaturalAlignIndirect(Ty: RetTy, AddrSpace: getDataLayout().getAllocaAddrSpace());
546
547 if (!passAsAggregateType(Ty: RetTy)) {
548 // Treat an enum type as its underlying type.
549 if (const EnumType *EnumTy = RetTy->getAs<EnumType>())
550 RetTy = EnumTy->getDecl()->getIntegerType();
551
552 if (const auto *EIT = RetTy->getAs<BitIntType>())
553 if (EIT->getNumBits() > 128)
554 return getNaturalAlignIndirect(Ty: RetTy,
555 AddrSpace: getDataLayout().getAllocaAddrSpace());
556
557 return (isPromotableIntegerTypeForABI(Ty: RetTy) && isDarwinPCS()
558 ? ABIArgInfo::getExtend(Ty: RetTy)
559 : ABIArgInfo::getDirect());
560 }
561
562 uint64_t Size = getContext().getTypeSize(T: RetTy);
563 if (!RetTy->isSVESizelessBuiltinType() &&
564 (isEmptyRecord(Context&: getContext(), T: RetTy, AllowArrays: true) || Size == 0))
565 return ABIArgInfo::getIgnore();
566
567 const Type *Base = nullptr;
568 uint64_t Members = 0;
569 if (isHomogeneousAggregate(Ty: RetTy, Base, Members) &&
570 !(getTarget().getTriple().getArch() == llvm::Triple::aarch64_32 &&
571 IsVariadicFn))
572 // Homogeneous Floating-point Aggregates (HFAs) are returned directly.
573 return ABIArgInfo::getDirect();
574
575 // In AAPCS return values of a Pure Scalable type are treated as a single
576 // named argument and passed expanded in registers, or indirectly if there are
577 // not enough registers.
578 if (Kind == AArch64ABIKind::AAPCS) {
579 unsigned NSRN = 0, NPRN = 0;
580 unsigned NVec = 0, NPred = 0;
581 SmallVector<llvm::Type *> UnpaddedCoerceToSeq;
582 if (passAsPureScalableType(Ty: RetTy, NV&: NVec, NP&: NPred, CoerceToSeq&: UnpaddedCoerceToSeq) &&
583 (NVec + NPred) > 0)
584 return coerceAndExpandPureScalableAggregate(
585 Ty: RetTy, /* IsNamedArg */ true, NVec, NPred, UnpaddedCoerceToSeq, NSRN,
586 NPRN);
587 }
588
589 // Aggregates <= 16 bytes are returned directly in registers or on the stack.
590 if (Size <= 128) {
591 if (Size <= 64 && getDataLayout().isLittleEndian()) {
592 // Composite types are returned in lower bits of a 64-bit register for LE,
593 // and in higher bits for BE. However, integer types are always returned
594 // in lower bits for both LE and BE, and they are not rounded up to
595 // 64-bits. We can skip rounding up of composite types for LE, but not for
596 // BE, otherwise composite types will be indistinguishable from integer
597 // types.
598 return ABIArgInfo::getDirect(
599 T: llvm::IntegerType::get(C&: getVMContext(), NumBits: Size));
600 }
601
602 unsigned Alignment = getContext().getTypeAlign(T: RetTy);
603 Size = llvm::alignTo(Value: Size, Align: 64); // round up to multiple of 8 bytes
604
605 // We use a pair of i64 for 16-byte aggregate with 8-byte alignment.
606 // For aggregates with 16-byte alignment, we use i128.
607 if (Alignment < 128 && Size == 128) {
608 llvm::Type *BaseTy = llvm::Type::getInt64Ty(C&: getVMContext());
609 return ABIArgInfo::getDirect(T: llvm::ArrayType::get(ElementType: BaseTy, NumElements: Size / 64));
610 }
611 return ABIArgInfo::getDirect(T: llvm::IntegerType::get(C&: getVMContext(), NumBits: Size));
612 }
613
614 return getNaturalAlignIndirect(Ty: RetTy, AddrSpace: getDataLayout().getAllocaAddrSpace());
615}
616
617/// isIllegalVectorType - check whether the vector type is legal for AArch64.
618bool AArch64ABIInfo::isIllegalVectorType(QualType Ty) const {
619 if (const VectorType *VT = Ty->getAs<VectorType>()) {
620 // Check whether VT is a fixed-length SVE vector. These types are
621 // represented as scalable vectors in function args/return and must be
622 // coerced from fixed vectors.
623 if (VT->getVectorKind() == VectorKind::SveFixedLengthData ||
624 VT->getVectorKind() == VectorKind::SveFixedLengthPredicate)
625 return true;
626
627 // Check whether VT is legal.
628 unsigned NumElements = VT->getNumElements();
629 uint64_t Size = getContext().getTypeSize(VT);
630 // NumElements should be power of 2.
631 if (!llvm::isPowerOf2_32(Value: NumElements))
632 return true;
633
634 // arm64_32 has to be compatible with the ARM logic here, which allows huge
635 // vectors for some reason.
636 llvm::Triple Triple = getTarget().getTriple();
637 if (Triple.getArch() == llvm::Triple::aarch64_32 &&
638 Triple.isOSBinFormatMachO())
639 return Size <= 32;
640
641 return Size != 64 && (Size != 128 || NumElements == 1);
642 }
643 return false;
644}
645
646bool AArch64SwiftABIInfo::isLegalVectorType(CharUnits VectorSize,
647 llvm::Type *EltTy,
648 unsigned NumElts) const {
649 if (!llvm::isPowerOf2_32(Value: NumElts))
650 return false;
651 if (VectorSize.getQuantity() != 8 &&
652 (VectorSize.getQuantity() != 16 || NumElts == 1))
653 return false;
654 return true;
655}
656
657bool AArch64ABIInfo::isHomogeneousAggregateBaseType(QualType Ty) const {
658 // For the soft-float ABI variant, no types are considered to be homogeneous
659 // aggregates.
660 if (isSoftFloat())
661 return false;
662
663 // Homogeneous aggregates for AAPCS64 must have base types of a floating
664 // point type or a short-vector type. This is the same as the 32-bit ABI,
665 // but with the difference that any floating-point type is allowed,
666 // including __fp16.
667 if (const BuiltinType *BT = Ty->getAs<BuiltinType>()) {
668 if (BT->isFloatingPoint())
669 return true;
670 } else if (const VectorType *VT = Ty->getAs<VectorType>()) {
671 if (auto Kind = VT->getVectorKind();
672 Kind == VectorKind::SveFixedLengthData ||
673 Kind == VectorKind::SveFixedLengthPredicate)
674 return false;
675
676 unsigned VecSize = getContext().getTypeSize(VT);
677 if (VecSize == 64 || VecSize == 128)
678 return true;
679 }
680 return false;
681}
682
683bool AArch64ABIInfo::isHomogeneousAggregateSmallEnough(const Type *Base,
684 uint64_t Members) const {
685 return Members <= 4;
686}
687
688bool AArch64ABIInfo::isZeroLengthBitfieldPermittedInHomogeneousAggregate()
689 const {
690 // AAPCS64 says that the rule for whether something is a homogeneous
691 // aggregate is applied to the output of the data layout decision. So
692 // anything that doesn't affect the data layout also does not affect
693 // homogeneity. In particular, zero-length bitfields don't stop a struct
694 // being homogeneous.
695 return true;
696}
697
698bool AArch64ABIInfo::passAsAggregateType(QualType Ty) const {
699 if (Kind == AArch64ABIKind::AAPCS && Ty->isSVESizelessBuiltinType()) {
700 const auto *BT = Ty->castAs<BuiltinType>();
701 return !BT->isSVECount() &&
702 getContext().getBuiltinVectorTypeInfo(VecTy: BT).NumVectors > 1;
703 }
704 return isAggregateTypeForABI(T: Ty);
705}
706
707// Check if a type needs to be passed in registers as a Pure Scalable Type (as
708// defined by AAPCS64). Return the number of data vectors and the number of
709// predicate vectors in the type, into `NVec` and `NPred`, respectively. Upon
710// return `CoerceToSeq` contains an expanded sequence of LLVM IR types, one
711// element for each non-composite member. For practical purposes, limit the
712// length of `CoerceToSeq` to about 12 (the maximum that could possibly fit
713// in registers) and return false, the effect of which will be to pass the
714// argument under the rules for a large (> 128 bytes) composite.
715bool AArch64ABIInfo::passAsPureScalableType(
716 QualType Ty, unsigned &NVec, unsigned &NPred,
717 SmallVectorImpl<llvm::Type *> &CoerceToSeq) const {
718 if (const ConstantArrayType *AT = getContext().getAsConstantArrayType(T: Ty)) {
719 uint64_t NElt = AT->getZExtSize();
720 if (NElt == 0)
721 return false;
722
723 unsigned NV = 0, NP = 0;
724 SmallVector<llvm::Type *> EltCoerceToSeq;
725 if (!passAsPureScalableType(Ty: AT->getElementType(), NVec&: NV, NPred&: NP, CoerceToSeq&: EltCoerceToSeq))
726 return false;
727
728 if (CoerceToSeq.size() + NElt * EltCoerceToSeq.size() > 12)
729 return false;
730
731 for (uint64_t I = 0; I < NElt; ++I)
732 llvm::append_range(C&: CoerceToSeq, R&: EltCoerceToSeq);
733
734 NVec += NElt * NV;
735 NPred += NElt * NP;
736 return true;
737 }
738
739 if (const RecordType *RT = Ty->getAs<RecordType>()) {
740 // If the record cannot be passed in registers, then it's not a PST.
741 if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(RT, CXXABI&: getCXXABI());
742 RAA != CGCXXABI::RAA_Default)
743 return false;
744
745 // Pure scalable types are never unions and never contain unions.
746 const RecordDecl *RD = RT->getDecl();
747 if (RD->isUnion())
748 return false;
749
750 // If this is a C++ record, check the bases.
751 if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(Val: RD)) {
752 for (const auto &I : CXXRD->bases()) {
753 if (isEmptyRecord(Context&: getContext(), T: I.getType(), AllowArrays: true))
754 continue;
755 if (!passAsPureScalableType(Ty: I.getType(), NVec, NPred, CoerceToSeq))
756 return false;
757 }
758 }
759
760 // Check members.
761 for (const auto *FD : RD->fields()) {
762 QualType FT = FD->getType();
763 if (isEmptyField(Context&: getContext(), FD, /* AllowArrays */ true))
764 continue;
765 if (!passAsPureScalableType(Ty: FT, NVec, NPred, CoerceToSeq))
766 return false;
767 }
768
769 return true;
770 }
771
772 if (const auto *VT = Ty->getAs<VectorType>()) {
773 if (VT->getVectorKind() == VectorKind::SveFixedLengthPredicate) {
774 ++NPred;
775 if (CoerceToSeq.size() + 1 > 12)
776 return false;
777 CoerceToSeq.push_back(Elt: convertFixedToScalableVectorType(VT));
778 return true;
779 }
780
781 if (VT->getVectorKind() == VectorKind::SveFixedLengthData) {
782 ++NVec;
783 if (CoerceToSeq.size() + 1 > 12)
784 return false;
785 CoerceToSeq.push_back(Elt: convertFixedToScalableVectorType(VT));
786 return true;
787 }
788
789 return false;
790 }
791
792 if (!Ty->isBuiltinType())
793 return false;
794
795 bool isPredicate;
796 switch (Ty->castAs<BuiltinType>()->getKind()) {
797#define SVE_VECTOR_TYPE(Name, MangledName, Id, SingletonId) \
798 case BuiltinType::Id: \
799 isPredicate = false; \
800 break;
801#define SVE_PREDICATE_TYPE(Name, MangledName, Id, SingletonId) \
802 case BuiltinType::Id: \
803 isPredicate = true; \
804 break;
805#include "clang/Basic/AArch64ACLETypes.def"
806 default:
807 return false;
808 }
809
810 ASTContext::BuiltinVectorTypeInfo Info =
811 getContext().getBuiltinVectorTypeInfo(VecTy: cast<BuiltinType>(Val&: Ty));
812 assert(Info.NumVectors > 0 && Info.NumVectors <= 4 &&
813 "Expected 1, 2, 3 or 4 vectors!");
814 if (isPredicate)
815 NPred += Info.NumVectors;
816 else
817 NVec += Info.NumVectors;
818 llvm::Type *EltTy = Info.ElementType->isMFloat8Type()
819 ? llvm::Type::getInt8Ty(C&: getVMContext())
820 : CGT.ConvertType(T: Info.ElementType);
821 auto *VTy = llvm::ScalableVectorType::get(ElementType: EltTy, MinNumElts: Info.EC.getKnownMinValue());
822
823 if (CoerceToSeq.size() + Info.NumVectors > 12)
824 return false;
825 std::fill_n(std::back_inserter(x&: CoerceToSeq), Info.NumVectors, VTy);
826
827 return true;
828}
829
830// Expand an LLVM IR type into a sequence with a element for each non-struct,
831// non-array member of the type, with the exception of the padding types, which
832// are retained.
833void AArch64ABIInfo::flattenType(
834 llvm::Type *Ty, SmallVectorImpl<llvm::Type *> &Flattened) const {
835
836 if (ABIArgInfo::isPaddingForCoerceAndExpand(eltType: Ty)) {
837 Flattened.push_back(Elt: Ty);
838 return;
839 }
840
841 if (const auto *AT = dyn_cast<llvm::ArrayType>(Val: Ty)) {
842 uint64_t NElt = AT->getNumElements();
843 if (NElt == 0)
844 return;
845
846 SmallVector<llvm::Type *> EltFlattened;
847 flattenType(Ty: AT->getElementType(), Flattened&: EltFlattened);
848
849 for (uint64_t I = 0; I < NElt; ++I)
850 llvm::append_range(C&: Flattened, R&: EltFlattened);
851 return;
852 }
853
854 if (const auto *ST = dyn_cast<llvm::StructType>(Val: Ty)) {
855 for (auto *ET : ST->elements())
856 flattenType(Ty: ET, Flattened);
857 return;
858 }
859
860 Flattened.push_back(Elt: Ty);
861}
862
863RValue AArch64ABIInfo::EmitAAPCSVAArg(Address VAListAddr, QualType Ty,
864 CodeGenFunction &CGF, AArch64ABIKind Kind,
865 AggValueSlot Slot) const {
866 // These numbers are not used for variadic arguments, hence it doesn't matter
867 // they don't retain their values across multiple calls to
868 // `classifyArgumentType` here.
869 unsigned NSRN = 0, NPRN = 0;
870 ABIArgInfo AI =
871 classifyArgumentType(Ty, /*IsVariadicFn=*/true, /* IsNamedArg */ false,
872 CallingConvention: CGF.CurFnInfo->getCallingConvention(), NSRN, NPRN);
873 // Empty records are ignored for parameter passing purposes.
874 if (AI.isIgnore())
875 return Slot.asRValue();
876
877 bool IsIndirect = AI.isIndirect();
878
879 llvm::Type *BaseTy = CGF.ConvertType(T: Ty);
880 if (IsIndirect)
881 BaseTy = llvm::PointerType::getUnqual(C&: BaseTy->getContext());
882 else if (AI.getCoerceToType())
883 BaseTy = AI.getCoerceToType();
884
885 unsigned NumRegs = 1;
886 if (llvm::ArrayType *ArrTy = dyn_cast<llvm::ArrayType>(Val: BaseTy)) {
887 BaseTy = ArrTy->getElementType();
888 NumRegs = ArrTy->getNumElements();
889 }
890 bool IsFPR =
891 !isSoftFloat() && (BaseTy->isFloatingPointTy() || BaseTy->isVectorTy());
892
893 // The AArch64 va_list type and handling is specified in the Procedure Call
894 // Standard, section B.4:
895 //
896 // struct {
897 // void *__stack;
898 // void *__gr_top;
899 // void *__vr_top;
900 // int __gr_offs;
901 // int __vr_offs;
902 // };
903
904 llvm::BasicBlock *MaybeRegBlock = CGF.createBasicBlock(name: "vaarg.maybe_reg");
905 llvm::BasicBlock *InRegBlock = CGF.createBasicBlock(name: "vaarg.in_reg");
906 llvm::BasicBlock *OnStackBlock = CGF.createBasicBlock(name: "vaarg.on_stack");
907 llvm::BasicBlock *ContBlock = CGF.createBasicBlock(name: "vaarg.end");
908
909 CharUnits TySize = getContext().getTypeSizeInChars(T: Ty);
910 CharUnits TyAlign = getContext().getTypeUnadjustedAlignInChars(T: Ty);
911
912 Address reg_offs_p = Address::invalid();
913 llvm::Value *reg_offs = nullptr;
914 int reg_top_index;
915 int RegSize = IsIndirect ? 8 : TySize.getQuantity();
916 if (!IsFPR) {
917 // 3 is the field number of __gr_offs
918 reg_offs_p = CGF.Builder.CreateStructGEP(Addr: VAListAddr, Index: 3, Name: "gr_offs_p");
919 reg_offs = CGF.Builder.CreateLoad(Addr: reg_offs_p, Name: "gr_offs");
920 reg_top_index = 1; // field number for __gr_top
921 RegSize = llvm::alignTo(Value: RegSize, Align: 8);
922 } else {
923 // 4 is the field number of __vr_offs.
924 reg_offs_p = CGF.Builder.CreateStructGEP(Addr: VAListAddr, Index: 4, Name: "vr_offs_p");
925 reg_offs = CGF.Builder.CreateLoad(Addr: reg_offs_p, Name: "vr_offs");
926 reg_top_index = 2; // field number for __vr_top
927 RegSize = 16 * NumRegs;
928 }
929
930 //=======================================
931 // Find out where argument was passed
932 //=======================================
933
934 // If reg_offs >= 0 we're already using the stack for this type of
935 // argument. We don't want to keep updating reg_offs (in case it overflows,
936 // though anyone passing 2GB of arguments, each at most 16 bytes, deserves
937 // whatever they get).
938 llvm::Value *UsingStack = nullptr;
939 UsingStack = CGF.Builder.CreateICmpSGE(
940 LHS: reg_offs, RHS: llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: 0));
941
942 CGF.Builder.CreateCondBr(Cond: UsingStack, True: OnStackBlock, False: MaybeRegBlock);
943
944 // Otherwise, at least some kind of argument could go in these registers, the
945 // question is whether this particular type is too big.
946 CGF.EmitBlock(BB: MaybeRegBlock);
947
948 // Integer arguments may need to correct register alignment (for example a
949 // "struct { __int128 a; };" gets passed in x_2N, x_{2N+1}). In this case we
950 // align __gr_offs to calculate the potential address.
951 if (!IsFPR && !IsIndirect && TyAlign.getQuantity() > 8) {
952 int Align = TyAlign.getQuantity();
953
954 reg_offs = CGF.Builder.CreateAdd(
955 LHS: reg_offs, RHS: llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: Align - 1),
956 Name: "align_regoffs");
957 reg_offs = CGF.Builder.CreateAnd(
958 LHS: reg_offs, RHS: llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: -Align),
959 Name: "aligned_regoffs");
960 }
961
962 // Update the gr_offs/vr_offs pointer for next call to va_arg on this va_list.
963 // The fact that this is done unconditionally reflects the fact that
964 // allocating an argument to the stack also uses up all the remaining
965 // registers of the appropriate kind.
966 llvm::Value *NewOffset = nullptr;
967 NewOffset = CGF.Builder.CreateAdd(
968 LHS: reg_offs, RHS: llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: RegSize), Name: "new_reg_offs");
969 CGF.Builder.CreateStore(Val: NewOffset, Addr: reg_offs_p);
970
971 // Now we're in a position to decide whether this argument really was in
972 // registers or not.
973 llvm::Value *InRegs = nullptr;
974 InRegs = CGF.Builder.CreateICmpSLE(
975 LHS: NewOffset, RHS: llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: 0), Name: "inreg");
976
977 CGF.Builder.CreateCondBr(Cond: InRegs, True: InRegBlock, False: OnStackBlock);
978
979 //=======================================
980 // Argument was in registers
981 //=======================================
982
983 // Now we emit the code for if the argument was originally passed in
984 // registers. First start the appropriate block:
985 CGF.EmitBlock(BB: InRegBlock);
986
987 llvm::Value *reg_top = nullptr;
988 Address reg_top_p =
989 CGF.Builder.CreateStructGEP(Addr: VAListAddr, Index: reg_top_index, Name: "reg_top_p");
990 reg_top = CGF.Builder.CreateLoad(Addr: reg_top_p, Name: "reg_top");
991 Address BaseAddr(CGF.Builder.CreateInBoundsGEP(Ty: CGF.Int8Ty, Ptr: reg_top, IdxList: reg_offs),
992 CGF.Int8Ty, CharUnits::fromQuantity(Quantity: IsFPR ? 16 : 8));
993 Address RegAddr = Address::invalid();
994 llvm::Type *MemTy = CGF.ConvertTypeForMem(T: Ty), *ElementTy = MemTy;
995
996 if (IsIndirect) {
997 // If it's been passed indirectly (actually a struct), whatever we find from
998 // stored registers or on the stack will actually be a struct **.
999 MemTy = llvm::PointerType::getUnqual(C&: MemTy->getContext());
1000 }
1001
1002 const Type *Base = nullptr;
1003 uint64_t NumMembers = 0;
1004 bool IsHFA = isHomogeneousAggregate(Ty, Base, Members&: NumMembers);
1005 if (IsHFA && NumMembers > 1) {
1006 // Homogeneous aggregates passed in registers will have their elements split
1007 // and stored 16-bytes apart regardless of size (they're notionally in qN,
1008 // qN+1, ...). We reload and store into a temporary local variable
1009 // contiguously.
1010 assert(!IsIndirect && "Homogeneous aggregates should be passed directly");
1011 auto BaseTyInfo = getContext().getTypeInfoInChars(T: QualType(Base, 0));
1012 llvm::Type *BaseTy = CGF.ConvertType(T: QualType(Base, 0));
1013 llvm::Type *HFATy = llvm::ArrayType::get(ElementType: BaseTy, NumElements: NumMembers);
1014 Address Tmp = CGF.CreateTempAlloca(Ty: HFATy,
1015 align: std::max(a: TyAlign, b: BaseTyInfo.Align));
1016
1017 // On big-endian platforms, the value will be right-aligned in its slot.
1018 int Offset = 0;
1019 if (CGF.CGM.getDataLayout().isBigEndian() &&
1020 BaseTyInfo.Width.getQuantity() < 16)
1021 Offset = 16 - BaseTyInfo.Width.getQuantity();
1022
1023 for (unsigned i = 0; i < NumMembers; ++i) {
1024 CharUnits BaseOffset = CharUnits::fromQuantity(Quantity: 16 * i + Offset);
1025 Address LoadAddr =
1026 CGF.Builder.CreateConstInBoundsByteGEP(Addr: BaseAddr, Offset: BaseOffset);
1027 LoadAddr = LoadAddr.withElementType(ElemTy: BaseTy);
1028
1029 Address StoreAddr = CGF.Builder.CreateConstArrayGEP(Addr: Tmp, Index: i);
1030
1031 llvm::Value *Elem = CGF.Builder.CreateLoad(Addr: LoadAddr);
1032 CGF.Builder.CreateStore(Val: Elem, Addr: StoreAddr);
1033 }
1034
1035 RegAddr = Tmp.withElementType(ElemTy: MemTy);
1036 } else {
1037 // Otherwise the object is contiguous in memory.
1038
1039 // It might be right-aligned in its slot.
1040 CharUnits SlotSize = BaseAddr.getAlignment();
1041 if (CGF.CGM.getDataLayout().isBigEndian() && !IsIndirect &&
1042 (IsHFA || !isAggregateTypeForABI(T: Ty)) &&
1043 TySize < SlotSize) {
1044 CharUnits Offset = SlotSize - TySize;
1045 BaseAddr = CGF.Builder.CreateConstInBoundsByteGEP(Addr: BaseAddr, Offset);
1046 }
1047
1048 RegAddr = BaseAddr.withElementType(ElemTy: MemTy);
1049 }
1050
1051 CGF.EmitBranch(Block: ContBlock);
1052
1053 //=======================================
1054 // Argument was on the stack
1055 //=======================================
1056 CGF.EmitBlock(BB: OnStackBlock);
1057
1058 Address stack_p = CGF.Builder.CreateStructGEP(Addr: VAListAddr, Index: 0, Name: "stack_p");
1059 llvm::Value *OnStackPtr = CGF.Builder.CreateLoad(Addr: stack_p, Name: "stack");
1060
1061 // Again, stack arguments may need realignment. In this case both integer and
1062 // floating-point ones might be affected.
1063 if (!IsIndirect && TyAlign.getQuantity() > 8) {
1064 OnStackPtr = emitRoundPointerUpToAlignment(CGF, Ptr: OnStackPtr, Align: TyAlign);
1065 }
1066 Address OnStackAddr = Address(OnStackPtr, CGF.Int8Ty,
1067 std::max(a: CharUnits::fromQuantity(Quantity: 8), b: TyAlign));
1068
1069 // All stack slots are multiples of 8 bytes.
1070 CharUnits StackSlotSize = CharUnits::fromQuantity(Quantity: 8);
1071 CharUnits StackSize;
1072 if (IsIndirect)
1073 StackSize = StackSlotSize;
1074 else
1075 StackSize = TySize.alignTo(Align: StackSlotSize);
1076
1077 llvm::Value *StackSizeC = CGF.Builder.getSize(N: StackSize);
1078 llvm::Value *NewStack = CGF.Builder.CreateInBoundsGEP(
1079 Ty: CGF.Int8Ty, Ptr: OnStackPtr, IdxList: StackSizeC, Name: "new_stack");
1080
1081 // Write the new value of __stack for the next call to va_arg
1082 CGF.Builder.CreateStore(Val: NewStack, Addr: stack_p);
1083
1084 if (CGF.CGM.getDataLayout().isBigEndian() && !isAggregateTypeForABI(T: Ty) &&
1085 TySize < StackSlotSize) {
1086 CharUnits Offset = StackSlotSize - TySize;
1087 OnStackAddr = CGF.Builder.CreateConstInBoundsByteGEP(Addr: OnStackAddr, Offset);
1088 }
1089
1090 OnStackAddr = OnStackAddr.withElementType(ElemTy: MemTy);
1091
1092 CGF.EmitBranch(Block: ContBlock);
1093
1094 //=======================================
1095 // Tidy up
1096 //=======================================
1097 CGF.EmitBlock(BB: ContBlock);
1098
1099 Address ResAddr = emitMergePHI(CGF, Addr1: RegAddr, Block1: InRegBlock, Addr2: OnStackAddr,
1100 Block2: OnStackBlock, Name: "vaargs.addr");
1101
1102 if (IsIndirect)
1103 return CGF.EmitLoadOfAnyValue(
1104 V: CGF.MakeAddrLValue(
1105 Addr: Address(CGF.Builder.CreateLoad(Addr: ResAddr, Name: "vaarg.addr"), ElementTy,
1106 TyAlign),
1107 T: Ty),
1108 Slot);
1109
1110 return CGF.EmitLoadOfAnyValue(V: CGF.MakeAddrLValue(Addr: ResAddr, T: Ty), Slot);
1111}
1112
1113RValue AArch64ABIInfo::EmitDarwinVAArg(Address VAListAddr, QualType Ty,
1114 CodeGenFunction &CGF,
1115 AggValueSlot Slot) const {
1116 // The backend's lowering doesn't support va_arg for aggregates or
1117 // illegal vector types. Lower VAArg here for these cases and use
1118 // the LLVM va_arg instruction for everything else.
1119 if (!isAggregateTypeForABI(T: Ty) && !isIllegalVectorType(Ty))
1120 return CGF.EmitLoadOfAnyValue(
1121 V: CGF.MakeAddrLValue(
1122 Addr: EmitVAArgInstr(CGF, VAListAddr, Ty, AI: ABIArgInfo::getDirect()), T: Ty),
1123 Slot);
1124
1125 uint64_t PointerSize = getTarget().getPointerWidth(AddrSpace: LangAS::Default) / 8;
1126 CharUnits SlotSize = CharUnits::fromQuantity(Quantity: PointerSize);
1127
1128 // Empty records are ignored for parameter passing purposes.
1129 if (isEmptyRecord(Context&: getContext(), T: Ty, AllowArrays: true))
1130 return Slot.asRValue();
1131
1132 // The size of the actual thing passed, which might end up just
1133 // being a pointer for indirect types.
1134 auto TyInfo = getContext().getTypeInfoInChars(T: Ty);
1135
1136 // Arguments bigger than 16 bytes which aren't homogeneous
1137 // aggregates should be passed indirectly.
1138 bool IsIndirect = false;
1139 if (TyInfo.Width.getQuantity() > 16) {
1140 const Type *Base = nullptr;
1141 uint64_t Members = 0;
1142 IsIndirect = !isHomogeneousAggregate(Ty, Base, Members);
1143 }
1144
1145 return emitVoidPtrVAArg(CGF, VAListAddr, ValueTy: Ty, IsIndirect, ValueInfo: TyInfo, SlotSizeAndAlign: SlotSize,
1146 /*AllowHigherAlign*/ true, Slot);
1147}
1148
1149RValue AArch64ABIInfo::EmitMSVAArg(CodeGenFunction &CGF, Address VAListAddr,
1150 QualType Ty, AggValueSlot Slot) const {
1151 bool IsIndirect = false;
1152
1153 // Composites larger than 16 bytes are passed by reference.
1154 if (isAggregateTypeForABI(T: Ty) && getContext().getTypeSize(T: Ty) > 128)
1155 IsIndirect = true;
1156
1157 return emitVoidPtrVAArg(CGF, VAListAddr, ValueTy: Ty, IsIndirect,
1158 ValueInfo: CGF.getContext().getTypeInfoInChars(T: Ty),
1159 SlotSizeAndAlign: CharUnits::fromQuantity(Quantity: 8),
1160 /*allowHigherAlign*/ AllowHigherAlign: false, Slot);
1161}
1162
1163static bool isStreamingCompatible(const FunctionDecl *F) {
1164 if (const auto *T = F->getType()->getAs<FunctionProtoType>())
1165 return T->getAArch64SMEAttributes() &
1166 FunctionType::SME_PStateSMCompatibleMask;
1167 return false;
1168}
1169
1170// Report an error if an argument or return value of type Ty would need to be
1171// passed in a floating-point register.
1172static void diagnoseIfNeedsFPReg(DiagnosticsEngine &Diags,
1173 const StringRef ABIName,
1174 const AArch64ABIInfo &ABIInfo,
1175 const QualType &Ty, const NamedDecl *D,
1176 SourceLocation loc) {
1177 const Type *HABase = nullptr;
1178 uint64_t HAMembers = 0;
1179 if (Ty->isFloatingType() || Ty->isVectorType() ||
1180 ABIInfo.isHomogeneousAggregate(Ty, Base&: HABase, Members&: HAMembers)) {
1181 Diags.Report(loc, diag::err_target_unsupported_type_for_abi)
1182 << D->getDeclName() << Ty << ABIName;
1183 }
1184}
1185
1186// If we are using a hard-float ABI, but do not have floating point registers,
1187// then report an error for any function arguments or returns which would be
1188// passed in floating-pint registers.
1189void AArch64TargetCodeGenInfo::checkFunctionABI(
1190 CodeGenModule &CGM, const FunctionDecl *FuncDecl) const {
1191 const AArch64ABIInfo &ABIInfo = getABIInfo<AArch64ABIInfo>();
1192 const TargetInfo &TI = ABIInfo.getContext().getTargetInfo();
1193
1194 if (!TI.hasFeature(Feature: "fp") && !ABIInfo.isSoftFloat()) {
1195 diagnoseIfNeedsFPReg(CGM.getDiags(), TI.getABI(), ABIInfo,
1196 FuncDecl->getReturnType(), FuncDecl,
1197 FuncDecl->getLocation());
1198 for (ParmVarDecl *PVD : FuncDecl->parameters()) {
1199 diagnoseIfNeedsFPReg(CGM.getDiags(), TI.getABI(), ABIInfo, PVD->getType(),
1200 PVD, FuncDecl->getLocation());
1201 }
1202 }
1203}
1204
1205enum class ArmSMEInlinability : uint8_t {
1206 Ok = 0,
1207 ErrorCalleeRequiresNewZA = 1 << 0,
1208 ErrorCalleeRequiresNewZT0 = 1 << 1,
1209 WarnIncompatibleStreamingModes = 1 << 2,
1210 ErrorIncompatibleStreamingModes = 1 << 3,
1211
1212 IncompatibleStreamingModes =
1213 WarnIncompatibleStreamingModes | ErrorIncompatibleStreamingModes,
1214
1215 LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/ErrorIncompatibleStreamingModes),
1216};
1217
1218/// Determines if there are any Arm SME ABI issues with inlining \p Callee into
1219/// \p Caller. Returns the issue (if any) in the ArmSMEInlinability bit enum.
1220static ArmSMEInlinability GetArmSMEInlinability(const FunctionDecl *Caller,
1221 const FunctionDecl *Callee) {
1222 bool CallerIsStreaming =
1223 IsArmStreamingFunction(FD: Caller, /*IncludeLocallyStreaming=*/true);
1224 bool CalleeIsStreaming =
1225 IsArmStreamingFunction(FD: Callee, /*IncludeLocallyStreaming=*/true);
1226 bool CallerIsStreamingCompatible = isStreamingCompatible(F: Caller);
1227 bool CalleeIsStreamingCompatible = isStreamingCompatible(F: Callee);
1228
1229 ArmSMEInlinability Inlinability = ArmSMEInlinability::Ok;
1230
1231 if (!CalleeIsStreamingCompatible &&
1232 (CallerIsStreaming != CalleeIsStreaming || CallerIsStreamingCompatible)) {
1233 if (CalleeIsStreaming)
1234 Inlinability |= ArmSMEInlinability::ErrorIncompatibleStreamingModes;
1235 else
1236 Inlinability |= ArmSMEInlinability::WarnIncompatibleStreamingModes;
1237 }
1238 if (auto *NewAttr = Callee->getAttr<ArmNewAttr>()) {
1239 if (NewAttr->isNewZA())
1240 Inlinability |= ArmSMEInlinability::ErrorCalleeRequiresNewZA;
1241 if (NewAttr->isNewZT0())
1242 Inlinability |= ArmSMEInlinability::ErrorCalleeRequiresNewZT0;
1243 }
1244
1245 return Inlinability;
1246}
1247
1248void AArch64TargetCodeGenInfo::checkFunctionCallABIStreaming(
1249 CodeGenModule &CGM, SourceLocation CallLoc, const FunctionDecl *Caller,
1250 const FunctionDecl *Callee) const {
1251 if (!Caller || !Callee || !Callee->hasAttr<AlwaysInlineAttr>())
1252 return;
1253
1254 ArmSMEInlinability Inlinability = GetArmSMEInlinability(Caller, Callee);
1255
1256 if ((Inlinability & ArmSMEInlinability::IncompatibleStreamingModes) !=
1257 ArmSMEInlinability::Ok)
1258 CGM.getDiags().Report(
1259 CallLoc,
1260 (Inlinability & ArmSMEInlinability::ErrorIncompatibleStreamingModes) ==
1261 ArmSMEInlinability::ErrorIncompatibleStreamingModes
1262 ? diag::err_function_always_inline_attribute_mismatch
1263 : diag::warn_function_always_inline_attribute_mismatch)
1264 << Caller->getDeclName() << Callee->getDeclName() << "streaming";
1265
1266 if ((Inlinability & ArmSMEInlinability::ErrorCalleeRequiresNewZA) ==
1267 ArmSMEInlinability::ErrorCalleeRequiresNewZA)
1268 CGM.getDiags().Report(CallLoc, diag::err_function_always_inline_new_za)
1269 << Callee->getDeclName();
1270
1271 if ((Inlinability & ArmSMEInlinability::ErrorCalleeRequiresNewZT0) ==
1272 ArmSMEInlinability::ErrorCalleeRequiresNewZT0)
1273 CGM.getDiags().Report(CallLoc, diag::err_function_always_inline_new_zt0)
1274 << Callee->getDeclName();
1275}
1276
1277// If the target does not have floating-point registers, but we are using a
1278// hard-float ABI, there is no way to pass floating-point, vector or HFA values
1279// to functions, so we report an error.
1280void AArch64TargetCodeGenInfo::checkFunctionCallABISoftFloat(
1281 CodeGenModule &CGM, SourceLocation CallLoc, const FunctionDecl *Caller,
1282 const FunctionDecl *Callee, const CallArgList &Args,
1283 QualType ReturnType) const {
1284 const AArch64ABIInfo &ABIInfo = getABIInfo<AArch64ABIInfo>();
1285 const TargetInfo &TI = ABIInfo.getContext().getTargetInfo();
1286
1287 if (!Caller || TI.hasFeature(Feature: "fp") || ABIInfo.isSoftFloat())
1288 return;
1289
1290 diagnoseIfNeedsFPReg(CGM.getDiags(), TI.getABI(), ABIInfo, ReturnType,
1291 Callee ? Callee : Caller, CallLoc);
1292
1293 for (const CallArg &Arg : Args)
1294 diagnoseIfNeedsFPReg(CGM.getDiags(), TI.getABI(), ABIInfo, Arg.getType(),
1295 Callee ? Callee : Caller, CallLoc);
1296}
1297
1298void AArch64TargetCodeGenInfo::checkFunctionCallABI(CodeGenModule &CGM,
1299 SourceLocation CallLoc,
1300 const FunctionDecl *Caller,
1301 const FunctionDecl *Callee,
1302 const CallArgList &Args,
1303 QualType ReturnType) const {
1304 checkFunctionCallABIStreaming(CGM, CallLoc, Caller, Callee);
1305 checkFunctionCallABISoftFloat(CGM, CallLoc, Caller, Callee, Args, ReturnType);
1306}
1307
1308bool AArch64TargetCodeGenInfo::wouldInliningViolateFunctionCallABI(
1309 const FunctionDecl *Caller, const FunctionDecl *Callee) const {
1310 return Caller && Callee &&
1311 GetArmSMEInlinability(Caller, Callee) != ArmSMEInlinability::Ok;
1312}
1313
1314void AArch64ABIInfo::appendAttributeMangling(TargetClonesAttr *Attr,
1315 unsigned Index,
1316 raw_ostream &Out) const {
1317 appendAttributeMangling(AttrStr: Attr->getFeatureStr(Index), Out);
1318}
1319
1320void AArch64ABIInfo::appendAttributeMangling(StringRef AttrStr,
1321 raw_ostream &Out) const {
1322 if (AttrStr == "default") {
1323 Out << ".default";
1324 return;
1325 }
1326
1327 Out << "._";
1328 SmallVector<StringRef, 8> Features;
1329 AttrStr.split(A&: Features, Separator: "+");
1330 for (auto &Feat : Features)
1331 Feat = Feat.trim();
1332
1333 llvm::sort(C&: Features, Comp: [](const StringRef LHS, const StringRef RHS) {
1334 return LHS.compare(RHS) < 0;
1335 });
1336
1337 llvm::SmallDenseSet<StringRef, 8> UniqueFeats;
1338 for (auto &Feat : Features)
1339 if (auto Ext = llvm::AArch64::parseFMVExtension(Feat))
1340 if (UniqueFeats.insert(Ext->Name).second)
1341 Out << 'M' << Ext->Name;
1342}
1343
1344std::unique_ptr<TargetCodeGenInfo>
1345CodeGen::createAArch64TargetCodeGenInfo(CodeGenModule &CGM,
1346 AArch64ABIKind Kind) {
1347 return std::make_unique<AArch64TargetCodeGenInfo>(args&: CGM.getTypes(), args&: Kind);
1348}
1349
1350std::unique_ptr<TargetCodeGenInfo>
1351CodeGen::createWindowsAArch64TargetCodeGenInfo(CodeGenModule &CGM,
1352 AArch64ABIKind K) {
1353 return std::make_unique<WindowsAArch64TargetCodeGenInfo>(args&: CGM.getTypes(), args&: K);
1354}
1355

source code of clang/lib/CodeGen/Targets/AArch64.cpp