1//===- AArch64.cpp --------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "ABIInfoImpl.h"
10#include "TargetInfo.h"
11#include "clang/Basic/DiagnosticFrontend.h"
12#include "llvm/TargetParser/AArch64TargetParser.h"
13
14using namespace clang;
15using namespace clang::CodeGen;
16
17//===----------------------------------------------------------------------===//
18// AArch64 ABI Implementation
19//===----------------------------------------------------------------------===//
20
21namespace {
22
23class AArch64ABIInfo : public ABIInfo {
24 AArch64ABIKind Kind;
25
26public:
27 AArch64ABIInfo(CodeGenTypes &CGT, AArch64ABIKind Kind)
28 : ABIInfo(CGT), Kind(Kind) {}
29
30 bool isSoftFloat() const { return Kind == AArch64ABIKind::AAPCSSoft; }
31
32private:
33 AArch64ABIKind getABIKind() const { return Kind; }
34 bool isDarwinPCS() const { return Kind == AArch64ABIKind::DarwinPCS; }
35
36 ABIArgInfo classifyReturnType(QualType RetTy, bool IsVariadic) const;
37 ABIArgInfo classifyArgumentType(QualType RetTy, bool IsVariadic,
38 unsigned CallingConvention) const;
39 ABIArgInfo coerceIllegalVector(QualType Ty) const;
40 bool isHomogeneousAggregateBaseType(QualType Ty) const override;
41 bool isHomogeneousAggregateSmallEnough(const Type *Ty,
42 uint64_t Members) const override;
43 bool isZeroLengthBitfieldPermittedInHomogeneousAggregate() const override;
44
45 bool isIllegalVectorType(QualType Ty) const;
46
47 void computeInfo(CGFunctionInfo &FI) const override {
48 if (!::classifyReturnType(CXXABI: getCXXABI(), FI, Info: *this))
49 FI.getReturnInfo() =
50 classifyReturnType(RetTy: FI.getReturnType(), IsVariadic: FI.isVariadic());
51
52 for (auto &it : FI.arguments())
53 it.info = classifyArgumentType(RetTy: it.type, IsVariadic: FI.isVariadic(),
54 CallingConvention: FI.getCallingConvention());
55 }
56
57 Address EmitDarwinVAArg(Address VAListAddr, QualType Ty,
58 CodeGenFunction &CGF) const;
59
60 Address EmitAAPCSVAArg(Address VAListAddr, QualType Ty, CodeGenFunction &CGF,
61 AArch64ABIKind Kind) const;
62
63 Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
64 QualType Ty) const override {
65 llvm::Type *BaseTy = CGF.ConvertType(T: Ty);
66 if (isa<llvm::ScalableVectorType>(Val: BaseTy))
67 llvm::report_fatal_error(reason: "Passing SVE types to variadic functions is "
68 "currently not supported");
69
70 return Kind == AArch64ABIKind::Win64 ? EmitMSVAArg(CGF, VAListAddr, Ty)
71 : isDarwinPCS() ? EmitDarwinVAArg(VAListAddr, Ty, CGF)
72 : EmitAAPCSVAArg(VAListAddr, Ty, CGF, Kind);
73 }
74
75 Address EmitMSVAArg(CodeGenFunction &CGF, Address VAListAddr,
76 QualType Ty) const override;
77
78 bool allowBFloatArgsAndRet() const override {
79 return getTarget().hasBFloat16Type();
80 }
81
82 using ABIInfo::appendAttributeMangling;
83 void appendAttributeMangling(TargetClonesAttr *Attr, unsigned Index,
84 raw_ostream &Out) const override;
85 void appendAttributeMangling(StringRef AttrStr,
86 raw_ostream &Out) const override;
87};
88
89class AArch64SwiftABIInfo : public SwiftABIInfo {
90public:
91 explicit AArch64SwiftABIInfo(CodeGenTypes &CGT)
92 : SwiftABIInfo(CGT, /*SwiftErrorInRegister=*/true) {}
93
94 bool isLegalVectorType(CharUnits VectorSize, llvm::Type *EltTy,
95 unsigned NumElts) const override;
96};
97
98class AArch64TargetCodeGenInfo : public TargetCodeGenInfo {
99public:
100 AArch64TargetCodeGenInfo(CodeGenTypes &CGT, AArch64ABIKind Kind)
101 : TargetCodeGenInfo(std::make_unique<AArch64ABIInfo>(args&: CGT, args&: Kind)) {
102 SwiftInfo = std::make_unique<AArch64SwiftABIInfo>(args&: CGT);
103 }
104
105 StringRef getARCRetainAutoreleasedReturnValueMarker() const override {
106 return "mov\tfp, fp\t\t// marker for objc_retainAutoreleaseReturnValue";
107 }
108
109 int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const override {
110 return 31;
111 }
112
113 bool doesReturnSlotInterfereWithArgs() const override { return false; }
114
115 void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
116 CodeGen::CodeGenModule &CGM) const override {
117 const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(Val: D);
118 if (!FD)
119 return;
120
121 const auto *TA = FD->getAttr<TargetAttr>();
122 if (TA == nullptr)
123 return;
124
125 ParsedTargetAttr Attr =
126 CGM.getTarget().parseTargetAttr(Str: TA->getFeaturesStr());
127 if (Attr.BranchProtection.empty())
128 return;
129
130 TargetInfo::BranchProtectionInfo BPI;
131 StringRef Error;
132 (void)CGM.getTarget().validateBranchProtection(Spec: Attr.BranchProtection,
133 Arch: Attr.CPU, BPI, Err&: Error);
134 assert(Error.empty());
135
136 auto *Fn = cast<llvm::Function>(Val: GV);
137 Fn->addFnAttr(Kind: "sign-return-address", Val: BPI.getSignReturnAddrStr());
138
139 if (BPI.SignReturnAddr != LangOptions::SignReturnAddressScopeKind::None) {
140 Fn->addFnAttr(Kind: "sign-return-address-key",
141 Val: BPI.SignKey == LangOptions::SignReturnAddressKeyKind::AKey
142 ? "a_key"
143 : "b_key");
144 }
145
146 Fn->addFnAttr(Kind: "branch-target-enforcement",
147 Val: BPI.BranchTargetEnforcement ? "true" : "false");
148 Fn->addFnAttr(Kind: "branch-protection-pauth-lr",
149 Val: BPI.BranchProtectionPAuthLR ? "true" : "false");
150 Fn->addFnAttr(Kind: "guarded-control-stack",
151 Val: BPI.GuardedControlStack ? "true" : "false");
152 }
153
154 bool isScalarizableAsmOperand(CodeGen::CodeGenFunction &CGF,
155 llvm::Type *Ty) const override {
156 if (CGF.getTarget().hasFeature(Feature: "ls64")) {
157 auto *ST = dyn_cast<llvm::StructType>(Val: Ty);
158 if (ST && ST->getNumElements() == 1) {
159 auto *AT = dyn_cast<llvm::ArrayType>(Val: ST->getElementType(N: 0));
160 if (AT && AT->getNumElements() == 8 &&
161 AT->getElementType()->isIntegerTy(Bitwidth: 64))
162 return true;
163 }
164 }
165 return TargetCodeGenInfo::isScalarizableAsmOperand(CGF, Ty);
166 }
167
168 void checkFunctionABI(CodeGenModule &CGM,
169 const FunctionDecl *Decl) const override;
170
171 void checkFunctionCallABI(CodeGenModule &CGM, SourceLocation CallLoc,
172 const FunctionDecl *Caller,
173 const FunctionDecl *Callee,
174 const CallArgList &Args) const override;
175};
176
177class WindowsAArch64TargetCodeGenInfo : public AArch64TargetCodeGenInfo {
178public:
179 WindowsAArch64TargetCodeGenInfo(CodeGenTypes &CGT, AArch64ABIKind K)
180 : AArch64TargetCodeGenInfo(CGT, K) {}
181
182 void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
183 CodeGen::CodeGenModule &CGM) const override;
184
185 void getDependentLibraryOption(llvm::StringRef Lib,
186 llvm::SmallString<24> &Opt) const override {
187 Opt = "/DEFAULTLIB:" + qualifyWindowsLibrary(Lib);
188 }
189
190 void getDetectMismatchOption(llvm::StringRef Name, llvm::StringRef Value,
191 llvm::SmallString<32> &Opt) const override {
192 Opt = "/FAILIFMISMATCH:\"" + Name.str() + "=" + Value.str() + "\"";
193 }
194};
195
196void WindowsAArch64TargetCodeGenInfo::setTargetAttributes(
197 const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &CGM) const {
198 AArch64TargetCodeGenInfo::setTargetAttributes(D, GV, CGM);
199 if (GV->isDeclaration())
200 return;
201 addStackProbeTargetAttributes(D, GV, CGM);
202}
203}
204
205ABIArgInfo AArch64ABIInfo::coerceIllegalVector(QualType Ty) const {
206 assert(Ty->isVectorType() && "expected vector type!");
207
208 const auto *VT = Ty->castAs<VectorType>();
209 if (VT->getVectorKind() == VectorKind::SveFixedLengthPredicate) {
210 assert(VT->getElementType()->isBuiltinType() && "expected builtin type!");
211 assert(VT->getElementType()->castAs<BuiltinType>()->getKind() ==
212 BuiltinType::UChar &&
213 "unexpected builtin type for SVE predicate!");
214 return ABIArgInfo::getDirect(T: llvm::ScalableVectorType::get(
215 ElementType: llvm::Type::getInt1Ty(C&: getVMContext()), MinNumElts: 16));
216 }
217
218 if (VT->getVectorKind() == VectorKind::SveFixedLengthData) {
219 assert(VT->getElementType()->isBuiltinType() && "expected builtin type!");
220
221 const auto *BT = VT->getElementType()->castAs<BuiltinType>();
222 llvm::ScalableVectorType *ResType = nullptr;
223 switch (BT->getKind()) {
224 default:
225 llvm_unreachable("unexpected builtin type for SVE vector!");
226 case BuiltinType::SChar:
227 case BuiltinType::UChar:
228 ResType = llvm::ScalableVectorType::get(
229 ElementType: llvm::Type::getInt8Ty(C&: getVMContext()), MinNumElts: 16);
230 break;
231 case BuiltinType::Short:
232 case BuiltinType::UShort:
233 ResType = llvm::ScalableVectorType::get(
234 ElementType: llvm::Type::getInt16Ty(C&: getVMContext()), MinNumElts: 8);
235 break;
236 case BuiltinType::Int:
237 case BuiltinType::UInt:
238 ResType = llvm::ScalableVectorType::get(
239 ElementType: llvm::Type::getInt32Ty(C&: getVMContext()), MinNumElts: 4);
240 break;
241 case BuiltinType::Long:
242 case BuiltinType::ULong:
243 ResType = llvm::ScalableVectorType::get(
244 ElementType: llvm::Type::getInt64Ty(C&: getVMContext()), MinNumElts: 2);
245 break;
246 case BuiltinType::Half:
247 ResType = llvm::ScalableVectorType::get(
248 ElementType: llvm::Type::getHalfTy(C&: getVMContext()), MinNumElts: 8);
249 break;
250 case BuiltinType::Float:
251 ResType = llvm::ScalableVectorType::get(
252 ElementType: llvm::Type::getFloatTy(C&: getVMContext()), MinNumElts: 4);
253 break;
254 case BuiltinType::Double:
255 ResType = llvm::ScalableVectorType::get(
256 ElementType: llvm::Type::getDoubleTy(C&: getVMContext()), MinNumElts: 2);
257 break;
258 case BuiltinType::BFloat16:
259 ResType = llvm::ScalableVectorType::get(
260 ElementType: llvm::Type::getBFloatTy(C&: getVMContext()), MinNumElts: 8);
261 break;
262 }
263 return ABIArgInfo::getDirect(T: ResType);
264 }
265
266 uint64_t Size = getContext().getTypeSize(T: Ty);
267 // Android promotes <2 x i8> to i16, not i32
268 if ((isAndroid() || isOHOSFamily()) && (Size <= 16)) {
269 llvm::Type *ResType = llvm::Type::getInt16Ty(C&: getVMContext());
270 return ABIArgInfo::getDirect(T: ResType);
271 }
272 if (Size <= 32) {
273 llvm::Type *ResType = llvm::Type::getInt32Ty(C&: getVMContext());
274 return ABIArgInfo::getDirect(T: ResType);
275 }
276 if (Size == 64) {
277 auto *ResType =
278 llvm::FixedVectorType::get(ElementType: llvm::Type::getInt32Ty(C&: getVMContext()), NumElts: 2);
279 return ABIArgInfo::getDirect(T: ResType);
280 }
281 if (Size == 128) {
282 auto *ResType =
283 llvm::FixedVectorType::get(ElementType: llvm::Type::getInt32Ty(C&: getVMContext()), NumElts: 4);
284 return ABIArgInfo::getDirect(T: ResType);
285 }
286 return getNaturalAlignIndirect(Ty, /*ByVal=*/false);
287}
288
289ABIArgInfo
290AArch64ABIInfo::classifyArgumentType(QualType Ty, bool IsVariadic,
291 unsigned CallingConvention) const {
292 Ty = useFirstFieldIfTransparentUnion(Ty);
293
294 // Handle illegal vector types here.
295 if (isIllegalVectorType(Ty))
296 return coerceIllegalVector(Ty);
297
298 if (!isAggregateTypeForABI(T: Ty)) {
299 // Treat an enum type as its underlying type.
300 if (const EnumType *EnumTy = Ty->getAs<EnumType>())
301 Ty = EnumTy->getDecl()->getIntegerType();
302
303 if (const auto *EIT = Ty->getAs<BitIntType>())
304 if (EIT->getNumBits() > 128)
305 return getNaturalAlignIndirect(Ty);
306
307 return (isPromotableIntegerTypeForABI(Ty) && isDarwinPCS()
308 ? ABIArgInfo::getExtend(Ty)
309 : ABIArgInfo::getDirect());
310 }
311
312 // Structures with either a non-trivial destructor or a non-trivial
313 // copy constructor are always indirect.
314 if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(T: Ty, CXXABI&: getCXXABI())) {
315 return getNaturalAlignIndirect(Ty, /*ByVal=*/RAA ==
316 CGCXXABI::RAA_DirectInMemory);
317 }
318
319 // Empty records are always ignored on Darwin, but actually passed in C++ mode
320 // elsewhere for GNU compatibility.
321 uint64_t Size = getContext().getTypeSize(T: Ty);
322 bool IsEmpty = isEmptyRecord(Context&: getContext(), T: Ty, AllowArrays: true);
323 if (IsEmpty || Size == 0) {
324 if (!getContext().getLangOpts().CPlusPlus || isDarwinPCS())
325 return ABIArgInfo::getIgnore();
326
327 // GNU C mode. The only argument that gets ignored is an empty one with size
328 // 0.
329 if (IsEmpty && Size == 0)
330 return ABIArgInfo::getIgnore();
331 return ABIArgInfo::getDirect(T: llvm::Type::getInt8Ty(C&: getVMContext()));
332 }
333
334 // Homogeneous Floating-point Aggregates (HFAs) need to be expanded.
335 const Type *Base = nullptr;
336 uint64_t Members = 0;
337 bool IsWin64 = Kind == AArch64ABIKind::Win64 ||
338 CallingConvention == llvm::CallingConv::Win64;
339 bool IsWinVariadic = IsWin64 && IsVariadic;
340 // In variadic functions on Windows, all composite types are treated alike,
341 // no special handling of HFAs/HVAs.
342 if (!IsWinVariadic && isHomogeneousAggregate(Ty, Base, Members)) {
343 if (Kind != AArch64ABIKind::AAPCS)
344 return ABIArgInfo::getDirect(
345 T: llvm::ArrayType::get(ElementType: CGT.ConvertType(T: QualType(Base, 0)), NumElements: Members));
346
347 // For HFAs/HVAs, cap the argument alignment to 16, otherwise
348 // set it to 8 according to the AAPCS64 document.
349 unsigned Align =
350 getContext().getTypeUnadjustedAlignInChars(T: Ty).getQuantity();
351 Align = (Align >= 16) ? 16 : 8;
352 return ABIArgInfo::getDirect(
353 T: llvm::ArrayType::get(ElementType: CGT.ConvertType(T: QualType(Base, 0)), NumElements: Members), Offset: 0,
354 Padding: nullptr, CanBeFlattened: true, Align);
355 }
356
357 // Aggregates <= 16 bytes are passed directly in registers or on the stack.
358 if (Size <= 128) {
359 // On RenderScript, coerce Aggregates <= 16 bytes to an integer array of
360 // same size and alignment.
361 if (getTarget().isRenderScriptTarget()) {
362 return coerceToIntArray(Ty, Context&: getContext(), LLVMContext&: getVMContext());
363 }
364 unsigned Alignment;
365 if (Kind == AArch64ABIKind::AAPCS) {
366 Alignment = getContext().getTypeUnadjustedAlign(T: Ty);
367 Alignment = Alignment < 128 ? 64 : 128;
368 } else {
369 Alignment =
370 std::max(a: getContext().getTypeAlign(T: Ty),
371 b: (unsigned)getTarget().getPointerWidth(AddrSpace: LangAS::Default));
372 }
373 Size = llvm::alignTo(Value: Size, Align: Alignment);
374
375 // We use a pair of i64 for 16-byte aggregate with 8-byte alignment.
376 // For aggregates with 16-byte alignment, we use i128.
377 llvm::Type *BaseTy = llvm::Type::getIntNTy(C&: getVMContext(), N: Alignment);
378 return ABIArgInfo::getDirect(
379 T: Size == Alignment ? BaseTy
380 : llvm::ArrayType::get(ElementType: BaseTy, NumElements: Size / Alignment));
381 }
382
383 return getNaturalAlignIndirect(Ty, /*ByVal=*/false);
384}
385
386ABIArgInfo AArch64ABIInfo::classifyReturnType(QualType RetTy,
387 bool IsVariadic) const {
388 if (RetTy->isVoidType())
389 return ABIArgInfo::getIgnore();
390
391 if (const auto *VT = RetTy->getAs<VectorType>()) {
392 if (VT->getVectorKind() == VectorKind::SveFixedLengthData ||
393 VT->getVectorKind() == VectorKind::SveFixedLengthPredicate)
394 return coerceIllegalVector(Ty: RetTy);
395 }
396
397 // Large vector types should be returned via memory.
398 if (RetTy->isVectorType() && getContext().getTypeSize(T: RetTy) > 128)
399 return getNaturalAlignIndirect(Ty: RetTy);
400
401 if (!isAggregateTypeForABI(T: RetTy)) {
402 // Treat an enum type as its underlying type.
403 if (const EnumType *EnumTy = RetTy->getAs<EnumType>())
404 RetTy = EnumTy->getDecl()->getIntegerType();
405
406 if (const auto *EIT = RetTy->getAs<BitIntType>())
407 if (EIT->getNumBits() > 128)
408 return getNaturalAlignIndirect(Ty: RetTy);
409
410 return (isPromotableIntegerTypeForABI(Ty: RetTy) && isDarwinPCS()
411 ? ABIArgInfo::getExtend(Ty: RetTy)
412 : ABIArgInfo::getDirect());
413 }
414
415 uint64_t Size = getContext().getTypeSize(T: RetTy);
416 if (isEmptyRecord(Context&: getContext(), T: RetTy, AllowArrays: true) || Size == 0)
417 return ABIArgInfo::getIgnore();
418
419 const Type *Base = nullptr;
420 uint64_t Members = 0;
421 if (isHomogeneousAggregate(Ty: RetTy, Base, Members) &&
422 !(getTarget().getTriple().getArch() == llvm::Triple::aarch64_32 &&
423 IsVariadic))
424 // Homogeneous Floating-point Aggregates (HFAs) are returned directly.
425 return ABIArgInfo::getDirect();
426
427 // Aggregates <= 16 bytes are returned directly in registers or on the stack.
428 if (Size <= 128) {
429 // On RenderScript, coerce Aggregates <= 16 bytes to an integer array of
430 // same size and alignment.
431 if (getTarget().isRenderScriptTarget()) {
432 return coerceToIntArray(Ty: RetTy, Context&: getContext(), LLVMContext&: getVMContext());
433 }
434
435 if (Size <= 64 && getDataLayout().isLittleEndian()) {
436 // Composite types are returned in lower bits of a 64-bit register for LE,
437 // and in higher bits for BE. However, integer types are always returned
438 // in lower bits for both LE and BE, and they are not rounded up to
439 // 64-bits. We can skip rounding up of composite types for LE, but not for
440 // BE, otherwise composite types will be indistinguishable from integer
441 // types.
442 return ABIArgInfo::getDirect(
443 T: llvm::IntegerType::get(C&: getVMContext(), NumBits: Size));
444 }
445
446 unsigned Alignment = getContext().getTypeAlign(T: RetTy);
447 Size = llvm::alignTo(Value: Size, Align: 64); // round up to multiple of 8 bytes
448
449 // We use a pair of i64 for 16-byte aggregate with 8-byte alignment.
450 // For aggregates with 16-byte alignment, we use i128.
451 if (Alignment < 128 && Size == 128) {
452 llvm::Type *BaseTy = llvm::Type::getInt64Ty(C&: getVMContext());
453 return ABIArgInfo::getDirect(T: llvm::ArrayType::get(ElementType: BaseTy, NumElements: Size / 64));
454 }
455 return ABIArgInfo::getDirect(T: llvm::IntegerType::get(C&: getVMContext(), NumBits: Size));
456 }
457
458 return getNaturalAlignIndirect(Ty: RetTy);
459}
460
461/// isIllegalVectorType - check whether the vector type is legal for AArch64.
462bool AArch64ABIInfo::isIllegalVectorType(QualType Ty) const {
463 if (const VectorType *VT = Ty->getAs<VectorType>()) {
464 // Check whether VT is a fixed-length SVE vector. These types are
465 // represented as scalable vectors in function args/return and must be
466 // coerced from fixed vectors.
467 if (VT->getVectorKind() == VectorKind::SveFixedLengthData ||
468 VT->getVectorKind() == VectorKind::SveFixedLengthPredicate)
469 return true;
470
471 // Check whether VT is legal.
472 unsigned NumElements = VT->getNumElements();
473 uint64_t Size = getContext().getTypeSize(VT);
474 // NumElements should be power of 2.
475 if (!llvm::isPowerOf2_32(Value: NumElements))
476 return true;
477
478 // arm64_32 has to be compatible with the ARM logic here, which allows huge
479 // vectors for some reason.
480 llvm::Triple Triple = getTarget().getTriple();
481 if (Triple.getArch() == llvm::Triple::aarch64_32 &&
482 Triple.isOSBinFormatMachO())
483 return Size <= 32;
484
485 return Size != 64 && (Size != 128 || NumElements == 1);
486 }
487 return false;
488}
489
490bool AArch64SwiftABIInfo::isLegalVectorType(CharUnits VectorSize,
491 llvm::Type *EltTy,
492 unsigned NumElts) const {
493 if (!llvm::isPowerOf2_32(Value: NumElts))
494 return false;
495 if (VectorSize.getQuantity() != 8 &&
496 (VectorSize.getQuantity() != 16 || NumElts == 1))
497 return false;
498 return true;
499}
500
501bool AArch64ABIInfo::isHomogeneousAggregateBaseType(QualType Ty) const {
502 // For the soft-float ABI variant, no types are considered to be homogeneous
503 // aggregates.
504 if (Kind == AArch64ABIKind::AAPCSSoft)
505 return false;
506
507 // Homogeneous aggregates for AAPCS64 must have base types of a floating
508 // point type or a short-vector type. This is the same as the 32-bit ABI,
509 // but with the difference that any floating-point type is allowed,
510 // including __fp16.
511 if (const BuiltinType *BT = Ty->getAs<BuiltinType>()) {
512 if (BT->isFloatingPoint())
513 return true;
514 } else if (const VectorType *VT = Ty->getAs<VectorType>()) {
515 unsigned VecSize = getContext().getTypeSize(VT);
516 if (VecSize == 64 || VecSize == 128)
517 return true;
518 }
519 return false;
520}
521
522bool AArch64ABIInfo::isHomogeneousAggregateSmallEnough(const Type *Base,
523 uint64_t Members) const {
524 return Members <= 4;
525}
526
527bool AArch64ABIInfo::isZeroLengthBitfieldPermittedInHomogeneousAggregate()
528 const {
529 // AAPCS64 says that the rule for whether something is a homogeneous
530 // aggregate is applied to the output of the data layout decision. So
531 // anything that doesn't affect the data layout also does not affect
532 // homogeneity. In particular, zero-length bitfields don't stop a struct
533 // being homogeneous.
534 return true;
535}
536
537Address AArch64ABIInfo::EmitAAPCSVAArg(Address VAListAddr, QualType Ty,
538 CodeGenFunction &CGF,
539 AArch64ABIKind Kind) const {
540 ABIArgInfo AI = classifyArgumentType(Ty, /*IsVariadic=*/true,
541 CallingConvention: CGF.CurFnInfo->getCallingConvention());
542 // Empty records are ignored for parameter passing purposes.
543 if (AI.isIgnore()) {
544 uint64_t PointerSize = getTarget().getPointerWidth(AddrSpace: LangAS::Default) / 8;
545 CharUnits SlotSize = CharUnits::fromQuantity(Quantity: PointerSize);
546 VAListAddr = VAListAddr.withElementType(ElemTy: CGF.Int8PtrTy);
547 auto *Load = CGF.Builder.CreateLoad(Addr: VAListAddr);
548 return Address(Load, CGF.ConvertTypeForMem(T: Ty), SlotSize);
549 }
550
551 bool IsIndirect = AI.isIndirect();
552
553 llvm::Type *BaseTy = CGF.ConvertType(T: Ty);
554 if (IsIndirect)
555 BaseTy = llvm::PointerType::getUnqual(ElementType: BaseTy);
556 else if (AI.getCoerceToType())
557 BaseTy = AI.getCoerceToType();
558
559 unsigned NumRegs = 1;
560 if (llvm::ArrayType *ArrTy = dyn_cast<llvm::ArrayType>(Val: BaseTy)) {
561 BaseTy = ArrTy->getElementType();
562 NumRegs = ArrTy->getNumElements();
563 }
564 bool IsFPR = Kind != AArch64ABIKind::AAPCSSoft &&
565 (BaseTy->isFloatingPointTy() || BaseTy->isVectorTy());
566
567 // The AArch64 va_list type and handling is specified in the Procedure Call
568 // Standard, section B.4:
569 //
570 // struct {
571 // void *__stack;
572 // void *__gr_top;
573 // void *__vr_top;
574 // int __gr_offs;
575 // int __vr_offs;
576 // };
577
578 llvm::BasicBlock *MaybeRegBlock = CGF.createBasicBlock(name: "vaarg.maybe_reg");
579 llvm::BasicBlock *InRegBlock = CGF.createBasicBlock(name: "vaarg.in_reg");
580 llvm::BasicBlock *OnStackBlock = CGF.createBasicBlock(name: "vaarg.on_stack");
581 llvm::BasicBlock *ContBlock = CGF.createBasicBlock(name: "vaarg.end");
582
583 CharUnits TySize = getContext().getTypeSizeInChars(T: Ty);
584 CharUnits TyAlign = getContext().getTypeUnadjustedAlignInChars(T: Ty);
585
586 Address reg_offs_p = Address::invalid();
587 llvm::Value *reg_offs = nullptr;
588 int reg_top_index;
589 int RegSize = IsIndirect ? 8 : TySize.getQuantity();
590 if (!IsFPR) {
591 // 3 is the field number of __gr_offs
592 reg_offs_p = CGF.Builder.CreateStructGEP(Addr: VAListAddr, Index: 3, Name: "gr_offs_p");
593 reg_offs = CGF.Builder.CreateLoad(Addr: reg_offs_p, Name: "gr_offs");
594 reg_top_index = 1; // field number for __gr_top
595 RegSize = llvm::alignTo(Value: RegSize, Align: 8);
596 } else {
597 // 4 is the field number of __vr_offs.
598 reg_offs_p = CGF.Builder.CreateStructGEP(Addr: VAListAddr, Index: 4, Name: "vr_offs_p");
599 reg_offs = CGF.Builder.CreateLoad(Addr: reg_offs_p, Name: "vr_offs");
600 reg_top_index = 2; // field number for __vr_top
601 RegSize = 16 * NumRegs;
602 }
603
604 //=======================================
605 // Find out where argument was passed
606 //=======================================
607
608 // If reg_offs >= 0 we're already using the stack for this type of
609 // argument. We don't want to keep updating reg_offs (in case it overflows,
610 // though anyone passing 2GB of arguments, each at most 16 bytes, deserves
611 // whatever they get).
612 llvm::Value *UsingStack = nullptr;
613 UsingStack = CGF.Builder.CreateICmpSGE(
614 LHS: reg_offs, RHS: llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: 0));
615
616 CGF.Builder.CreateCondBr(Cond: UsingStack, True: OnStackBlock, False: MaybeRegBlock);
617
618 // Otherwise, at least some kind of argument could go in these registers, the
619 // question is whether this particular type is too big.
620 CGF.EmitBlock(BB: MaybeRegBlock);
621
622 // Integer arguments may need to correct register alignment (for example a
623 // "struct { __int128 a; };" gets passed in x_2N, x_{2N+1}). In this case we
624 // align __gr_offs to calculate the potential address.
625 if (!IsFPR && !IsIndirect && TyAlign.getQuantity() > 8) {
626 int Align = TyAlign.getQuantity();
627
628 reg_offs = CGF.Builder.CreateAdd(
629 LHS: reg_offs, RHS: llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: Align - 1),
630 Name: "align_regoffs");
631 reg_offs = CGF.Builder.CreateAnd(
632 LHS: reg_offs, RHS: llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: -Align),
633 Name: "aligned_regoffs");
634 }
635
636 // Update the gr_offs/vr_offs pointer for next call to va_arg on this va_list.
637 // The fact that this is done unconditionally reflects the fact that
638 // allocating an argument to the stack also uses up all the remaining
639 // registers of the appropriate kind.
640 llvm::Value *NewOffset = nullptr;
641 NewOffset = CGF.Builder.CreateAdd(
642 LHS: reg_offs, RHS: llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: RegSize), Name: "new_reg_offs");
643 CGF.Builder.CreateStore(Val: NewOffset, Addr: reg_offs_p);
644
645 // Now we're in a position to decide whether this argument really was in
646 // registers or not.
647 llvm::Value *InRegs = nullptr;
648 InRegs = CGF.Builder.CreateICmpSLE(
649 LHS: NewOffset, RHS: llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: 0), Name: "inreg");
650
651 CGF.Builder.CreateCondBr(Cond: InRegs, True: InRegBlock, False: OnStackBlock);
652
653 //=======================================
654 // Argument was in registers
655 //=======================================
656
657 // Now we emit the code for if the argument was originally passed in
658 // registers. First start the appropriate block:
659 CGF.EmitBlock(BB: InRegBlock);
660
661 llvm::Value *reg_top = nullptr;
662 Address reg_top_p =
663 CGF.Builder.CreateStructGEP(Addr: VAListAddr, Index: reg_top_index, Name: "reg_top_p");
664 reg_top = CGF.Builder.CreateLoad(Addr: reg_top_p, Name: "reg_top");
665 Address BaseAddr(CGF.Builder.CreateInBoundsGEP(Ty: CGF.Int8Ty, Ptr: reg_top, IdxList: reg_offs),
666 CGF.Int8Ty, CharUnits::fromQuantity(Quantity: IsFPR ? 16 : 8));
667 Address RegAddr = Address::invalid();
668 llvm::Type *MemTy = CGF.ConvertTypeForMem(T: Ty), *ElementTy = MemTy;
669
670 if (IsIndirect) {
671 // If it's been passed indirectly (actually a struct), whatever we find from
672 // stored registers or on the stack will actually be a struct **.
673 MemTy = llvm::PointerType::getUnqual(ElementType: MemTy);
674 }
675
676 const Type *Base = nullptr;
677 uint64_t NumMembers = 0;
678 bool IsHFA = isHomogeneousAggregate(Ty, Base, Members&: NumMembers);
679 if (IsHFA && NumMembers > 1) {
680 // Homogeneous aggregates passed in registers will have their elements split
681 // and stored 16-bytes apart regardless of size (they're notionally in qN,
682 // qN+1, ...). We reload and store into a temporary local variable
683 // contiguously.
684 assert(!IsIndirect && "Homogeneous aggregates should be passed directly");
685 auto BaseTyInfo = getContext().getTypeInfoInChars(T: QualType(Base, 0));
686 llvm::Type *BaseTy = CGF.ConvertType(T: QualType(Base, 0));
687 llvm::Type *HFATy = llvm::ArrayType::get(ElementType: BaseTy, NumElements: NumMembers);
688 Address Tmp = CGF.CreateTempAlloca(Ty: HFATy,
689 align: std::max(a: TyAlign, b: BaseTyInfo.Align));
690
691 // On big-endian platforms, the value will be right-aligned in its slot.
692 int Offset = 0;
693 if (CGF.CGM.getDataLayout().isBigEndian() &&
694 BaseTyInfo.Width.getQuantity() < 16)
695 Offset = 16 - BaseTyInfo.Width.getQuantity();
696
697 for (unsigned i = 0; i < NumMembers; ++i) {
698 CharUnits BaseOffset = CharUnits::fromQuantity(Quantity: 16 * i + Offset);
699 Address LoadAddr =
700 CGF.Builder.CreateConstInBoundsByteGEP(Addr: BaseAddr, Offset: BaseOffset);
701 LoadAddr = LoadAddr.withElementType(ElemTy: BaseTy);
702
703 Address StoreAddr = CGF.Builder.CreateConstArrayGEP(Addr: Tmp, Index: i);
704
705 llvm::Value *Elem = CGF.Builder.CreateLoad(Addr: LoadAddr);
706 CGF.Builder.CreateStore(Val: Elem, Addr: StoreAddr);
707 }
708
709 RegAddr = Tmp.withElementType(ElemTy: MemTy);
710 } else {
711 // Otherwise the object is contiguous in memory.
712
713 // It might be right-aligned in its slot.
714 CharUnits SlotSize = BaseAddr.getAlignment();
715 if (CGF.CGM.getDataLayout().isBigEndian() && !IsIndirect &&
716 (IsHFA || !isAggregateTypeForABI(T: Ty)) &&
717 TySize < SlotSize) {
718 CharUnits Offset = SlotSize - TySize;
719 BaseAddr = CGF.Builder.CreateConstInBoundsByteGEP(Addr: BaseAddr, Offset);
720 }
721
722 RegAddr = BaseAddr.withElementType(ElemTy: MemTy);
723 }
724
725 CGF.EmitBranch(Block: ContBlock);
726
727 //=======================================
728 // Argument was on the stack
729 //=======================================
730 CGF.EmitBlock(BB: OnStackBlock);
731
732 Address stack_p = CGF.Builder.CreateStructGEP(Addr: VAListAddr, Index: 0, Name: "stack_p");
733 llvm::Value *OnStackPtr = CGF.Builder.CreateLoad(Addr: stack_p, Name: "stack");
734
735 // Again, stack arguments may need realignment. In this case both integer and
736 // floating-point ones might be affected.
737 if (!IsIndirect && TyAlign.getQuantity() > 8) {
738 int Align = TyAlign.getQuantity();
739
740 OnStackPtr = CGF.Builder.CreatePtrToInt(V: OnStackPtr, DestTy: CGF.Int64Ty);
741
742 OnStackPtr = CGF.Builder.CreateAdd(
743 LHS: OnStackPtr, RHS: llvm::ConstantInt::get(Ty: CGF.Int64Ty, V: Align - 1),
744 Name: "align_stack");
745 OnStackPtr = CGF.Builder.CreateAnd(
746 LHS: OnStackPtr, RHS: llvm::ConstantInt::get(Ty: CGF.Int64Ty, V: -Align),
747 Name: "align_stack");
748
749 OnStackPtr = CGF.Builder.CreateIntToPtr(V: OnStackPtr, DestTy: CGF.Int8PtrTy);
750 }
751 Address OnStackAddr = Address(OnStackPtr, CGF.Int8Ty,
752 std::max(a: CharUnits::fromQuantity(Quantity: 8), b: TyAlign));
753
754 // All stack slots are multiples of 8 bytes.
755 CharUnits StackSlotSize = CharUnits::fromQuantity(Quantity: 8);
756 CharUnits StackSize;
757 if (IsIndirect)
758 StackSize = StackSlotSize;
759 else
760 StackSize = TySize.alignTo(Align: StackSlotSize);
761
762 llvm::Value *StackSizeC = CGF.Builder.getSize(N: StackSize);
763 llvm::Value *NewStack = CGF.Builder.CreateInBoundsGEP(
764 Ty: CGF.Int8Ty, Ptr: OnStackPtr, IdxList: StackSizeC, Name: "new_stack");
765
766 // Write the new value of __stack for the next call to va_arg
767 CGF.Builder.CreateStore(Val: NewStack, Addr: stack_p);
768
769 if (CGF.CGM.getDataLayout().isBigEndian() && !isAggregateTypeForABI(T: Ty) &&
770 TySize < StackSlotSize) {
771 CharUnits Offset = StackSlotSize - TySize;
772 OnStackAddr = CGF.Builder.CreateConstInBoundsByteGEP(Addr: OnStackAddr, Offset);
773 }
774
775 OnStackAddr = OnStackAddr.withElementType(ElemTy: MemTy);
776
777 CGF.EmitBranch(Block: ContBlock);
778
779 //=======================================
780 // Tidy up
781 //=======================================
782 CGF.EmitBlock(BB: ContBlock);
783
784 Address ResAddr = emitMergePHI(CGF, Addr1: RegAddr, Block1: InRegBlock, Addr2: OnStackAddr,
785 Block2: OnStackBlock, Name: "vaargs.addr");
786
787 if (IsIndirect)
788 return Address(CGF.Builder.CreateLoad(Addr: ResAddr, Name: "vaarg.addr"), ElementTy,
789 TyAlign);
790
791 return ResAddr;
792}
793
794Address AArch64ABIInfo::EmitDarwinVAArg(Address VAListAddr, QualType Ty,
795 CodeGenFunction &CGF) const {
796 // The backend's lowering doesn't support va_arg for aggregates or
797 // illegal vector types. Lower VAArg here for these cases and use
798 // the LLVM va_arg instruction for everything else.
799 if (!isAggregateTypeForABI(T: Ty) && !isIllegalVectorType(Ty))
800 return EmitVAArgInstr(CGF, VAListAddr, Ty, AI: ABIArgInfo::getDirect());
801
802 uint64_t PointerSize = getTarget().getPointerWidth(AddrSpace: LangAS::Default) / 8;
803 CharUnits SlotSize = CharUnits::fromQuantity(Quantity: PointerSize);
804
805 // Empty records are ignored for parameter passing purposes.
806 if (isEmptyRecord(Context&: getContext(), T: Ty, AllowArrays: true))
807 return Address(CGF.Builder.CreateLoad(Addr: VAListAddr, Name: "ap.cur"),
808 CGF.ConvertTypeForMem(T: Ty), SlotSize);
809
810 // The size of the actual thing passed, which might end up just
811 // being a pointer for indirect types.
812 auto TyInfo = getContext().getTypeInfoInChars(T: Ty);
813
814 // Arguments bigger than 16 bytes which aren't homogeneous
815 // aggregates should be passed indirectly.
816 bool IsIndirect = false;
817 if (TyInfo.Width.getQuantity() > 16) {
818 const Type *Base = nullptr;
819 uint64_t Members = 0;
820 IsIndirect = !isHomogeneousAggregate(Ty, Base, Members);
821 }
822
823 return emitVoidPtrVAArg(CGF, VAListAddr, ValueTy: Ty, IsIndirect,
824 ValueInfo: TyInfo, SlotSizeAndAlign: SlotSize, /*AllowHigherAlign*/ true);
825}
826
827Address AArch64ABIInfo::EmitMSVAArg(CodeGenFunction &CGF, Address VAListAddr,
828 QualType Ty) const {
829 bool IsIndirect = false;
830
831 // Composites larger than 16 bytes are passed by reference.
832 if (isAggregateTypeForABI(T: Ty) && getContext().getTypeSize(T: Ty) > 128)
833 IsIndirect = true;
834
835 return emitVoidPtrVAArg(CGF, VAListAddr, ValueTy: Ty, IsIndirect,
836 ValueInfo: CGF.getContext().getTypeInfoInChars(T: Ty),
837 SlotSizeAndAlign: CharUnits::fromQuantity(Quantity: 8),
838 /*allowHigherAlign*/ AllowHigherAlign: false);
839}
840
841static bool isStreaming(const FunctionDecl *F) {
842 if (F->hasAttr<ArmLocallyStreamingAttr>())
843 return true;
844 if (const auto *T = F->getType()->getAs<FunctionProtoType>())
845 return T->getAArch64SMEAttributes() & FunctionType::SME_PStateSMEnabledMask;
846 return false;
847}
848
849static bool isStreamingCompatible(const FunctionDecl *F) {
850 if (const auto *T = F->getType()->getAs<FunctionProtoType>())
851 return T->getAArch64SMEAttributes() &
852 FunctionType::SME_PStateSMCompatibleMask;
853 return false;
854}
855
856void AArch64TargetCodeGenInfo::checkFunctionABI(
857 CodeGenModule &CGM, const FunctionDecl *FuncDecl) const {
858 const AArch64ABIInfo &ABIInfo = getABIInfo<AArch64ABIInfo>();
859 const TargetInfo &TI = ABIInfo.getContext().getTargetInfo();
860
861 // If we are using a hard-float ABI, but do not have floating point
862 // registers, then report an error for any function arguments or returns
863 // which would be passed in floating-pint registers.
864 auto CheckType = [&CGM, &TI, &ABIInfo](const QualType &Ty,
865 const NamedDecl *D) {
866 const Type *HABase = nullptr;
867 uint64_t HAMembers = 0;
868 if (Ty->isFloatingType() || Ty->isVectorType() ||
869 ABIInfo.isHomogeneousAggregate(Ty, Base&: HABase, Members&: HAMembers)) {
870 CGM.getDiags().Report(D->getLocation(),
871 diag::err_target_unsupported_type_for_abi)
872 << D->getDeclName() << Ty << TI.getABI();
873 }
874 };
875
876 if (!TI.hasFeature(Feature: "fp") && !ABIInfo.isSoftFloat()) {
877 CheckType(FuncDecl->getReturnType(), FuncDecl);
878 for (ParmVarDecl *PVD : FuncDecl->parameters()) {
879 CheckType(PVD->getType(), PVD);
880 }
881 }
882}
883
884void AArch64TargetCodeGenInfo::checkFunctionCallABI(
885 CodeGenModule &CGM, SourceLocation CallLoc, const FunctionDecl *Caller,
886 const FunctionDecl *Callee, const CallArgList &Args) const {
887 if (!Caller || !Callee || !Callee->hasAttr<AlwaysInlineAttr>())
888 return;
889
890 bool CallerIsStreaming = isStreaming(F: Caller);
891 bool CalleeIsStreaming = isStreaming(F: Callee);
892 bool CallerIsStreamingCompatible = isStreamingCompatible(F: Caller);
893 bool CalleeIsStreamingCompatible = isStreamingCompatible(F: Callee);
894
895 if (!CalleeIsStreamingCompatible &&
896 (CallerIsStreaming != CalleeIsStreaming || CallerIsStreamingCompatible))
897 CGM.getDiags().Report(CallLoc,
898 diag::err_function_always_inline_attribute_mismatch)
899 << Caller->getDeclName() << Callee->getDeclName() << "streaming";
900 if (auto *NewAttr = Callee->getAttr<ArmNewAttr>())
901 if (NewAttr->isNewZA())
902 CGM.getDiags().Report(CallLoc, diag::err_function_always_inline_new_za)
903 << Callee->getDeclName();
904}
905
906void AArch64ABIInfo::appendAttributeMangling(TargetClonesAttr *Attr,
907 unsigned Index,
908 raw_ostream &Out) const {
909 appendAttributeMangling(AttrStr: Attr->getFeatureStr(Index), Out);
910}
911
912void AArch64ABIInfo::appendAttributeMangling(StringRef AttrStr,
913 raw_ostream &Out) const {
914 if (AttrStr == "default") {
915 Out << ".default";
916 return;
917 }
918
919 Out << "._";
920 SmallVector<StringRef, 8> Features;
921 AttrStr.split(A&: Features, Separator: "+");
922 for (auto &Feat : Features)
923 Feat = Feat.trim();
924
925 llvm::sort(C&: Features, Comp: [](const StringRef LHS, const StringRef RHS) {
926 return LHS.compare(RHS) < 0;
927 });
928
929 llvm::SmallDenseSet<StringRef, 8> UniqueFeats;
930 for (auto &Feat : Features)
931 if (auto Ext = llvm::AArch64::parseArchExtension(Extension: Feat))
932 if (UniqueFeats.insert(V: Ext->Name).second)
933 Out << 'M' << Ext->Name;
934}
935
936std::unique_ptr<TargetCodeGenInfo>
937CodeGen::createAArch64TargetCodeGenInfo(CodeGenModule &CGM,
938 AArch64ABIKind Kind) {
939 return std::make_unique<AArch64TargetCodeGenInfo>(args&: CGM.getTypes(), args&: Kind);
940}
941
942std::unique_ptr<TargetCodeGenInfo>
943CodeGen::createWindowsAArch64TargetCodeGenInfo(CodeGenModule &CGM,
944 AArch64ABIKind K) {
945 return std::make_unique<WindowsAArch64TargetCodeGenInfo>(args&: CGM.getTypes(), args&: K);
946}
947

source code of clang/lib/CodeGen/Targets/AArch64.cpp