1	//===- X86.cpp ------------------------------------------------------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8
9	#include "ABIInfoImpl.h"
10	#include "TargetInfo.h"
11	#include "clang/Basic/DiagnosticFrontend.h"
12	#include "llvm/ADT/SmallBitVector.h"
13
14	using namespace clang;
15	using namespace clang::CodeGen;
16
17	namespace {
18
19	/// IsX86_MMXType - Return true if this is an MMX type.
20	bool IsX86_MMXType(llvm::Type *IRType) {
21	// Return true if the type is an MMX type <2 x i32>, <4 x i16>, or <8 x i8>.
22	return IRType->isVectorTy() && IRType->getPrimitiveSizeInBits() == 64 &&
23	cast<llvm::VectorType>(Val: IRType)->getElementType()->isIntegerTy() &&
24	IRType->getScalarSizeInBits() != 64;
25	}
26
27	static llvm::Type *X86AdjustInlineAsmType(CodeGen::CodeGenFunction &CGF,
28	StringRef Constraint,
29	llvm::Type *Ty) {
30	if (Constraint == "k") {
31	llvm::Type *Int1Ty = llvm::Type::getInt1Ty(C&: CGF.getLLVMContext());
32	return llvm::FixedVectorType::get(ElementType: Int1Ty, NumElts: Ty->getScalarSizeInBits());
33	}
34
35	// No operation needed
36	return Ty;
37	}
38
39	/// Returns true if this type can be passed in SSE registers with the
40	/// X86_VectorCall calling convention. Shared between x86_32 and x86_64.
41	static bool isX86VectorTypeForVectorCall(ASTContext &Context, QualType Ty) {
42	if (const BuiltinType *BT = Ty->getAs<BuiltinType>()) {
43	if (BT->isFloatingPoint() && BT->getKind() != BuiltinType::Half) {
44	if (BT->getKind() == BuiltinType::LongDouble) {
45	if (&Context.getTargetInfo().getLongDoubleFormat() ==
46	&llvm::APFloat::x87DoubleExtended())
47	return false;
48	}
49	return true;
50	}
51	} else if (const VectorType *VT = Ty->getAs<VectorType>()) {
52	// vectorcall can pass XMM, YMM, and ZMM vectors. We don't pass SSE1 MMX
53	// registers specially.
54	unsigned VecSize = Context.getTypeSize(VT);
55	if (VecSize == 128 || VecSize == 256 || VecSize == 512)
56	return true;
57	}
58	return false;
59	}
60
61	/// Returns true if this aggregate is small enough to be passed in SSE registers
62	/// in the X86_VectorCall calling convention. Shared between x86_32 and x86_64.
63	static bool isX86VectorCallAggregateSmallEnough(uint64_t NumMembers) {
64	return NumMembers <= 4;
65	}
66
67	/// Returns a Homogeneous Vector Aggregate ABIArgInfo, used in X86.
68	static ABIArgInfo getDirectX86Hva(llvm::Type* T = nullptr) {
69	auto AI = ABIArgInfo::getDirect(T);
70	AI.setInReg(true);
71	AI.setCanBeFlattened(false);
72	return AI;
73	}
74
75	//===----------------------------------------------------------------------===//
76	// X86-32 ABI Implementation
77	//===----------------------------------------------------------------------===//
78
79	/// Similar to llvm::CCState, but for Clang.
80	struct CCState {
81	CCState(CGFunctionInfo &FI)
82	: IsPreassigned(FI.arg_size()), CC(FI.getCallingConvention()),
83	Required(FI.getRequiredArgs()), IsDelegateCall(FI.isDelegateCall()) {}
84
85	llvm::SmallBitVector IsPreassigned;
86	unsigned CC = CallingConv::CC_C;
87	unsigned FreeRegs = 0;
88	unsigned FreeSSERegs = 0;
89	RequiredArgs Required;
90	bool IsDelegateCall = false;
91	};
92
93	/// X86_32ABIInfo - The X86-32 ABI information.
94	class X86_32ABIInfo : public ABIInfo {
95	enum Class {
96	Integer,
97	Float
98	};
99
100	static const unsigned MinABIStackAlignInBytes = 4;
101
102	bool IsDarwinVectorABI;
103	bool IsRetSmallStructInRegABI;
104	bool IsWin32StructABI;
105	bool IsSoftFloatABI;
106	bool IsMCUABI;
107	bool IsLinuxABI;
108	unsigned DefaultNumRegisterParameters;
109
110	static bool isRegisterSize(unsigned Size) {
111	return (Size == 8 || Size == 16 || Size == 32 || Size == 64);
112	}
113
114	bool isHomogeneousAggregateBaseType(QualType Ty) const override {
115	// FIXME: Assumes vectorcall is in use.
116	return isX86VectorTypeForVectorCall(Context&: getContext(), Ty);
117	}
118
119	bool isHomogeneousAggregateSmallEnough(const Type *Ty,
120	uint64_t NumMembers) const override {
121	// FIXME: Assumes vectorcall is in use.
122	return isX86VectorCallAggregateSmallEnough(NumMembers);
123	}
124
125	bool shouldReturnTypeInRegister(QualType Ty, ASTContext &Context) const;
126
127	/// getIndirectResult - Give a source type \arg Ty, return a suitable result
128	/// such that the argument will be passed in memory.
129	ABIArgInfo getIndirectResult(QualType Ty, bool ByVal, CCState &State) const;
130
131	ABIArgInfo getIndirectReturnResult(QualType Ty, CCState &State) const;
132
133	/// Return the alignment to use for the given type on the stack.
134	unsigned getTypeStackAlignInBytes(QualType Ty, unsigned Align) const;
135
136	Class classify(QualType Ty) const;
137	ABIArgInfo classifyReturnType(QualType RetTy, CCState &State) const;
138	ABIArgInfo classifyArgumentType(QualType RetTy, CCState &State,
139	unsigned ArgIndex) const;
140
141	/// Updates the number of available free registers, returns
142	/// true if any registers were allocated.
143	bool updateFreeRegs(QualType Ty, CCState &State) const;
144
145	bool shouldAggregateUseDirect(QualType Ty, CCState &State, bool &InReg,
146	bool &NeedsPadding) const;
147	bool shouldPrimitiveUseInReg(QualType Ty, CCState &State) const;
148
149	bool canExpandIndirectArgument(QualType Ty) const;
150
151	/// Rewrite the function info so that all memory arguments use
152	/// inalloca.
153	void rewriteWithInAlloca(CGFunctionInfo &FI) const;
154
155	void addFieldToArgStruct(SmallVector<llvm::Type *, 6> &FrameFields,
156	CharUnits &StackOffset, ABIArgInfo &Info,
157	QualType Type) const;
158	void runVectorCallFirstPass(CGFunctionInfo &FI, CCState &State) const;
159
160	public:
161
162	void computeInfo(CGFunctionInfo &FI) const override;
163	RValue EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, QualType Ty,
164	AggValueSlot Slot) const override;
165
166	X86_32ABIInfo(CodeGen::CodeGenTypes &CGT, bool DarwinVectorABI,
167	bool RetSmallStructInRegABI, bool Win32StructABI,
168	unsigned NumRegisterParameters, bool SoftFloatABI)
169	: ABIInfo(CGT), IsDarwinVectorABI(DarwinVectorABI),
170	IsRetSmallStructInRegABI(RetSmallStructInRegABI),
171	IsWin32StructABI(Win32StructABI), IsSoftFloatABI(SoftFloatABI),
172	IsMCUABI(CGT.getTarget().getTriple().isOSIAMCU()),
173	IsLinuxABI(CGT.getTarget().getTriple().isOSLinux() ||
174	CGT.getTarget().getTriple().isOSCygMing()),
175	DefaultNumRegisterParameters(NumRegisterParameters) {}
176	};
177
178	class X86_32SwiftABIInfo : public SwiftABIInfo {
179	public:
180	explicit X86_32SwiftABIInfo(CodeGenTypes &CGT)
181	: SwiftABIInfo(CGT, /*SwiftErrorInRegister=*/false) {}
182
183	bool shouldPassIndirectly(ArrayRef<llvm::Type *> ComponentTys,
184	bool AsReturnValue) const override {
185	// LLVM's x86-32 lowering currently only assigns up to three
186	// integer registers and three fp registers. Oddly, it'll use up to
187	// four vector registers for vectors, but those can overlap with the
188	// scalar registers.
189	return occupiesMoreThan(scalarTypes: ComponentTys, /*total=*/maxAllRegisters: 3);
190	}
191	};
192
193	class X86_32TargetCodeGenInfo : public TargetCodeGenInfo {
194	public:
195	X86_32TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT, bool DarwinVectorABI,
196	bool RetSmallStructInRegABI, bool Win32StructABI,
197	unsigned NumRegisterParameters, bool SoftFloatABI)
198	: TargetCodeGenInfo(std::make_unique<X86_32ABIInfo>(
199	args&: CGT, args&: DarwinVectorABI, args&: RetSmallStructInRegABI, args&: Win32StructABI,
200	args&: NumRegisterParameters, args&: SoftFloatABI)) {
201	SwiftInfo = std::make_unique<X86_32SwiftABIInfo>(args&: CGT);
202	}
203
204	static bool isStructReturnInRegABI(
205	const llvm::Triple &Triple, const CodeGenOptions &Opts);
206
207	void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
208	CodeGen::CodeGenModule &CGM) const override;
209
210	int getDwarfEHStackPointer(CodeGen::CodeGenModule &CGM) const override {
211	// Darwin uses different dwarf register numbers for EH.
212	if (CGM.getTarget().getTriple().isOSDarwin()) return 5;
213	return 4;
214	}
215
216	bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
217	llvm::Value *Address) const override;
218
219	llvm::Type* adjustInlineAsmType(CodeGen::CodeGenFunction &CGF,
220	StringRef Constraint,
221	llvm::Type* Ty) const override {
222	return X86AdjustInlineAsmType(CGF, Constraint, Ty);
223	}
224
225	void addReturnRegisterOutputs(CodeGenFunction &CGF, LValue ReturnValue,
226	std::string &Constraints,
227	std::vector<llvm::Type *> &ResultRegTypes,
228	std::vector<llvm::Type *> &ResultTruncRegTypes,
229	std::vector<LValue> &ResultRegDests,
230	std::string &AsmString,
231	unsigned NumOutputs) const override;
232
233	StringRef getARCRetainAutoreleasedReturnValueMarker() const override {
234	return "movl\t%ebp, %ebp"
235	"\t\t// marker for objc_retainAutoreleaseReturnValue";
236	}
237	};
238
239	}
240
241	/// Rewrite input constraint references after adding some output constraints.
242	/// In the case where there is one output and one input and we add one output,
243	/// we need to replace all operand references greater than or equal to 1:
244	/// mov $0, $1
245	/// mov eax, $1
246	/// The result will be:
247	/// mov $0, $2
248	/// mov eax, $2
249	static void rewriteInputConstraintReferences(unsigned FirstIn,
250	unsigned NumNewOuts,
251	std::string &AsmString) {
252	std::string Buf;
253	llvm::raw_string_ostream OS(Buf);
254	size_t Pos = 0;
255	while (Pos < AsmString.size()) {
256	size_t DollarStart = AsmString.find(c: '$', pos: Pos);
257	if (DollarStart == std::string::npos)
258	DollarStart = AsmString.size();
259	size_t DollarEnd = AsmString.find_first_not_of(c: '$', pos: DollarStart);
260	if (DollarEnd == std::string::npos)
261	DollarEnd = AsmString.size();
262	OS << StringRef(&AsmString[Pos], DollarEnd - Pos);
263	Pos = DollarEnd;
264	size_t NumDollars = DollarEnd - DollarStart;
265	if (NumDollars % 2 != 0 && Pos < AsmString.size()) {
266	// We have an operand reference.
267	size_t DigitStart = Pos;
268	if (AsmString[DigitStart] == '{') {
269	OS << '{';
270	++DigitStart;
271	}
272	size_t DigitEnd = AsmString.find_first_not_of(s: "0123456789", pos: DigitStart);
273	if (DigitEnd == std::string::npos)
274	DigitEnd = AsmString.size();
275	StringRef OperandStr(&AsmString[DigitStart], DigitEnd - DigitStart);
276	unsigned OperandIndex;
277	if (!OperandStr.getAsInteger(Radix: 10, Result&: OperandIndex)) {
278	if (OperandIndex >= FirstIn)
279	OperandIndex += NumNewOuts;
280	OS << OperandIndex;
281	} else {
282	OS << OperandStr;
283	}
284	Pos = DigitEnd;
285	}
286	}
287	AsmString = std::move(Buf);
288	}
289
290	/// Add output constraints for EAX:EDX because they are return registers.
291	void X86_32TargetCodeGenInfo::addReturnRegisterOutputs(
292	CodeGenFunction &CGF, LValue ReturnSlot, std::string &Constraints,
293	std::vector<llvm::Type *> &ResultRegTypes,
294	std::vector<llvm::Type *> &ResultTruncRegTypes,
295	std::vector<LValue> &ResultRegDests, std::string &AsmString,
296	unsigned NumOutputs) const {
297	uint64_t RetWidth = CGF.getContext().getTypeSize(T: ReturnSlot.getType());
298
299	// Use the EAX constraint if the width is 32 or smaller and EAX:EDX if it is
300	// larger.
301	if (!Constraints.empty())
302	Constraints += ',';
303	if (RetWidth <= 32) {
304	Constraints += "={eax}";
305	ResultRegTypes.push_back(x: CGF.Int32Ty);
306	} else {
307	// Use the 'A' constraint for EAX:EDX.
308	Constraints += "=A";
309	ResultRegTypes.push_back(x: CGF.Int64Ty);
310	}
311
312	// Truncate EAX or EAX:EDX to an integer of the appropriate size.
313	llvm::Type *CoerceTy = llvm::IntegerType::get(C&: CGF.getLLVMContext(), NumBits: RetWidth);
314	ResultTruncRegTypes.push_back(x: CoerceTy);
315
316	// Coerce the integer by bitcasting the return slot pointer.
317	ReturnSlot.setAddress(ReturnSlot.getAddress().withElementType(ElemTy: CoerceTy));
318	ResultRegDests.push_back(x: ReturnSlot);
319
320	rewriteInputConstraintReferences(FirstIn: NumOutputs, NumNewOuts: 1, AsmString);
321	}
322
323	/// shouldReturnTypeInRegister - Determine if the given type should be
324	/// returned in a register (for the Darwin and MCU ABI).
325	bool X86_32ABIInfo::shouldReturnTypeInRegister(QualType Ty,
326	ASTContext &Context) const {
327	uint64_t Size = Context.getTypeSize(T: Ty);
328
329	// For i386, type must be register sized.
330	// For the MCU ABI, it only needs to be <= 8-byte
331	if ((IsMCUABI && Size > 64) || (!IsMCUABI && !isRegisterSize(Size)))
332	return false;
333
334	if (Ty->isVectorType()) {
335	// 64- and 128- bit vectors inside structures are not returned in
336	// registers.
337	if (Size == 64 || Size == 128)
338	return false;
339
340	return true;
341	}
342
343	// If this is a builtin, pointer, enum, complex type, member pointer, or
344	// member function pointer it is ok.
345	if (Ty->getAs<BuiltinType>() || Ty->hasPointerRepresentation() ||
346	Ty->isAnyComplexType() || Ty->isEnumeralType() ||
347	Ty->isBlockPointerType() || Ty->isMemberPointerType())
348	return true;
349
350	// Arrays are treated like records.
351	if (const ConstantArrayType *AT = Context.getAsConstantArrayType(T: Ty))
352	return shouldReturnTypeInRegister(Ty: AT->getElementType(), Context);
353
354	// Otherwise, it must be a record type.
355	const RecordType *RT = Ty->getAs<RecordType>();
356	if (!RT) return false;
357
358	// FIXME: Traverse bases here too.
359
360	// Structure types are passed in register if all fields would be
361	// passed in a register.
362	for (const auto *FD : RT->getDecl()->fields()) {
363	// Empty fields are ignored.
364	if (isEmptyField(Context, FD, AllowArrays: true))
365	continue;
366
367	// Check fields recursively.
368	if (!shouldReturnTypeInRegister(Ty: FD->getType(), Context))
369	return false;
370	}
371	return true;
372	}
373
374	static bool is32Or64BitBasicType(QualType Ty, ASTContext &Context) {
375	// Treat complex types as the element type.
376	if (const ComplexType *CTy = Ty->getAs<ComplexType>())
377	Ty = CTy->getElementType();
378
379	// Check for a type which we know has a simple scalar argument-passing
380	// convention without any padding. (We're specifically looking for 32
381	// and 64-bit integer and integer-equivalents, float, and double.)
382	if (!Ty->getAs<BuiltinType>() && !Ty->hasPointerRepresentation() &&
383	!Ty->isEnumeralType() && !Ty->isBlockPointerType())
384	return false;
385
386	uint64_t Size = Context.getTypeSize(T: Ty);
387	return Size == 32 || Size == 64;
388	}
389
390	static bool addFieldSizes(ASTContext &Context, const RecordDecl *RD,
391	uint64_t &Size) {
392	for (const auto *FD : RD->fields()) {
393	// Scalar arguments on the stack get 4 byte alignment on x86. If the
394	// argument is smaller than 32-bits, expanding the struct will create
395	// alignment padding.
396	if (!is32Or64BitBasicType(FD->getType(), Context))
397	return false;
398
399	// FIXME: Reject bit-fields wholesale; there are two problems, we don't know
400	// how to expand them yet, and the predicate for telling if a bitfield still
401	// counts as "basic" is more complicated than what we were doing previously.
402	if (FD->isBitField())
403	return false;
404
405	Size += Context.getTypeSize(FD->getType());
406	}
407	return true;
408	}
409
410	static bool addBaseAndFieldSizes(ASTContext &Context, const CXXRecordDecl *RD,
411	uint64_t &Size) {
412	// Don't do this if there are any non-empty bases.
413	for (const CXXBaseSpecifier &Base : RD->bases()) {
414	if (!addBaseAndFieldSizes(Context, RD: Base.getType()->getAsCXXRecordDecl(),
415	Size))
416	return false;
417	}
418	if (!addFieldSizes(Context, RD, Size))
419	return false;
420	return true;
421	}
422
423	/// Test whether an argument type which is to be passed indirectly (on the
424	/// stack) would have the equivalent layout if it was expanded into separate
425	/// arguments. If so, we prefer to do the latter to avoid inhibiting
426	/// optimizations.
427	bool X86_32ABIInfo::canExpandIndirectArgument(QualType Ty) const {
428	// We can only expand structure types.
429	const RecordType *RT = Ty->getAs<RecordType>();
430	if (!RT)
431	return false;
432	const RecordDecl *RD = RT->getDecl();
433	uint64_t Size = 0;
434	if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(Val: RD)) {
435	if (!IsWin32StructABI) {
436	// On non-Windows, we have to conservatively match our old bitcode
437	// prototypes in order to be ABI-compatible at the bitcode level.
438	if (!CXXRD->isCLike())
439	return false;
440	} else {
441	// Don't do this for dynamic classes.
442	if (CXXRD->isDynamicClass())
443	return false;
444	}
445	if (!addBaseAndFieldSizes(Context&: getContext(), RD: CXXRD, Size))
446	return false;
447	} else {
448	if (!addFieldSizes(Context&: getContext(), RD, Size))
449	return false;
450	}
451
452	// We can do this if there was no alignment padding.
453	return Size == getContext().getTypeSize(T: Ty);
454	}
455
456	ABIArgInfo X86_32ABIInfo::getIndirectReturnResult(QualType RetTy, CCState &State) const {
457	// If the return value is indirect, then the hidden argument is consuming one
458	// integer register.
459	if (State.CC != llvm::CallingConv::X86_FastCall &&
460	State.CC != llvm::CallingConv::X86_VectorCall && State.FreeRegs) {
461	--State.FreeRegs;
462	if (!IsMCUABI)
463	return getNaturalAlignIndirectInReg(Ty: RetTy);
464	}
465	return getNaturalAlignIndirect(
466	Ty: RetTy, /*AddrSpace=*/getDataLayout().getAllocaAddrSpace(),
467	/*ByVal=*/false);
468	}
469
470	ABIArgInfo X86_32ABIInfo::classifyReturnType(QualType RetTy,
471	CCState &State) const {
472	if (RetTy->isVoidType())
473	return ABIArgInfo::getIgnore();
474
475	const Type *Base = nullptr;
476	uint64_t NumElts = 0;
477	if ((State.CC == llvm::CallingConv::X86_VectorCall ||
478	State.CC == llvm::CallingConv::X86_RegCall) &&
479	isHomogeneousAggregate(Ty: RetTy, Base, Members&: NumElts)) {
480	// The LLVM struct type for such an aggregate should lower properly.
481	return ABIArgInfo::getDirect();
482	}
483
484	if (const VectorType *VT = RetTy->getAs<VectorType>()) {
485	// On Darwin, some vectors are returned in registers.
486	if (IsDarwinVectorABI) {
487	uint64_t Size = getContext().getTypeSize(T: RetTy);
488
489	// 128-bit vectors are a special case; they are returned in
490	// registers and we need to make sure to pick a type the LLVM
491	// backend will like.
492	if (Size == 128)
493	return ABIArgInfo::getDirect(T: llvm::FixedVectorType::get(
494	ElementType: llvm::Type::getInt64Ty(C&: getVMContext()), NumElts: 2));
495
496	// Always return in register if it fits in a general purpose
497	// register, or if it is 64 bits and has a single element.
498	if ((Size == 8 || Size == 16 || Size == 32) ||
499	(Size == 64 && VT->getNumElements() == 1))
500	return ABIArgInfo::getDirect(T: llvm::IntegerType::get(C&: getVMContext(),
501	NumBits: Size));
502
503	return getIndirectReturnResult(RetTy, State);
504	}
505
506	return ABIArgInfo::getDirect();
507	}
508
509	if (isAggregateTypeForABI(T: RetTy)) {
510	if (const RecordType *RT = RetTy->getAs<RecordType>()) {
511	// Structures with flexible arrays are always indirect.
512	if (RT->getDecl()->hasFlexibleArrayMember())
513	return getIndirectReturnResult(RetTy, State);
514	}
515
516	// If specified, structs and unions are always indirect.
517	if (!IsRetSmallStructInRegABI && !RetTy->isAnyComplexType())
518	return getIndirectReturnResult(RetTy, State);
519
520	// Ignore empty structs/unions.
521	if (isEmptyRecord(Context&: getContext(), T: RetTy, AllowArrays: true))
522	return ABIArgInfo::getIgnore();
523
524	// Return complex of _Float16 as <2 x half> so the backend will use xmm0.
525	if (const ComplexType *CT = RetTy->getAs<ComplexType>()) {
526	QualType ET = getContext().getCanonicalType(T: CT->getElementType());
527	if (ET->isFloat16Type())
528	return ABIArgInfo::getDirect(T: llvm::FixedVectorType::get(
529	ElementType: llvm::Type::getHalfTy(C&: getVMContext()), NumElts: 2));
530	}
531
532	// Small structures which are register sized are generally returned
533	// in a register.
534	if (shouldReturnTypeInRegister(Ty: RetTy, Context&: getContext())) {
535	uint64_t Size = getContext().getTypeSize(T: RetTy);
536
537	// As a special-case, if the struct is a "single-element" struct, and
538	// the field is of type "float" or "double", return it in a
539	// floating-point register. (MSVC does not apply this special case.)
540	// We apply a similar transformation for pointer types to improve the
541	// quality of the generated IR.
542	if (const Type *SeltTy = isSingleElementStruct(T: RetTy, Context&: getContext()))
543	if ((!IsWin32StructABI && SeltTy->isRealFloatingType())
544	|| SeltTy->hasPointerRepresentation())
545	return ABIArgInfo::getDirect(T: CGT.ConvertType(T: QualType(SeltTy, 0)));
546
547	// FIXME: We should be able to narrow this integer in cases with dead
548	// padding.
549	return ABIArgInfo::getDirect(T: llvm::IntegerType::get(C&: getVMContext(),NumBits: Size));
550	}
551
552	return getIndirectReturnResult(RetTy, State);
553	}
554
555	// Treat an enum type as its underlying type.
556	if (const EnumType *EnumTy = RetTy->getAs<EnumType>())
557	RetTy = EnumTy->getDecl()->getIntegerType();
558
559	if (const auto *EIT = RetTy->getAs<BitIntType>())
560	if (EIT->getNumBits() > 64)
561	return getIndirectReturnResult(RetTy, State);
562
563	return (isPromotableIntegerTypeForABI(Ty: RetTy) ? ABIArgInfo::getExtend(Ty: RetTy)
564	: ABIArgInfo::getDirect());
565	}
566
567	unsigned X86_32ABIInfo::getTypeStackAlignInBytes(QualType Ty,
568	unsigned Align) const {
569	// Otherwise, if the alignment is less than or equal to the minimum ABI
570	// alignment, just use the default; the backend will handle this.
571	if (Align <= MinABIStackAlignInBytes)
572	return 0; // Use default alignment.
573
574	if (IsLinuxABI) {
575	// Exclude other System V OS (e.g Darwin, PS4 and FreeBSD) since we don't
576	// want to spend any effort dealing with the ramifications of ABI breaks.
577	//
578	// If the vector type is __m128/__m256/__m512, return the default alignment.
579	if (Ty->isVectorType() && (Align == 16 || Align == 32 || Align == 64))
580	return Align;
581	}
582	// On non-Darwin, the stack type alignment is always 4.
583	if (!IsDarwinVectorABI) {
584	// Set explicit alignment, since we may need to realign the top.
585	return MinABIStackAlignInBytes;
586	}
587
588	// Otherwise, if the type contains an SSE vector type, the alignment is 16.
589	if (Align >= 16 && (isSIMDVectorType(Context&: getContext(), Ty) ||
590	isRecordWithSIMDVectorType(Context&: getContext(), Ty)))
591	return 16;
592
593	return MinABIStackAlignInBytes;
594	}
595
596	ABIArgInfo X86_32ABIInfo::getIndirectResult(QualType Ty, bool ByVal,
597	CCState &State) const {
598	if (!ByVal) {
599	if (State.FreeRegs) {
600	--State.FreeRegs; // Non-byval indirects just use one pointer.
601	if (!IsMCUABI)
602	return getNaturalAlignIndirectInReg(Ty);
603	}
604	return getNaturalAlignIndirect(Ty, AddrSpace: getDataLayout().getAllocaAddrSpace(),
605	ByVal: false);
606	}
607
608	// Compute the byval alignment.
609	unsigned TypeAlign = getContext().getTypeAlign(T: Ty) / 8;
610	unsigned StackAlign = getTypeStackAlignInBytes(Ty, Align: TypeAlign);
611	if (StackAlign == 0)
612	return ABIArgInfo::getIndirect(
613	Alignment: CharUnits::fromQuantity(Quantity: 4),
614	/*AddrSpace=*/getDataLayout().getAllocaAddrSpace(),
615	/*ByVal=*/true);
616
617	// If the stack alignment is less than the type alignment, realign the
618	// argument.
619	bool Realign = TypeAlign > StackAlign;
620	return ABIArgInfo::getIndirect(
621	Alignment: CharUnits::fromQuantity(Quantity: StackAlign),
622	/*AddrSpace=*/getDataLayout().getAllocaAddrSpace(), /*ByVal=*/true,
623	Realign);
624	}
625
626	X86_32ABIInfo::Class X86_32ABIInfo::classify(QualType Ty) const {
627	const Type *T = isSingleElementStruct(T: Ty, Context&: getContext());
628	if (!T)
629	T = Ty.getTypePtr();
630
631	if (const BuiltinType *BT = T->getAs<BuiltinType>()) {
632	BuiltinType::Kind K = BT->getKind();
633	if (K == BuiltinType::Float || K == BuiltinType::Double)
634	return Float;
635	}
636	return Integer;
637	}
638
639	bool X86_32ABIInfo::updateFreeRegs(QualType Ty, CCState &State) const {
640	if (!IsSoftFloatABI) {
641	Class C = classify(Ty);
642	if (C == Float)
643	return false;
644	}
645
646	unsigned Size = getContext().getTypeSize(T: Ty);
647	unsigned SizeInRegs = (Size + 31) / 32;
648
649	if (SizeInRegs == 0)
650	return false;
651
652	if (!IsMCUABI) {
653	if (SizeInRegs > State.FreeRegs) {
654	State.FreeRegs = 0;
655	return false;
656	}
657	} else {
658	// The MCU psABI allows passing parameters in-reg even if there are
659	// earlier parameters that are passed on the stack. Also,
660	// it does not allow passing >8-byte structs in-register,
661	// even if there are 3 free registers available.
662	if (SizeInRegs > State.FreeRegs || SizeInRegs > 2)
663	return false;
664	}
665
666	State.FreeRegs -= SizeInRegs;
667	return true;
668	}
669
670	bool X86_32ABIInfo::shouldAggregateUseDirect(QualType Ty, CCState &State,
671	bool &InReg,
672	bool &NeedsPadding) const {
673	// On Windows, aggregates other than HFAs are never passed in registers, and
674	// they do not consume register slots. Homogenous floating-point aggregates
675	// (HFAs) have already been dealt with at this point.
676	if (IsWin32StructABI && isAggregateTypeForABI(T: Ty))
677	return false;
678
679	NeedsPadding = false;
680	InReg = !IsMCUABI;
681
682	if (!updateFreeRegs(Ty, State))
683	return false;
684
685	if (IsMCUABI)
686	return true;
687
688	if (State.CC == llvm::CallingConv::X86_FastCall ||
689	State.CC == llvm::CallingConv::X86_VectorCall ||
690	State.CC == llvm::CallingConv::X86_RegCall) {
691	if (getContext().getTypeSize(T: Ty) <= 32 && State.FreeRegs)
692	NeedsPadding = true;
693
694	return false;
695	}
696
697	return true;
698	}
699
700	bool X86_32ABIInfo::shouldPrimitiveUseInReg(QualType Ty, CCState &State) const {
701	bool IsPtrOrInt = (getContext().getTypeSize(T: Ty) <= 32) &&
702	(Ty->isIntegralOrEnumerationType() || Ty->isPointerType() ||
703	Ty->isReferenceType());
704
705	if (!IsPtrOrInt && (State.CC == llvm::CallingConv::X86_FastCall ||
706	State.CC == llvm::CallingConv::X86_VectorCall))
707	return false;
708
709	if (!updateFreeRegs(Ty, State))
710	return false;
711
712	if (!IsPtrOrInt && State.CC == llvm::CallingConv::X86_RegCall)
713	return false;
714
715	// Return true to apply inreg to all legal parameters except for MCU targets.
716	return !IsMCUABI;
717	}
718
719	void X86_32ABIInfo::runVectorCallFirstPass(CGFunctionInfo &FI, CCState &State) const {
720	// Vectorcall x86 works subtly different than in x64, so the format is
721	// a bit different than the x64 version. First, all vector types (not HVAs)
722	// are assigned, with the first 6 ending up in the [XYZ]MM0-5 registers.
723	// This differs from the x64 implementation, where the first 6 by INDEX get
724	// registers.
725	// In the second pass over the arguments, HVAs are passed in the remaining
726	// vector registers if possible, or indirectly by address. The address will be
727	// passed in ECX/EDX if available. Any other arguments are passed according to
728	// the usual fastcall rules.
729	MutableArrayRef<CGFunctionInfoArgInfo> Args = FI.arguments();
730	for (int I = 0, E = Args.size(); I < E; ++I) {
731	const Type *Base = nullptr;
732	uint64_t NumElts = 0;
733	const QualType &Ty = Args[I].type;
734	if ((Ty->isVectorType() || Ty->isBuiltinType()) &&
735	isHomogeneousAggregate(Ty, Base, Members&: NumElts)) {
736	if (State.FreeSSERegs >= NumElts) {
737	State.FreeSSERegs -= NumElts;
738	Args[I].info = ABIArgInfo::getDirectInReg();
739	State.IsPreassigned.set(I);
740	}
741	}
742	}
743	}
744
745	ABIArgInfo X86_32ABIInfo::classifyArgumentType(QualType Ty, CCState &State,
746	unsigned ArgIndex) const {
747	// FIXME: Set alignment on indirect arguments.
748	bool IsFastCall = State.CC == llvm::CallingConv::X86_FastCall;
749	bool IsRegCall = State.CC == llvm::CallingConv::X86_RegCall;
750	bool IsVectorCall = State.CC == llvm::CallingConv::X86_VectorCall;
751
752	Ty = useFirstFieldIfTransparentUnion(Ty);
753	TypeInfo TI = getContext().getTypeInfo(T: Ty);
754
755	// Check with the C++ ABI first.
756	const RecordType *RT = Ty->getAs<RecordType>();
757	if (RT) {
758	CGCXXABI::RecordArgABI RAA = getRecordArgABI(RT, CXXABI&: getCXXABI());
759	if (RAA == CGCXXABI::RAA_Indirect) {
760	return getIndirectResult(Ty, ByVal: false, State);
761	} else if (State.IsDelegateCall) {
762	// Avoid having different alignments on delegate call args by always
763	// setting the alignment to 4, which is what we do for inallocas.
764	ABIArgInfo Res = getIndirectResult(Ty, ByVal: false, State);
765	Res.setIndirectAlign(CharUnits::fromQuantity(Quantity: 4));
766	return Res;
767	} else if (RAA == CGCXXABI::RAA_DirectInMemory) {
768	// The field index doesn't matter, we'll fix it up later.
769	return ABIArgInfo::getInAlloca(/*FieldIndex=*/0);
770	}
771	}
772
773	// Regcall uses the concept of a homogenous vector aggregate, similar
774	// to other targets.
775	const Type *Base = nullptr;
776	uint64_t NumElts = 0;
777	if ((IsRegCall || IsVectorCall) &&
778	isHomogeneousAggregate(Ty, Base, Members&: NumElts)) {
779	if (State.FreeSSERegs >= NumElts) {
780	State.FreeSSERegs -= NumElts;
781
782	// Vectorcall passes HVAs directly and does not flatten them, but regcall
783	// does.
784	if (IsVectorCall)
785	return getDirectX86Hva();
786
787	if (Ty->isBuiltinType() || Ty->isVectorType())
788	return ABIArgInfo::getDirect();
789	return ABIArgInfo::getExpand();
790	}
791	if (IsVectorCall && Ty->isBuiltinType())
792	return ABIArgInfo::getDirect();
793	return getIndirectResult(Ty, /*ByVal=*/false, State);
794	}
795
796	if (isAggregateTypeForABI(T: Ty)) {
797	// Structures with flexible arrays are always indirect.
798	// FIXME: This should not be byval!
799	if (RT && RT->getDecl()->hasFlexibleArrayMember())
800	return getIndirectResult(Ty, ByVal: true, State);
801
802	// Ignore empty structs/unions on non-Windows.
803	if (!IsWin32StructABI && isEmptyRecord(Context&: getContext(), T: Ty, AllowArrays: true))
804	return ABIArgInfo::getIgnore();
805
806	// Ignore 0 sized structs.
807	if (TI.Width == 0)
808	return ABIArgInfo::getIgnore();
809
810	llvm::LLVMContext &LLVMContext = getVMContext();
811	llvm::IntegerType *Int32 = llvm::Type::getInt32Ty(C&: LLVMContext);
812	bool NeedsPadding = false;
813	bool InReg;
814	if (shouldAggregateUseDirect(Ty, State, InReg, NeedsPadding)) {
815	unsigned SizeInRegs = (TI.Width + 31) / 32;
816	SmallVector<llvm::Type*, 3> Elements(SizeInRegs, Int32);
817	llvm::Type *Result = llvm::StructType::get(Context&: LLVMContext, Elements);
818	if (InReg)
819	return ABIArgInfo::getDirectInReg(T: Result);
820	else
821	return ABIArgInfo::getDirect(T: Result);
822	}
823	llvm::IntegerType *PaddingType = NeedsPadding ? Int32 : nullptr;
824
825	// Pass over-aligned aggregates to non-variadic functions on Windows
826	// indirectly. This behavior was added in MSVC 2015. Use the required
827	// alignment from the record layout, since that may be less than the
828	// regular type alignment, and types with required alignment of less than 4
829	// bytes are not passed indirectly.
830	if (IsWin32StructABI && State.Required.isRequiredArg(argIdx: ArgIndex)) {
831	unsigned AlignInBits = 0;
832	if (RT) {
833	const ASTRecordLayout &Layout =
834	getContext().getASTRecordLayout(D: RT->getDecl());
835	AlignInBits = getContext().toBits(CharSize: Layout.getRequiredAlignment());
836	} else if (TI.isAlignRequired()) {
837	AlignInBits = TI.Align;
838	}
839	if (AlignInBits > 32)
840	return getIndirectResult(Ty, /*ByVal=*/false, State);
841	}
842
843	// Expand small (<= 128-bit) record types when we know that the stack layout
844	// of those arguments will match the struct. This is important because the
845	// LLVM backend isn't smart enough to remove byval, which inhibits many
846	// optimizations.
847	// Don't do this for the MCU if there are still free integer registers
848	// (see X86_64 ABI for full explanation).
849	if (TI.Width <= 4 * 32 && (!IsMCUABI || State.FreeRegs == 0) &&
850	canExpandIndirectArgument(Ty))
851	return ABIArgInfo::getExpandWithPadding(
852	PaddingInReg: IsFastCall || IsVectorCall || IsRegCall, Padding: PaddingType);
853
854	return getIndirectResult(Ty, ByVal: true, State);
855	}
856
857	if (const VectorType *VT = Ty->getAs<VectorType>()) {
858	// On Windows, vectors are passed directly if registers are available, or
859	// indirectly if not. This avoids the need to align argument memory. Pass
860	// user-defined vector types larger than 512 bits indirectly for simplicity.
861	if (IsWin32StructABI) {
862	if (TI.Width <= 512 && State.FreeSSERegs > 0) {
863	--State.FreeSSERegs;
864	return ABIArgInfo::getDirectInReg();
865	}
866	return getIndirectResult(Ty, /*ByVal=*/false, State);
867	}
868
869	// On Darwin, some vectors are passed in memory, we handle this by passing
870	// it as an i8/i16/i32/i64.
871	if (IsDarwinVectorABI) {
872	if ((TI.Width == 8 || TI.Width == 16 || TI.Width == 32) ||
873	(TI.Width == 64 && VT->getNumElements() == 1))
874	return ABIArgInfo::getDirect(
875	T: llvm::IntegerType::get(C&: getVMContext(), NumBits: TI.Width));
876	}
877
878	if (IsX86_MMXType(IRType: CGT.ConvertType(T: Ty)))
879	return ABIArgInfo::getDirect(T: llvm::IntegerType::get(C&: getVMContext(), NumBits: 64));
880
881	return ABIArgInfo::getDirect();
882	}
883
884
885	if (const EnumType *EnumTy = Ty->getAs<EnumType>())
886	Ty = EnumTy->getDecl()->getIntegerType();
887
888	bool InReg = shouldPrimitiveUseInReg(Ty, State);
889
890	if (isPromotableIntegerTypeForABI(Ty)) {
891	if (InReg)
892	return ABIArgInfo::getExtendInReg(Ty, T: CGT.ConvertType(T: Ty));
893	return ABIArgInfo::getExtend(Ty, T: CGT.ConvertType(T: Ty));
894	}
895
896	if (const auto *EIT = Ty->getAs<BitIntType>()) {
897	if (EIT->getNumBits() <= 64) {
898	if (InReg)
899	return ABIArgInfo::getDirectInReg();
900	return ABIArgInfo::getDirect();
901	}
902	return getIndirectResult(Ty, /*ByVal=*/false, State);
903	}
904
905	if (InReg)
906	return ABIArgInfo::getDirectInReg();
907	return ABIArgInfo::getDirect();
908	}
909
910	void X86_32ABIInfo::computeInfo(CGFunctionInfo &FI) const {
911	CCState State(FI);
912	if (IsMCUABI)
913	State.FreeRegs = 3;
914	else if (State.CC == llvm::CallingConv::X86_FastCall) {
915	State.FreeRegs = 2;
916	State.FreeSSERegs = 3;
917	} else if (State.CC == llvm::CallingConv::X86_VectorCall) {
918	State.FreeRegs = 2;
919	State.FreeSSERegs = 6;
920	} else if (FI.getHasRegParm())
921	State.FreeRegs = FI.getRegParm();
922	else if (State.CC == llvm::CallingConv::X86_RegCall) {
923	State.FreeRegs = 5;
924	State.FreeSSERegs = 8;
925	} else if (IsWin32StructABI) {
926	// Since MSVC 2015, the first three SSE vectors have been passed in
927	// registers. The rest are passed indirectly.
928	State.FreeRegs = DefaultNumRegisterParameters;
929	State.FreeSSERegs = 3;
930	} else
931	State.FreeRegs = DefaultNumRegisterParameters;
932
933	if (!::classifyReturnType(CXXABI: getCXXABI(), FI, Info: *this)) {
934	FI.getReturnInfo() = classifyReturnType(RetTy: FI.getReturnType(), State);
935	} else if (FI.getReturnInfo().isIndirect()) {
936	// The C++ ABI is not aware of register usage, so we have to check if the
937	// return value was sret and put it in a register ourselves if appropriate.
938	if (State.FreeRegs) {
939	--State.FreeRegs; // The sret parameter consumes a register.
940	if (!IsMCUABI)
941	FI.getReturnInfo().setInReg(true);
942	}
943	}
944
945	// The chain argument effectively gives us another free register.
946	if (FI.isChainCall())
947	++State.FreeRegs;
948
949	// For vectorcall, do a first pass over the arguments, assigning FP and vector
950	// arguments to XMM registers as available.
951	if (State.CC == llvm::CallingConv::X86_VectorCall)
952	runVectorCallFirstPass(FI, State);
953
954	bool UsedInAlloca = false;
955	MutableArrayRef<CGFunctionInfoArgInfo> Args = FI.arguments();
956	for (unsigned I = 0, E = Args.size(); I < E; ++I) {
957	// Skip arguments that have already been assigned.
958	if (State.IsPreassigned.test(Idx: I))
959	continue;
960
961	Args[I].info =
962	classifyArgumentType(Ty: Args[I].type, State, ArgIndex: I);
963	UsedInAlloca |= (Args[I].info.getKind() == ABIArgInfo::InAlloca);
964	}
965
966	// If we needed to use inalloca for any argument, do a second pass and rewrite
967	// all the memory arguments to use inalloca.
968	if (UsedInAlloca)
969	rewriteWithInAlloca(FI);
970	}
971
972	void
973	X86_32ABIInfo::addFieldToArgStruct(SmallVector<llvm::Type *, 6> &FrameFields,
974	CharUnits &StackOffset, ABIArgInfo &Info,
975	QualType Type) const {
976	// Arguments are always 4-byte-aligned.
977	CharUnits WordSize = CharUnits::fromQuantity(Quantity: 4);
978	assert(StackOffset.isMultipleOf(WordSize) && "unaligned inalloca struct");
979
980	// sret pointers and indirect things will require an extra pointer
981	// indirection, unless they are byval. Most things are byval, and will not
982	// require this indirection.
983	bool IsIndirect = false;
984	if (Info.isIndirect() && !Info.getIndirectByVal())
985	IsIndirect = true;
986	Info = ABIArgInfo::getInAlloca(FieldIndex: FrameFields.size(), Indirect: IsIndirect);
987	llvm::Type *LLTy = CGT.ConvertTypeForMem(T: Type);
988	if (IsIndirect)
989	LLTy = llvm::PointerType::getUnqual(C&: getVMContext());
990	FrameFields.push_back(Elt: LLTy);
991	StackOffset += IsIndirect ? WordSize : getContext().getTypeSizeInChars(T: Type);
992
993	// Insert padding bytes to respect alignment.
994	CharUnits FieldEnd = StackOffset;
995	StackOffset = FieldEnd.alignTo(Align: WordSize);
996	if (StackOffset != FieldEnd) {
997	CharUnits NumBytes = StackOffset - FieldEnd;
998	llvm::Type *Ty = llvm::Type::getInt8Ty(C&: getVMContext());
999	Ty = llvm::ArrayType::get(ElementType: Ty, NumElements: NumBytes.getQuantity());
1000	FrameFields.push_back(Elt: Ty);
1001	}
1002	}
1003
1004	static bool isArgInAlloca(const ABIArgInfo &Info) {
1005	// Leave ignored and inreg arguments alone.
1006	switch (Info.getKind()) {
1007	case ABIArgInfo::InAlloca:
1008	return true;
1009	case ABIArgInfo::Ignore:
1010	case ABIArgInfo::IndirectAliased:
1011	return false;
1012	case ABIArgInfo::Indirect:
1013	case ABIArgInfo::Direct:
1014	case ABIArgInfo::Extend:
1015	return !Info.getInReg();
1016	case ABIArgInfo::Expand:
1017	case ABIArgInfo::CoerceAndExpand:
1018	// These are aggregate types which are never passed in registers when
1019	// inalloca is involved.
1020	return true;
1021	}
1022	llvm_unreachable("invalid enum");
1023	}
1024
1025	void X86_32ABIInfo::rewriteWithInAlloca(CGFunctionInfo &FI) const {
1026	assert(IsWin32StructABI && "inalloca only supported on win32");
1027
1028	// Build a packed struct type for all of the arguments in memory.
1029	SmallVector<llvm::Type *, 6> FrameFields;
1030
1031	// The stack alignment is always 4.
1032	CharUnits StackAlign = CharUnits::fromQuantity(Quantity: 4);
1033
1034	CharUnits StackOffset;
1035	CGFunctionInfo::arg_iterator I = FI.arg_begin(), E = FI.arg_end();
1036
1037	// Put 'this' into the struct before 'sret', if necessary.
1038	bool IsThisCall =
1039	FI.getCallingConvention() == llvm::CallingConv::X86_ThisCall;
1040	ABIArgInfo &Ret = FI.getReturnInfo();
1041	if (Ret.isIndirect() && Ret.isSRetAfterThis() && !IsThisCall &&
1042	isArgInAlloca(Info: I->info)) {
1043	addFieldToArgStruct(FrameFields, StackOffset, Info&: I->info, Type: I->type);
1044	++I;
1045	}
1046
1047	// Put the sret parameter into the inalloca struct if it's in memory.
1048	if (Ret.isIndirect() && !Ret.getInReg()) {
1049	addFieldToArgStruct(FrameFields, StackOffset, Info&: Ret, Type: FI.getReturnType());
1050	// On Windows, the hidden sret parameter is always returned in eax.
1051	Ret.setInAllocaSRet(IsWin32StructABI);
1052	}
1053
1054	// Skip the 'this' parameter in ecx.
1055	if (IsThisCall)
1056	++I;
1057
1058	// Put arguments passed in memory into the struct.
1059	for (; I != E; ++I) {
1060	if (isArgInAlloca(Info: I->info))
1061	addFieldToArgStruct(FrameFields, StackOffset, Info&: I->info, Type: I->type);
1062	}
1063
1064	FI.setArgStruct(Ty: llvm::StructType::get(Context&: getVMContext(), Elements: FrameFields,
1065	/*isPacked=*/true),
1066	Align: StackAlign);
1067	}
1068
1069	RValue X86_32ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
1070	QualType Ty, AggValueSlot Slot) const {
1071
1072	auto TypeInfo = getContext().getTypeInfoInChars(T: Ty);
1073
1074	CCState State(*const_cast<CGFunctionInfo *>(CGF.CurFnInfo));
1075	ABIArgInfo AI = classifyArgumentType(Ty, State, /*ArgIndex*/ 0);
1076	// Empty records are ignored for parameter passing purposes.
1077	if (AI.isIgnore())
1078	return Slot.asRValue();
1079
1080	// x86-32 changes the alignment of certain arguments on the stack.
1081	//
1082	// Just messing with TypeInfo like this works because we never pass
1083	// anything indirectly.
1084	TypeInfo.Align = CharUnits::fromQuantity(
1085	Quantity: getTypeStackAlignInBytes(Ty, Align: TypeInfo.Align.getQuantity()));
1086
1087	return emitVoidPtrVAArg(CGF, VAListAddr, ValueTy: Ty, /*Indirect*/ IsIndirect: false, ValueInfo: TypeInfo,
1088	SlotSizeAndAlign: CharUnits::fromQuantity(Quantity: 4),
1089	/*AllowHigherAlign*/ true, Slot);
1090	}
1091
1092	bool X86_32TargetCodeGenInfo::isStructReturnInRegABI(
1093	const llvm::Triple &Triple, const CodeGenOptions &Opts) {
1094	assert(Triple.getArch() == llvm::Triple::x86);
1095
1096	switch (Opts.getStructReturnConvention()) {
1097	case CodeGenOptions::SRCK_Default:
1098	break;
1099	case CodeGenOptions::SRCK_OnStack: // -fpcc-struct-return
1100	return false;
1101	case CodeGenOptions::SRCK_InRegs: // -freg-struct-return
1102	return true;
1103	}
1104
1105	if (Triple.isOSDarwin() || Triple.isOSIAMCU())
1106	return true;
1107
1108	switch (Triple.getOS()) {
1109	case llvm::Triple::DragonFly:
1110	case llvm::Triple::FreeBSD:
1111	case llvm::Triple::OpenBSD:
1112	case llvm::Triple::Win32:
1113	return true;
1114	default:
1115	return false;
1116	}
1117	}
1118
1119	static void addX86InterruptAttrs(const FunctionDecl *FD, llvm::GlobalValue *GV,
1120	CodeGen::CodeGenModule &CGM) {
1121	if (!FD->hasAttr<AnyX86InterruptAttr>())
1122	return;
1123
1124	llvm::Function *Fn = cast<llvm::Function>(Val: GV);
1125	Fn->setCallingConv(llvm::CallingConv::X86_INTR);
1126	if (FD->getNumParams() == 0)
1127	return;
1128
1129	auto PtrTy = cast<PointerType>(FD->getParamDecl(i: 0)->getType());
1130	llvm::Type *ByValTy = CGM.getTypes().ConvertType(T: PtrTy->getPointeeType());
1131	llvm::Attribute NewAttr = llvm::Attribute::getWithByValType(
1132	Context&: Fn->getContext(), Ty: ByValTy);
1133	Fn->addParamAttr(ArgNo: 0, Attr: NewAttr);
1134	}
1135
1136	void X86_32TargetCodeGenInfo::setTargetAttributes(
1137	const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &CGM) const {
1138	if (GV->isDeclaration())
1139	return;
1140	if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(Val: D)) {
1141	if (FD->hasAttr<X86ForceAlignArgPointerAttr>()) {
1142	llvm::Function *Fn = cast<llvm::Function>(Val: GV);
1143	Fn->addFnAttr(Kind: "stackrealign");
1144	}
1145
1146	addX86InterruptAttrs(FD, GV, CGM);
1147	}
1148	}
1149
1150	bool X86_32TargetCodeGenInfo::initDwarfEHRegSizeTable(
1151	CodeGen::CodeGenFunction &CGF,
1152	llvm::Value *Address) const {
1153	CodeGen::CGBuilderTy &Builder = CGF.Builder;
1154
1155	llvm::Value *Four8 = llvm::ConstantInt::get(Ty: CGF.Int8Ty, V: 4);
1156
1157	// 0-7 are the eight integer registers; the order is different
1158	// on Darwin (for EH), but the range is the same.
1159	// 8 is %eip.
1160	AssignToArrayRange(Builder, Array: Address, Value: Four8, FirstIndex: 0, LastIndex: 8);
1161
1162	if (CGF.CGM.getTarget().getTriple().isOSDarwin()) {
1163	// 12-16 are st(0..4). Not sure why we stop at 4.
1164	// These have size 16, which is sizeof(long double) on
1165	// platforms with 8-byte alignment for that type.
1166	llvm::Value *Sixteen8 = llvm::ConstantInt::get(Ty: CGF.Int8Ty, V: 16);
1167	AssignToArrayRange(Builder, Array: Address, Value: Sixteen8, FirstIndex: 12, LastIndex: 16);
1168
1169	} else {
1170	// 9 is %eflags, which doesn't get a size on Darwin for some
1171	// reason.
1172	Builder.CreateAlignedStore(
1173	Val: Four8, Addr: Builder.CreateConstInBoundsGEP1_32(Ty: CGF.Int8Ty, Ptr: Address, Idx0: 9),
1174	Align: CharUnits::One());
1175
1176	// 11-16 are st(0..5). Not sure why we stop at 5.
1177	// These have size 12, which is sizeof(long double) on
1178	// platforms with 4-byte alignment for that type.
1179	llvm::Value *Twelve8 = llvm::ConstantInt::get(Ty: CGF.Int8Ty, V: 12);
1180	AssignToArrayRange(Builder, Array: Address, Value: Twelve8, FirstIndex: 11, LastIndex: 16);
1181	}
1182
1183	return false;
1184	}
1185
1186	//===----------------------------------------------------------------------===//
1187	// X86-64 ABI Implementation
1188	//===----------------------------------------------------------------------===//
1189
1190
1191	namespace {
1192
1193	/// \p returns the size in bits of the largest (native) vector for \p AVXLevel.
1194	static unsigned getNativeVectorSizeForAVXABI(X86AVXABILevel AVXLevel) {
1195	switch (AVXLevel) {
1196	case X86AVXABILevel::AVX512:
1197	return 512;
1198	case X86AVXABILevel::AVX:
1199	return 256;
1200	case X86AVXABILevel::None:
1201	return 128;
1202	}
1203	llvm_unreachable("Unknown AVXLevel");
1204	}
1205
1206	/// X86_64ABIInfo - The X86_64 ABI information.
1207	class X86_64ABIInfo : public ABIInfo {
1208	enum Class {
1209	Integer = 0,
1210	SSE,
1211	SSEUp,
1212	X87,
1213	X87Up,
1214	ComplexX87,
1215	NoClass,
1216	Memory
1217	};
1218
1219	/// merge - Implement the X86_64 ABI merging algorithm.
1220	///
1221	/// Merge an accumulating classification \arg Accum with a field
1222	/// classification \arg Field.
1223	///
1224	/// \param Accum - The accumulating classification. This should
1225	/// always be either NoClass or the result of a previous merge
1226	/// call. In addition, this should never be Memory (the caller
1227	/// should just return Memory for the aggregate).
1228	static Class merge(Class Accum, Class Field);
1229
1230	/// postMerge - Implement the X86_64 ABI post merging algorithm.
1231	///
1232	/// Post merger cleanup, reduces a malformed Hi and Lo pair to
1233	/// final MEMORY or SSE classes when necessary.
1234	///
1235	/// \param AggregateSize - The size of the current aggregate in
1236	/// the classification process.
1237	///
1238	/// \param Lo - The classification for the parts of the type
1239	/// residing in the low word of the containing object.
1240	///
1241	/// \param Hi - The classification for the parts of the type
1242	/// residing in the higher words of the containing object.
1243	///
1244	void postMerge(unsigned AggregateSize, Class &Lo, Class &Hi) const;
1245
1246	/// classify - Determine the x86_64 register classes in which the
1247	/// given type T should be passed.
1248	///
1249	/// \param Lo - The classification for the parts of the type
1250	/// residing in the low word of the containing object.
1251	///
1252	/// \param Hi - The classification for the parts of the type
1253	/// residing in the high word of the containing object.
1254	///
1255	/// \param OffsetBase - The bit offset of this type in the
1256	/// containing object. Some parameters are classified different
1257	/// depending on whether they straddle an eightbyte boundary.
1258	///
1259	/// \param isNamedArg - Whether the argument in question is a "named"
1260	/// argument, as used in AMD64-ABI 3.5.7.
1261	///
1262	/// \param IsRegCall - Whether the calling conversion is regcall.
1263	///
1264	/// If a word is unused its result will be NoClass; if a type should
1265	/// be passed in Memory then at least the classification of \arg Lo
1266	/// will be Memory.
1267	///
1268	/// The \arg Lo class will be NoClass iff the argument is ignored.
1269	///
1270	/// If the \arg Lo class is ComplexX87, then the \arg Hi class will
1271	/// also be ComplexX87.
1272	void classify(QualType T, uint64_t OffsetBase, Class &Lo, Class &Hi,
1273	bool isNamedArg, bool IsRegCall = false) const;
1274
1275	llvm::Type *GetByteVectorType(QualType Ty) const;
1276	llvm::Type *GetSSETypeAtOffset(llvm::Type *IRType,
1277	unsigned IROffset, QualType SourceTy,
1278	unsigned SourceOffset) const;
1279	llvm::Type *GetINTEGERTypeAtOffset(llvm::Type *IRType,
1280	unsigned IROffset, QualType SourceTy,
1281	unsigned SourceOffset) const;
1282
1283	/// getIndirectResult - Give a source type \arg Ty, return a suitable result
1284	/// such that the argument will be returned in memory.
1285	ABIArgInfo getIndirectReturnResult(QualType Ty) const;
1286
1287	/// getIndirectResult - Give a source type \arg Ty, return a suitable result
1288	/// such that the argument will be passed in memory.
1289	///
1290	/// \param freeIntRegs - The number of free integer registers remaining
1291	/// available.
1292	ABIArgInfo getIndirectResult(QualType Ty, unsigned freeIntRegs) const;
1293
1294	ABIArgInfo classifyReturnType(QualType RetTy) const;
1295
1296	ABIArgInfo classifyArgumentType(QualType Ty, unsigned freeIntRegs,
1297	unsigned &neededInt, unsigned &neededSSE,
1298	bool isNamedArg,
1299	bool IsRegCall = false) const;
1300
1301	ABIArgInfo classifyRegCallStructType(QualType Ty, unsigned &NeededInt,
1302	unsigned &NeededSSE,
1303	unsigned &MaxVectorWidth) const;
1304
1305	ABIArgInfo classifyRegCallStructTypeImpl(QualType Ty, unsigned &NeededInt,
1306	unsigned &NeededSSE,
1307	unsigned &MaxVectorWidth) const;
1308
1309	bool IsIllegalVectorType(QualType Ty) const;
1310
1311	/// The 0.98 ABI revision clarified a lot of ambiguities,
1312	/// unfortunately in ways that were not always consistent with
1313	/// certain previous compilers. In particular, platforms which
1314	/// required strict binary compatibility with older versions of GCC
1315	/// may need to exempt themselves.
1316	bool honorsRevision0_98() const {
1317	return !getTarget().getTriple().isOSDarwin();
1318	}
1319
1320	/// GCC classifies <1 x long long> as SSE but some platform ABIs choose to
1321	/// classify it as INTEGER (for compatibility with older clang compilers).
1322	bool classifyIntegerMMXAsSSE() const {
1323	// Clang <= 3.8 did not do this.
1324	if (getContext().getLangOpts().getClangABICompat() <=
1325	LangOptions::ClangABI::Ver3_8)
1326	return false;
1327
1328	const llvm::Triple &Triple = getTarget().getTriple();
1329	if (Triple.isOSDarwin() || Triple.isPS() || Triple.isOSFreeBSD())
1330	return false;
1331	return true;
1332	}
1333
1334	// GCC classifies vectors of __int128 as memory.
1335	bool passInt128VectorsInMem() const {
1336	// Clang <= 9.0 did not do this.
1337	if (getContext().getLangOpts().getClangABICompat() <=
1338	LangOptions::ClangABI::Ver9)
1339	return false;
1340
1341	const llvm::Triple &T = getTarget().getTriple();
1342	return T.isOSLinux() || T.isOSNetBSD();
1343	}
1344
1345	bool returnCXXRecordGreaterThan128InMem() const {
1346	// Clang <= 20.0 did not do this.
1347	if (getContext().getLangOpts().getClangABICompat() <=
1348	LangOptions::ClangABI::Ver20)
1349	return false;
1350
1351	return true;
1352	}
1353
1354	X86AVXABILevel AVXLevel;
1355	// Some ABIs (e.g. X32 ABI and Native Client OS) use 32 bit pointers on
1356	// 64-bit hardware.
1357	bool Has64BitPointers;
1358
1359	public:
1360	X86_64ABIInfo(CodeGen::CodeGenTypes &CGT, X86AVXABILevel AVXLevel)
1361	: ABIInfo(CGT), AVXLevel(AVXLevel),
1362	Has64BitPointers(CGT.getDataLayout().getPointerSize(AS: 0) == 8) {}
1363
1364	bool isPassedUsingAVXType(QualType type) const {
1365	unsigned neededInt, neededSSE;
1366	// The freeIntRegs argument doesn't matter here.
1367	ABIArgInfo info = classifyArgumentType(Ty: type, freeIntRegs: 0, neededInt, neededSSE,
1368	/*isNamedArg*/true);
1369	if (info.isDirect()) {
1370	llvm::Type *ty = info.getCoerceToType();
1371	if (llvm::VectorType *vectorTy = dyn_cast_or_null<llvm::VectorType>(Val: ty))
1372	return vectorTy->getPrimitiveSizeInBits().getFixedValue() > 128;
1373	}
1374	return false;
1375	}
1376
1377	void computeInfo(CGFunctionInfo &FI) const override;
1378
1379	RValue EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, QualType Ty,
1380	AggValueSlot Slot) const override;
1381	RValue EmitMSVAArg(CodeGenFunction &CGF, Address VAListAddr, QualType Ty,
1382	AggValueSlot Slot) const override;
1383
1384	bool has64BitPointers() const {
1385	return Has64BitPointers;
1386	}
1387	};
1388
1389	/// WinX86_64ABIInfo - The Windows X86_64 ABI information.
1390	class WinX86_64ABIInfo : public ABIInfo {
1391	public:
1392	WinX86_64ABIInfo(CodeGen::CodeGenTypes &CGT, X86AVXABILevel AVXLevel)
1393	: ABIInfo(CGT), AVXLevel(AVXLevel),
1394	IsMingw64(getTarget().getTriple().isWindowsGNUEnvironment()) {}
1395
1396	void computeInfo(CGFunctionInfo &FI) const override;
1397
1398	RValue EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, QualType Ty,
1399	AggValueSlot Slot) const override;
1400
1401	bool isHomogeneousAggregateBaseType(QualType Ty) const override {
1402	// FIXME: Assumes vectorcall is in use.
1403	return isX86VectorTypeForVectorCall(Context&: getContext(), Ty);
1404	}
1405
1406	bool isHomogeneousAggregateSmallEnough(const Type *Ty,
1407	uint64_t NumMembers) const override {
1408	// FIXME: Assumes vectorcall is in use.
1409	return isX86VectorCallAggregateSmallEnough(NumMembers);
1410	}
1411
1412	private:
1413	ABIArgInfo classify(QualType Ty, unsigned &FreeSSERegs, bool IsReturnType,
1414	bool IsVectorCall, bool IsRegCall) const;
1415	ABIArgInfo reclassifyHvaArgForVectorCall(QualType Ty, unsigned &FreeSSERegs,
1416	const ABIArgInfo &current) const;
1417
1418	X86AVXABILevel AVXLevel;
1419
1420	bool IsMingw64;
1421	};
1422
1423	class X86_64TargetCodeGenInfo : public TargetCodeGenInfo {
1424	public:
1425	X86_64TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT, X86AVXABILevel AVXLevel)
1426	: TargetCodeGenInfo(std::make_unique<X86_64ABIInfo>(args&: CGT, args&: AVXLevel)) {
1427	SwiftInfo =
1428	std::make_unique<SwiftABIInfo>(args&: CGT, /*SwiftErrorInRegister=*/args: true);
1429	}
1430
1431	/// Disable tail call on x86-64. The epilogue code before the tail jump blocks
1432	/// autoreleaseRV/retainRV and autoreleaseRV/unsafeClaimRV optimizations.
1433	bool markARCOptimizedReturnCallsAsNoTail() const override { return true; }
1434
1435	int getDwarfEHStackPointer(CodeGen::CodeGenModule &CGM) const override {
1436	return 7;
1437	}
1438
1439	bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
1440	llvm::Value *Address) const override {
1441	llvm::Value *Eight8 = llvm::ConstantInt::get(Ty: CGF.Int8Ty, V: 8);
1442
1443	// 0-15 are the 16 integer registers.
1444	// 16 is %rip.
1445	AssignToArrayRange(Builder&: CGF.Builder, Array: Address, Value: Eight8, FirstIndex: 0, LastIndex: 16);
1446	return false;
1447	}
1448
1449	llvm::Type* adjustInlineAsmType(CodeGen::CodeGenFunction &CGF,
1450	StringRef Constraint,
1451	llvm::Type* Ty) const override {
1452	return X86AdjustInlineAsmType(CGF, Constraint, Ty);
1453	}
1454
1455	bool isNoProtoCallVariadic(const CallArgList &args,
1456	const FunctionNoProtoType *fnType) const override {
1457	// The default CC on x86-64 sets %al to the number of SSA
1458	// registers used, and GCC sets this when calling an unprototyped
1459	// function, so we override the default behavior. However, don't do
1460	// that when AVX types are involved: the ABI explicitly states it is
1461	// undefined, and it doesn't work in practice because of how the ABI
1462	// defines varargs anyway.
1463	if (fnType->getCallConv() == CC_C) {
1464	bool HasAVXType = false;
1465	for (CallArgList::const_iterator
1466	it = args.begin(), ie = args.end(); it != ie; ++it) {
1467	if (getABIInfo<X86_64ABIInfo>().isPassedUsingAVXType(type: it->Ty)) {
1468	HasAVXType = true;
1469	break;
1470	}
1471	}
1472
1473	if (!HasAVXType)
1474	return true;
1475	}
1476
1477	return TargetCodeGenInfo::isNoProtoCallVariadic(args, fnType);
1478	}
1479
1480	void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
1481	CodeGen::CodeGenModule &CGM) const override {
1482	if (GV->isDeclaration())
1483	return;
1484	if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(Val: D)) {
1485	if (FD->hasAttr<X86ForceAlignArgPointerAttr>()) {
1486	llvm::Function *Fn = cast<llvm::Function>(Val: GV);
1487	Fn->addFnAttr(Kind: "stackrealign");
1488	}
1489
1490	addX86InterruptAttrs(FD, GV, CGM);
1491	}
1492	}
1493
1494	void checkFunctionCallABI(CodeGenModule &CGM, SourceLocation CallLoc,
1495	const FunctionDecl *Caller,
1496	const FunctionDecl *Callee, const CallArgList &Args,
1497	QualType ReturnType) const override;
1498	};
1499	} // namespace
1500
1501	static void initFeatureMaps(const ASTContext &Ctx,
1502	llvm::StringMap<bool> &CallerMap,
1503	const FunctionDecl *Caller,
1504	llvm::StringMap<bool> &CalleeMap,
1505	const FunctionDecl *Callee) {
1506	if (CalleeMap.empty() && CallerMap.empty()) {
1507	// The caller is potentially nullptr in the case where the call isn't in a
1508	// function. In this case, the getFunctionFeatureMap ensures we just get
1509	// the TU level setting (since it cannot be modified by 'target'..
1510	Ctx.getFunctionFeatureMap(CallerMap, Caller);
1511	Ctx.getFunctionFeatureMap(CalleeMap, Callee);
1512	}
1513	}
1514
1515	static bool checkAVXParamFeature(DiagnosticsEngine &Diag,
1516	SourceLocation CallLoc,
1517	const llvm::StringMap<bool> &CallerMap,
1518	const llvm::StringMap<bool> &CalleeMap,
1519	QualType Ty, StringRef Feature,
1520	bool IsArgument) {
1521	bool CallerHasFeat = CallerMap.lookup(Key: Feature);
1522	bool CalleeHasFeat = CalleeMap.lookup(Key: Feature);
1523	if (!CallerHasFeat && !CalleeHasFeat)
1524	return Diag.Report(CallLoc, diag::warn_avx_calling_convention)
1525	<< IsArgument << Ty << Feature;
1526
1527	// Mixing calling conventions here is very clearly an error.
1528	if (!CallerHasFeat || !CalleeHasFeat)
1529	return Diag.Report(CallLoc, diag::err_avx_calling_convention)
1530	<< IsArgument << Ty << Feature;
1531
1532	// Else, both caller and callee have the required feature, so there is no need
1533	// to diagnose.
1534	return false;
1535	}
1536
1537	static bool checkAVX512ParamFeature(DiagnosticsEngine &Diag,
1538	SourceLocation CallLoc,
1539	const llvm::StringMap<bool> &CallerMap,
1540	const llvm::StringMap<bool> &CalleeMap,
1541	QualType Ty, bool IsArgument) {
1542	bool Caller256 = CallerMap.lookup(Key: "avx512f") && !CallerMap.lookup(Key: "evex512");
1543	bool Callee256 = CalleeMap.lookup(Key: "avx512f") && !CalleeMap.lookup(Key: "evex512");
1544
1545	// Forbid 512-bit or larger vector pass or return when we disabled ZMM
1546	// instructions.
1547	if (Caller256 || Callee256)
1548	return Diag.Report(CallLoc, diag::err_avx_calling_convention)
1549	<< IsArgument << Ty << "evex512";
1550
1551	return checkAVXParamFeature(Diag, CallLoc, CallerMap, CalleeMap, Ty,
1552	Feature: "avx512f", IsArgument);
1553	}
1554
1555	static bool checkAVXParam(DiagnosticsEngine &Diag, ASTContext &Ctx,
1556	SourceLocation CallLoc,
1557	const llvm::StringMap<bool> &CallerMap,
1558	const llvm::StringMap<bool> &CalleeMap, QualType Ty,
1559	bool IsArgument) {
1560	uint64_t Size = Ctx.getTypeSize(T: Ty);
1561	if (Size > 256)
1562	return checkAVX512ParamFeature(Diag, CallLoc, CallerMap, CalleeMap, Ty,
1563	IsArgument);
1564
1565	if (Size > 128)
1566	return checkAVXParamFeature(Diag, CallLoc, CallerMap, CalleeMap, Ty, Feature: "avx",
1567	IsArgument);
1568
1569	return false;
1570	}
1571
1572	void X86_64TargetCodeGenInfo::checkFunctionCallABI(CodeGenModule &CGM,
1573	SourceLocation CallLoc,
1574	const FunctionDecl *Caller,
1575	const FunctionDecl *Callee,
1576	const CallArgList &Args,
1577	QualType ReturnType) const {
1578	if (!Callee)
1579	return;
1580
1581	llvm::StringMap<bool> CallerMap;
1582	llvm::StringMap<bool> CalleeMap;
1583	unsigned ArgIndex = 0;
1584
1585	// We need to loop through the actual call arguments rather than the
1586	// function's parameters, in case this variadic.
1587	for (const CallArg &Arg : Args) {
1588	// The "avx" feature changes how vectors >128 in size are passed. "avx512f"
1589	// additionally changes how vectors >256 in size are passed. Like GCC, we
1590	// warn when a function is called with an argument where this will change.
1591	// Unlike GCC, we also error when it is an obvious ABI mismatch, that is,
1592	// the caller and callee features are mismatched.
1593	// Unfortunately, we cannot do this diagnostic in SEMA, since the callee can
1594	// change its ABI with attribute-target after this call.
1595	if (Arg.getType()->isVectorType() &&
1596	CGM.getContext().getTypeSize(T: Arg.getType()) > 128) {
1597	initFeatureMaps(Ctx: CGM.getContext(), CallerMap, Caller, CalleeMap, Callee);
1598	QualType Ty = Arg.getType();
1599	// The CallArg seems to have desugared the type already, so for clearer
1600	// diagnostics, replace it with the type in the FunctionDecl if possible.
1601	if (ArgIndex < Callee->getNumParams())
1602	Ty = Callee->getParamDecl(i: ArgIndex)->getType();
1603
1604	if (checkAVXParam(Diag&: CGM.getDiags(), Ctx&: CGM.getContext(), CallLoc, CallerMap,
1605	CalleeMap, Ty, /*IsArgument*/ true))
1606	return;
1607	}
1608	++ArgIndex;
1609	}
1610
1611	// Check return always, as we don't have a good way of knowing in codegen
1612	// whether this value is used, tail-called, etc.
1613	if (Callee->getReturnType()->isVectorType() &&
1614	CGM.getContext().getTypeSize(T: Callee->getReturnType()) > 128) {
1615	initFeatureMaps(Ctx: CGM.getContext(), CallerMap, Caller, CalleeMap, Callee);
1616	checkAVXParam(Diag&: CGM.getDiags(), Ctx&: CGM.getContext(), CallLoc, CallerMap,
1617	CalleeMap, Ty: Callee->getReturnType(),
1618	/*IsArgument*/ false);
1619	}
1620	}
1621
1622	std::string TargetCodeGenInfo::qualifyWindowsLibrary(StringRef Lib) {
1623	// If the argument does not end in .lib, automatically add the suffix.
1624	// If the argument contains a space, enclose it in quotes.
1625	// This matches the behavior of MSVC.
1626	bool Quote = Lib.contains(C: ' ');
1627	std::string ArgStr = Quote ? "\"" : "";
1628	ArgStr += Lib;
1629	if (!Lib.ends_with_insensitive(Suffix: ".lib") && !Lib.ends_with_insensitive(Suffix: ".a"))
1630	ArgStr += ".lib";
1631	ArgStr += Quote ? "\"" : "";
1632	return ArgStr;
1633	}
1634
1635	namespace {
1636	class WinX86_32TargetCodeGenInfo : public X86_32TargetCodeGenInfo {
1637	public:
1638	WinX86_32TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT,
1639	bool DarwinVectorABI, bool RetSmallStructInRegABI, bool Win32StructABI,
1640	unsigned NumRegisterParameters)
1641	: X86_32TargetCodeGenInfo(CGT, DarwinVectorABI, RetSmallStructInRegABI,
1642	Win32StructABI, NumRegisterParameters, false) {}
1643
1644	void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
1645	CodeGen::CodeGenModule &CGM) const override;
1646
1647	void getDependentLibraryOption(llvm::StringRef Lib,
1648	llvm::SmallString<24> &Opt) const override {
1649	Opt = "/DEFAULTLIB:";
1650	Opt += qualifyWindowsLibrary(Lib);
1651	}
1652
1653	void getDetectMismatchOption(llvm::StringRef Name,
1654	llvm::StringRef Value,
1655	llvm::SmallString<32> &Opt) const override {
1656	Opt = "/FAILIFMISMATCH:\"" + Name.str() + "=" + Value.str() + "\"";
1657	}
1658	};
1659	} // namespace
1660
1661	void WinX86_32TargetCodeGenInfo::setTargetAttributes(
1662	const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &CGM) const {
1663	X86_32TargetCodeGenInfo::setTargetAttributes(D, GV, CGM);
1664	if (GV->isDeclaration())
1665	return;
1666	addStackProbeTargetAttributes(D, GV, CGM);
1667	}
1668
1669	namespace {
1670	class WinX86_64TargetCodeGenInfo : public TargetCodeGenInfo {
1671	public:
1672	WinX86_64TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT,
1673	X86AVXABILevel AVXLevel)
1674	: TargetCodeGenInfo(std::make_unique<WinX86_64ABIInfo>(args&: CGT, args&: AVXLevel)) {
1675	SwiftInfo =
1676	std::make_unique<SwiftABIInfo>(args&: CGT, /*SwiftErrorInRegister=*/args: true);
1677	}
1678
1679	void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
1680	CodeGen::CodeGenModule &CGM) const override;
1681
1682	int getDwarfEHStackPointer(CodeGen::CodeGenModule &CGM) const override {
1683	return 7;
1684	}
1685
1686	bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
1687	llvm::Value *Address) const override {
1688	llvm::Value *Eight8 = llvm::ConstantInt::get(Ty: CGF.Int8Ty, V: 8);
1689
1690	// 0-15 are the 16 integer registers.
1691	// 16 is %rip.
1692	AssignToArrayRange(Builder&: CGF.Builder, Array: Address, Value: Eight8, FirstIndex: 0, LastIndex: 16);
1693	return false;
1694	}
1695
1696	void getDependentLibraryOption(llvm::StringRef Lib,
1697	llvm::SmallString<24> &Opt) const override {
1698	Opt = "/DEFAULTLIB:";
1699	Opt += qualifyWindowsLibrary(Lib);
1700	}
1701
1702	void getDetectMismatchOption(llvm::StringRef Name,
1703	llvm::StringRef Value,
1704	llvm::SmallString<32> &Opt) const override {
1705	Opt = "/FAILIFMISMATCH:\"" + Name.str() + "=" + Value.str() + "\"";
1706	}
1707	};
1708	} // namespace
1709
1710	void WinX86_64TargetCodeGenInfo::setTargetAttributes(
1711	const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &CGM) const {
1712	TargetCodeGenInfo::setTargetAttributes(D, GV, M&: CGM);
1713	if (GV->isDeclaration())
1714	return;
1715	if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(Val: D)) {
1716	if (FD->hasAttr<X86ForceAlignArgPointerAttr>()) {
1717	llvm::Function *Fn = cast<llvm::Function>(Val: GV);
1718	Fn->addFnAttr(Kind: "stackrealign");
1719	}
1720
1721	addX86InterruptAttrs(FD, GV, CGM);
1722	}
1723
1724	addStackProbeTargetAttributes(D, GV, CGM);
1725	}
1726
1727	void X86_64ABIInfo::postMerge(unsigned AggregateSize, Class &Lo,
1728	Class &Hi) const {
1729	// AMD64-ABI 3.2.3p2: Rule 5. Then a post merger cleanup is done:
1730	//
1731	// (a) If one of the classes is Memory, the whole argument is passed in
1732	// memory.
1733	//
1734	// (b) If X87UP is not preceded by X87, the whole argument is passed in
1735	// memory.
1736	//
1737	// (c) If the size of the aggregate exceeds two eightbytes and the first
1738	// eightbyte isn't SSE or any other eightbyte isn't SSEUP, the whole
1739	// argument is passed in memory. NOTE: This is necessary to keep the
1740	// ABI working for processors that don't support the __m256 type.
1741	//
1742	// (d) If SSEUP is not preceded by SSE or SSEUP, it is converted to SSE.
1743	//
1744	// Some of these are enforced by the merging logic. Others can arise
1745	// only with unions; for example:
1746	// union { _Complex double; unsigned; }
1747	//
1748	// Note that clauses (b) and (c) were added in 0.98.
1749	//
1750	if (Hi == Memory)
1751	Lo = Memory;
1752	if (Hi == X87Up && Lo != X87 && honorsRevision0_98())
1753	Lo = Memory;
1754	if (AggregateSize > 128 && (Lo != SSE || Hi != SSEUp))
1755	Lo = Memory;
1756	if (Hi == SSEUp && Lo != SSE)
1757	Hi = SSE;
1758	}
1759
1760	X86_64ABIInfo::Class X86_64ABIInfo::merge(Class Accum, Class Field) {
1761	// AMD64-ABI 3.2.3p2: Rule 4. Each field of an object is
1762	// classified recursively so that always two fields are
1763	// considered. The resulting class is calculated according to
1764	// the classes of the fields in the eightbyte:
1765	//
1766	// (a) If both classes are equal, this is the resulting class.
1767	//
1768	// (b) If one of the classes is NO_CLASS, the resulting class is
1769	// the other class.
1770	//
1771	// (c) If one of the classes is MEMORY, the result is the MEMORY
1772	// class.
1773	//
1774	// (d) If one of the classes is INTEGER, the result is the
1775	// INTEGER.
1776	//
1777	// (e) If one of the classes is X87, X87UP, COMPLEX_X87 class,
1778	// MEMORY is used as class.
1779	//
1780	// (f) Otherwise class SSE is used.
1781
1782	// Accum should never be memory (we should have returned) or
1783	// ComplexX87 (because this cannot be passed in a structure).
1784	assert((Accum != Memory && Accum != ComplexX87) &&
1785	"Invalid accumulated classification during merge.");
1786	if (Accum == Field || Field == NoClass)
1787	return Accum;
1788	if (Field == Memory)
1789	return Memory;
1790	if (Accum == NoClass)
1791	return Field;
1792	if (Accum == Integer || Field == Integer)
1793	return Integer;
1794	if (Field == X87 || Field == X87Up || Field == ComplexX87 ||
1795	Accum == X87 || Accum == X87Up)
1796	return Memory;
1797	return SSE;
1798	}
1799
1800	void X86_64ABIInfo::classify(QualType Ty, uint64_t OffsetBase, Class &Lo,
1801	Class &Hi, bool isNamedArg, bool IsRegCall) const {
1802	// FIXME: This code can be simplified by introducing a simple value class for
1803	// Class pairs with appropriate constructor methods for the various
1804	// situations.
1805
1806	// FIXME: Some of the split computations are wrong; unaligned vectors
1807	// shouldn't be passed in registers for example, so there is no chance they
1808	// can straddle an eightbyte. Verify & simplify.
1809
1810	Lo = Hi = NoClass;
1811
1812	Class &Current = OffsetBase < 64 ? Lo : Hi;
1813	Current = Memory;
1814
1815	if (const BuiltinType *BT = Ty->getAs<BuiltinType>()) {
1816	BuiltinType::Kind k = BT->getKind();
1817
1818	if (k == BuiltinType::Void) {
1819	Current = NoClass;
1820	} else if (k == BuiltinType::Int128 || k == BuiltinType::UInt128) {
1821	Lo = Integer;
1822	Hi = Integer;
1823	} else if (k >= BuiltinType::Bool && k <= BuiltinType::LongLong) {
1824	Current = Integer;
1825	} else if (k == BuiltinType::Float || k == BuiltinType::Double ||
1826	k == BuiltinType::Float16 || k == BuiltinType::BFloat16) {
1827	Current = SSE;
1828	} else if (k == BuiltinType::Float128) {
1829	Lo = SSE;
1830	Hi = SSEUp;
1831	} else if (k == BuiltinType::LongDouble) {
1832	const llvm::fltSemantics *LDF = &getTarget().getLongDoubleFormat();
1833	if (LDF == &llvm::APFloat::IEEEquad()) {
1834	Lo = SSE;
1835	Hi = SSEUp;
1836	} else if (LDF == &llvm::APFloat::x87DoubleExtended()) {
1837	Lo = X87;
1838	Hi = X87Up;
1839	} else if (LDF == &llvm::APFloat::IEEEdouble()) {
1840	Current = SSE;
1841	} else
1842	llvm_unreachable("unexpected long double representation!");
1843	}
1844	// FIXME: _Decimal32 and _Decimal64 are SSE.
1845	// FIXME: _float128 and _Decimal128 are (SSE, SSEUp).
1846	return;
1847	}
1848
1849	if (const EnumType *ET = Ty->getAs<EnumType>()) {
1850	// Classify the underlying integer type.
1851	classify(Ty: ET->getDecl()->getIntegerType(), OffsetBase, Lo, Hi, isNamedArg);
1852	return;
1853	}
1854
1855	if (Ty->hasPointerRepresentation()) {
1856	Current = Integer;
1857	return;
1858	}
1859
1860	if (Ty->isMemberPointerType()) {
1861	if (Ty->isMemberFunctionPointerType()) {
1862	if (Has64BitPointers) {
1863	// If Has64BitPointers, this is an {i64, i64}, so classify both
1864	// Lo and Hi now.
1865	Lo = Hi = Integer;
1866	} else {
1867	// Otherwise, with 32-bit pointers, this is an {i32, i32}. If that
1868	// straddles an eightbyte boundary, Hi should be classified as well.
1869	uint64_t EB_FuncPtr = (OffsetBase) / 64;
1870	uint64_t EB_ThisAdj = (OffsetBase + 64 - 1) / 64;
1871	if (EB_FuncPtr != EB_ThisAdj) {
1872	Lo = Hi = Integer;
1873	} else {
1874	Current = Integer;
1875	}
1876	}
1877	} else {
1878	Current = Integer;
1879	}
1880	return;
1881	}
1882
1883	if (const VectorType *VT = Ty->getAs<VectorType>()) {
1884	uint64_t Size = getContext().getTypeSize(VT);
1885	if (Size == 1 || Size == 8 || Size == 16 || Size == 32) {
1886	// gcc passes the following as integer:
1887	// 4 bytes - <4 x char>, <2 x short>, <1 x int>, <1 x float>
1888	// 2 bytes - <2 x char>, <1 x short>
1889	// 1 byte - <1 x char>
1890	Current = Integer;
1891
1892	// If this type crosses an eightbyte boundary, it should be
1893	// split.
1894	uint64_t EB_Lo = (OffsetBase) / 64;
1895	uint64_t EB_Hi = (OffsetBase + Size - 1) / 64;
1896	if (EB_Lo != EB_Hi)
1897	Hi = Lo;
1898	} else if (Size == 64) {
1899	QualType ElementType = VT->getElementType();
1900
1901	// gcc passes <1 x double> in memory. :(
1902	if (ElementType->isSpecificBuiltinType(K: BuiltinType::Double))
1903	return;
1904
1905	// gcc passes <1 x long long> as SSE but clang used to unconditionally
1906	// pass them as integer. For platforms where clang is the de facto
1907	// platform compiler, we must continue to use integer.
1908	if (!classifyIntegerMMXAsSSE() &&
1909	(ElementType->isSpecificBuiltinType(K: BuiltinType::LongLong) ||
1910	ElementType->isSpecificBuiltinType(K: BuiltinType::ULongLong) ||
1911	ElementType->isSpecificBuiltinType(K: BuiltinType::Long) ||
1912	ElementType->isSpecificBuiltinType(K: BuiltinType::ULong)))
1913	Current = Integer;
1914	else
1915	Current = SSE;
1916
1917	// If this type crosses an eightbyte boundary, it should be
1918	// split.
1919	if (OffsetBase && OffsetBase != 64)
1920	Hi = Lo;
1921	} else if (Size == 128 ||
1922	(isNamedArg && Size <= getNativeVectorSizeForAVXABI(AVXLevel))) {
1923	QualType ElementType = VT->getElementType();
1924
1925	// gcc passes 256 and 512 bit <X x __int128> vectors in memory. :(
1926	if (passInt128VectorsInMem() && Size != 128 &&
1927	(ElementType->isSpecificBuiltinType(K: BuiltinType::Int128) ||
1928	ElementType->isSpecificBuiltinType(K: BuiltinType::UInt128)))
1929	return;
1930
1931	// Arguments of 256-bits are split into four eightbyte chunks. The
1932	// least significant one belongs to class SSE and all the others to class
1933	// SSEUP. The original Lo and Hi design considers that types can't be
1934	// greater than 128-bits, so a 64-bit split in Hi and Lo makes sense.
1935	// This design isn't correct for 256-bits, but since there're no cases
1936	// where the upper parts would need to be inspected, avoid adding
1937	// complexity and just consider Hi to match the 64-256 part.
1938	//
1939	// Note that per 3.5.7 of AMD64-ABI, 256-bit args are only passed in
1940	// registers if they are "named", i.e. not part of the "..." of a
1941	// variadic function.
1942	//
1943	// Similarly, per 3.2.3. of the AVX512 draft, 512-bits ("named") args are
1944	// split into eight eightbyte chunks, one SSE and seven SSEUP.
1945	Lo = SSE;
1946	Hi = SSEUp;
1947	}
1948	return;
1949	}
1950
1951	if (const ComplexType *CT = Ty->getAs<ComplexType>()) {
1952	QualType ET = getContext().getCanonicalType(T: CT->getElementType());
1953
1954	uint64_t Size = getContext().getTypeSize(T: Ty);
1955	if (ET->isIntegralOrEnumerationType()) {
1956	if (Size <= 64)
1957	Current = Integer;
1958	else if (Size <= 128)
1959	Lo = Hi = Integer;
1960	} else if (ET->isFloat16Type() || ET == getContext().FloatTy ||
1961	ET->isBFloat16Type()) {
1962	Current = SSE;
1963	} else if (ET == getContext().DoubleTy) {
1964	Lo = Hi = SSE;
1965	} else if (ET == getContext().LongDoubleTy) {
1966	const llvm::fltSemantics *LDF = &getTarget().getLongDoubleFormat();
1967	if (LDF == &llvm::APFloat::IEEEquad())
1968	Current = Memory;
1969	else if (LDF == &llvm::APFloat::x87DoubleExtended())
1970	Current = ComplexX87;
1971	else if (LDF == &llvm::APFloat::IEEEdouble())
1972	Lo = Hi = SSE;
1973	else
1974	llvm_unreachable("unexpected long double representation!");
1975	}
1976
1977	// If this complex type crosses an eightbyte boundary then it
1978	// should be split.
1979	uint64_t EB_Real = (OffsetBase) / 64;
1980	uint64_t EB_Imag = (OffsetBase + getContext().getTypeSize(T: ET)) / 64;
1981	if (Hi == NoClass && EB_Real != EB_Imag)
1982	Hi = Lo;
1983
1984	return;
1985	}
1986
1987	if (const auto *EITy = Ty->getAs<BitIntType>()) {
1988	if (EITy->getNumBits() <= 64)
1989	Current = Integer;
1990	else if (EITy->getNumBits() <= 128)
1991	Lo = Hi = Integer;
1992	// Larger values need to get passed in memory.
1993	return;
1994	}
1995
1996	if (const ConstantArrayType *AT = getContext().getAsConstantArrayType(T: Ty)) {
1997	// Arrays are treated like structures.
1998
1999	uint64_t Size = getContext().getTypeSize(T: Ty);
2000
2001	// AMD64-ABI 3.2.3p2: Rule 1. If the size of an object is larger
2002	// than eight eightbytes, ..., it has class MEMORY.
2003	// regcall ABI doesn't have limitation to an object. The only limitation
2004	// is the free registers, which will be checked in computeInfo.
2005	if (!IsRegCall && Size > 512)
2006	return;
2007
2008	// AMD64-ABI 3.2.3p2: Rule 1. If ..., or it contains unaligned
2009	// fields, it has class MEMORY.
2010	//
2011	// Only need to check alignment of array base.
2012	if (OffsetBase % getContext().getTypeAlign(AT->getElementType()))
2013	return;
2014
2015	// Otherwise implement simplified merge. We could be smarter about
2016	// this, but it isn't worth it and would be harder to verify.
2017	Current = NoClass;
2018	uint64_t EltSize = getContext().getTypeSize(AT->getElementType());
2019	uint64_t ArraySize = AT->getZExtSize();
2020
2021	// The only case a 256-bit wide vector could be used is when the array
2022	// contains a single 256-bit element. Since Lo and Hi logic isn't extended
2023	// to work for sizes wider than 128, early check and fallback to memory.
2024	//
2025	if (Size > 128 &&
2026	(Size != EltSize || Size > getNativeVectorSizeForAVXABI(AVXLevel)))
2027	return;
2028
2029	for (uint64_t i=0, Offset=OffsetBase; i<ArraySize; ++i, Offset += EltSize) {
2030	Class FieldLo, FieldHi;
2031	classify(Ty: AT->getElementType(), OffsetBase: Offset, Lo&: FieldLo, Hi&: FieldHi, isNamedArg);
2032	Lo = merge(Accum: Lo, Field: FieldLo);
2033	Hi = merge(Accum: Hi, Field: FieldHi);
2034	if (Lo == Memory || Hi == Memory)
2035	break;
2036	}
2037
2038	postMerge(AggregateSize: Size, Lo, Hi);
2039	assert((Hi != SSEUp || Lo == SSE) && "Invalid SSEUp array classification.");
2040	return;
2041	}
2042
2043	if (const RecordType *RT = Ty->getAs<RecordType>()) {
2044	uint64_t Size = getContext().getTypeSize(T: Ty);
2045
2046	// AMD64-ABI 3.2.3p2: Rule 1. If the size of an object is larger
2047	// than eight eightbytes, ..., it has class MEMORY.
2048	if (Size > 512)
2049	return;
2050
2051	// AMD64-ABI 3.2.3p2: Rule 2. If a C++ object has either a non-trivial
2052	// copy constructor or a non-trivial destructor, it is passed by invisible
2053	// reference.
2054	if (getRecordArgABI(RT, CXXABI&: getCXXABI()))
2055	return;
2056
2057	const RecordDecl *RD = RT->getDecl();
2058
2059	// Assume variable sized types are passed in memory.
2060	if (RD->hasFlexibleArrayMember())
2061	return;
2062
2063	const ASTRecordLayout &Layout = getContext().getASTRecordLayout(D: RD);
2064
2065	// Reset Lo class, this will be recomputed.
2066	Current = NoClass;
2067
2068	// If this is a C++ record, classify the bases first.
2069	if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(Val: RD)) {
2070	for (const auto &I : CXXRD->bases()) {
2071	assert(!I.isVirtual() && !I.getType()->isDependentType() &&
2072	"Unexpected base class!");
2073	const auto *Base =
2074	cast<CXXRecordDecl>(Val: I.getType()->castAs<RecordType>()->getDecl());
2075
2076	// Classify this field.
2077	//
2078	// AMD64-ABI 3.2.3p2: Rule 3. If the size of the aggregate exceeds a
2079	// single eightbyte, each is classified separately. Each eightbyte gets
2080	// initialized to class NO_CLASS.
2081	Class FieldLo, FieldHi;
2082	uint64_t Offset =
2083	OffsetBase + getContext().toBits(CharSize: Layout.getBaseClassOffset(Base));
2084	classify(Ty: I.getType(), OffsetBase: Offset, Lo&: FieldLo, Hi&: FieldHi, isNamedArg);
2085	Lo = merge(Accum: Lo, Field: FieldLo);
2086	Hi = merge(Accum: Hi, Field: FieldHi);
2087	if (returnCXXRecordGreaterThan128InMem() &&
2088	(Size > 128 && (Size != getContext().getTypeSize(T: I.getType()) ||
2089	Size > getNativeVectorSizeForAVXABI(AVXLevel)))) {
2090	// The only case a 256(or 512)-bit wide vector could be used to return
2091	// is when CXX record contains a single 256(or 512)-bit element.
2092	Lo = Memory;
2093	}
2094	if (Lo == Memory || Hi == Memory) {
2095	postMerge(AggregateSize: Size, Lo, Hi);
2096	return;
2097	}
2098	}
2099	}
2100
2101	// Classify the fields one at a time, merging the results.
2102	unsigned idx = 0;
2103	bool UseClang11Compat = getContext().getLangOpts().getClangABICompat() <=
2104	LangOptions::ClangABI::Ver11 ||
2105	getContext().getTargetInfo().getTriple().isPS();
2106	bool IsUnion = RT->isUnionType() && !UseClang11Compat;
2107
2108	for (RecordDecl::field_iterator i = RD->field_begin(), e = RD->field_end();
2109	i != e; ++i, ++idx) {
2110	uint64_t Offset = OffsetBase + Layout.getFieldOffset(FieldNo: idx);
2111	bool BitField = i->isBitField();
2112
2113	// Ignore padding bit-fields.
2114	if (BitField && i->isUnnamedBitField())
2115	continue;
2116
2117	// AMD64-ABI 3.2.3p2: Rule 1. If the size of an object is larger than
2118	// eight eightbytes, or it contains unaligned fields, it has class MEMORY.
2119	//
2120	// The only case a 256-bit or a 512-bit wide vector could be used is when
2121	// the struct contains a single 256-bit or 512-bit element. Early check
2122	// and fallback to memory.
2123	//
2124	// FIXME: Extended the Lo and Hi logic properly to work for size wider
2125	// than 128.
2126	if (Size > 128 &&
2127	((!IsUnion && Size != getContext().getTypeSize(i->getType())) ||
2128	Size > getNativeVectorSizeForAVXABI(AVXLevel))) {
2129	Lo = Memory;
2130	postMerge(AggregateSize: Size, Lo, Hi);
2131	return;
2132	}
2133
2134	bool IsInMemory =
2135	Offset % getContext().getTypeAlign(i->getType().getCanonicalType());
2136	// Note, skip this test for bit-fields, see below.
2137	if (!BitField && IsInMemory) {
2138	Lo = Memory;
2139	postMerge(AggregateSize: Size, Lo, Hi);
2140	return;
2141	}
2142
2143	// Classify this field.
2144	//
2145	// AMD64-ABI 3.2.3p2: Rule 3. If the size of the aggregate
2146	// exceeds a single eightbyte, each is classified
2147	// separately. Each eightbyte gets initialized to class
2148	// NO_CLASS.
2149	Class FieldLo, FieldHi;
2150
2151	// Bit-fields require special handling, they do not force the
2152	// structure to be passed in memory even if unaligned, and
2153	// therefore they can straddle an eightbyte.
2154	if (BitField) {
2155	assert(!i->isUnnamedBitField());
2156	uint64_t Offset = OffsetBase + Layout.getFieldOffset(FieldNo: idx);
2157	uint64_t Size = i->getBitWidthValue();
2158
2159	uint64_t EB_Lo = Offset / 64;
2160	uint64_t EB_Hi = (Offset + Size - 1) / 64;
2161
2162	if (EB_Lo) {
2163	assert(EB_Hi == EB_Lo && "Invalid classification, type > 16 bytes.");
2164	FieldLo = NoClass;
2165	FieldHi = Integer;
2166	} else {
2167	FieldLo = Integer;
2168	FieldHi = EB_Hi ? Integer : NoClass;
2169	}
2170	} else
2171	classify(Ty: i->getType(), OffsetBase: Offset, Lo&: FieldLo, Hi&: FieldHi, isNamedArg);
2172	Lo = merge(Accum: Lo, Field: FieldLo);
2173	Hi = merge(Accum: Hi, Field: FieldHi);
2174	if (Lo == Memory || Hi == Memory)
2175	break;
2176	}
2177
2178	postMerge(AggregateSize: Size, Lo, Hi);
2179	}
2180	}
2181
2182	ABIArgInfo X86_64ABIInfo::getIndirectReturnResult(QualType Ty) const {
2183	// If this is a scalar LLVM value then assume LLVM will pass it in the right
2184	// place naturally.
2185	if (!isAggregateTypeForABI(T: Ty)) {
2186	// Treat an enum type as its underlying type.
2187	if (const EnumType *EnumTy = Ty->getAs<EnumType>())
2188	Ty = EnumTy->getDecl()->getIntegerType();
2189
2190	if (Ty->isBitIntType())
2191	return getNaturalAlignIndirect(Ty, AddrSpace: getDataLayout().getAllocaAddrSpace());
2192
2193	return (isPromotableIntegerTypeForABI(Ty) ? ABIArgInfo::getExtend(Ty)
2194	: ABIArgInfo::getDirect());
2195	}
2196
2197	return getNaturalAlignIndirect(Ty, AddrSpace: getDataLayout().getAllocaAddrSpace());
2198	}
2199
2200	bool X86_64ABIInfo::IsIllegalVectorType(QualType Ty) const {
2201	if (const VectorType *VecTy = Ty->getAs<VectorType>()) {
2202	uint64_t Size = getContext().getTypeSize(VecTy);
2203	unsigned LargestVector = getNativeVectorSizeForAVXABI(AVXLevel);
2204	if (Size <= 64 || Size > LargestVector)
2205	return true;
2206	QualType EltTy = VecTy->getElementType();
2207	if (passInt128VectorsInMem() &&
2208	(EltTy->isSpecificBuiltinType(K: BuiltinType::Int128) ||
2209	EltTy->isSpecificBuiltinType(K: BuiltinType::UInt128)))
2210	return true;
2211	}
2212
2213	return false;
2214	}
2215
2216	ABIArgInfo X86_64ABIInfo::getIndirectResult(QualType Ty,
2217	unsigned freeIntRegs) const {
2218	// If this is a scalar LLVM value then assume LLVM will pass it in the right
2219	// place naturally.
2220	//
2221	// This assumption is optimistic, as there could be free registers available
2222	// when we need to pass this argument in memory, and LLVM could try to pass
2223	// the argument in the free register. This does not seem to happen currently,
2224	// but this code would be much safer if we could mark the argument with
2225	// 'onstack'. See PR12193.
2226	if (!isAggregateTypeForABI(T: Ty) && !IsIllegalVectorType(Ty) &&
2227	!Ty->isBitIntType()) {
2228	// Treat an enum type as its underlying type.
2229	if (const EnumType *EnumTy = Ty->getAs<EnumType>())
2230	Ty = EnumTy->getDecl()->getIntegerType();
2231
2232	return (isPromotableIntegerTypeForABI(Ty) ? ABIArgInfo::getExtend(Ty)
2233	: ABIArgInfo::getDirect());
2234	}
2235
2236	if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(T: Ty, CXXABI&: getCXXABI()))
2237	return getNaturalAlignIndirect(Ty, AddrSpace: getDataLayout().getAllocaAddrSpace(),
2238	ByVal: RAA == CGCXXABI::RAA_DirectInMemory);
2239
2240	// Compute the byval alignment. We specify the alignment of the byval in all
2241	// cases so that the mid-level optimizer knows the alignment of the byval.
2242	unsigned Align = std::max(a: getContext().getTypeAlign(T: Ty) / 8, b: 8U);
2243
2244	// Attempt to avoid passing indirect results using byval when possible. This
2245	// is important for good codegen.
2246	//
2247	// We do this by coercing the value into a scalar type which the backend can
2248	// handle naturally (i.e., without using byval).
2249	//
2250	// For simplicity, we currently only do this when we have exhausted all of the
2251	// free integer registers. Doing this when there are free integer registers
2252	// would require more care, as we would have to ensure that the coerced value
2253	// did not claim the unused register. That would require either reording the
2254	// arguments to the function (so that any subsequent inreg values came first),
2255	// or only doing this optimization when there were no following arguments that
2256	// might be inreg.
2257	//
2258	// We currently expect it to be rare (particularly in well written code) for
2259	// arguments to be passed on the stack when there are still free integer
2260	// registers available (this would typically imply large structs being passed
2261	// by value), so this seems like a fair tradeoff for now.
2262	//
2263	// We can revisit this if the backend grows support for 'onstack' parameter
2264	// attributes. See PR12193.
2265	if (freeIntRegs == 0) {
2266	uint64_t Size = getContext().getTypeSize(T: Ty);
2267
2268	// If this type fits in an eightbyte, coerce it into the matching integral
2269	// type, which will end up on the stack (with alignment 8).
2270	if (Align == 8 && Size <= 64)
2271	return ABIArgInfo::getDirect(T: llvm::IntegerType::get(C&: getVMContext(),
2272	NumBits: Size));
2273	}
2274
2275	return ABIArgInfo::getIndirect(Alignment: CharUnits::fromQuantity(Quantity: Align),
2276	AddrSpace: getDataLayout().getAllocaAddrSpace());
2277	}
2278
2279	/// The ABI specifies that a value should be passed in a full vector XMM/YMM
2280	/// register. Pick an LLVM IR type that will be passed as a vector register.
2281	llvm::Type *X86_64ABIInfo::GetByteVectorType(QualType Ty) const {
2282	// Wrapper structs/arrays that only contain vectors are passed just like
2283	// vectors; strip them off if present.
2284	if (const Type *InnerTy = isSingleElementStruct(T: Ty, Context&: getContext()))
2285	Ty = QualType(InnerTy, 0);
2286
2287	llvm::Type *IRType = CGT.ConvertType(T: Ty);
2288	if (isa<llvm::VectorType>(Val: IRType)) {
2289	// Don't pass vXi128 vectors in their native type, the backend can't
2290	// legalize them.
2291	if (passInt128VectorsInMem() &&
2292	cast<llvm::VectorType>(Val: IRType)->getElementType()->isIntegerTy(Bitwidth: 128)) {
2293	// Use a vXi64 vector.
2294	uint64_t Size = getContext().getTypeSize(T: Ty);
2295	return llvm::FixedVectorType::get(ElementType: llvm::Type::getInt64Ty(C&: getVMContext()),
2296	NumElts: Size / 64);
2297	}
2298
2299	return IRType;
2300	}
2301
2302	if (IRType->getTypeID() == llvm::Type::FP128TyID)
2303	return IRType;
2304
2305	// We couldn't find the preferred IR vector type for 'Ty'.
2306	uint64_t Size = getContext().getTypeSize(T: Ty);
2307	assert((Size == 128 || Size == 256 || Size == 512) && "Invalid type found!");
2308
2309
2310	// Return a LLVM IR vector type based on the size of 'Ty'.
2311	return llvm::FixedVectorType::get(ElementType: llvm::Type::getDoubleTy(C&: getVMContext()),
2312	NumElts: Size / 64);
2313	}
2314
2315	/// BitsContainNoUserData - Return true if the specified [start,end) bit range
2316	/// is known to either be off the end of the specified type or being in
2317	/// alignment padding. The user type specified is known to be at most 128 bits
2318	/// in size, and have passed through X86_64ABIInfo::classify with a successful
2319	/// classification that put one of the two halves in the INTEGER class.
2320	///
2321	/// It is conservatively correct to return false.
2322	static bool BitsContainNoUserData(QualType Ty, unsigned StartBit,
2323	unsigned EndBit, ASTContext &Context) {
2324	// If the bytes being queried are off the end of the type, there is no user
2325	// data hiding here. This handles analysis of builtins, vectors and other
2326	// types that don't contain interesting padding.
2327	unsigned TySize = (unsigned)Context.getTypeSize(T: Ty);
2328	if (TySize <= StartBit)
2329	return true;
2330
2331	if (const ConstantArrayType *AT = Context.getAsConstantArrayType(T: Ty)) {
2332	unsigned EltSize = (unsigned)Context.getTypeSize(AT->getElementType());
2333	unsigned NumElts = (unsigned)AT->getZExtSize();
2334
2335	// Check each element to see if the element overlaps with the queried range.
2336	for (unsigned i = 0; i != NumElts; ++i) {
2337	// If the element is after the span we care about, then we're done..
2338	unsigned EltOffset = i*EltSize;
2339	if (EltOffset >= EndBit) break;
2340
2341	unsigned EltStart = EltOffset < StartBit ? StartBit-EltOffset :0;
2342	if (!BitsContainNoUserData(AT->getElementType(), EltStart,
2343	EndBit-EltOffset, Context))
2344	return false;
2345	}
2346	// If it overlaps no elements, then it is safe to process as padding.
2347	return true;
2348	}
2349
2350	if (const RecordType *RT = Ty->getAs<RecordType>()) {
2351	const RecordDecl *RD = RT->getDecl();
2352	const ASTRecordLayout &Layout = Context.getASTRecordLayout(D: RD);
2353
2354	// If this is a C++ record, check the bases first.
2355	if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(Val: RD)) {
2356	for (const auto &I : CXXRD->bases()) {
2357	assert(!I.isVirtual() && !I.getType()->isDependentType() &&
2358	"Unexpected base class!");
2359	const auto *Base =
2360	cast<CXXRecordDecl>(Val: I.getType()->castAs<RecordType>()->getDecl());
2361
2362	// If the base is after the span we care about, ignore it.
2363	unsigned BaseOffset = Context.toBits(CharSize: Layout.getBaseClassOffset(Base));
2364	if (BaseOffset >= EndBit) continue;
2365
2366	unsigned BaseStart = BaseOffset < StartBit ? StartBit-BaseOffset :0;
2367	if (!BitsContainNoUserData(Ty: I.getType(), StartBit: BaseStart,
2368	EndBit: EndBit-BaseOffset, Context))
2369	return false;
2370	}
2371	}
2372
2373	// Verify that no field has data that overlaps the region of interest. Yes
2374	// this could be sped up a lot by being smarter about queried fields,
2375	// however we're only looking at structs up to 16 bytes, so we don't care
2376	// much.
2377	unsigned idx = 0;
2378	for (RecordDecl::field_iterator i = RD->field_begin(), e = RD->field_end();
2379	i != e; ++i, ++idx) {
2380	unsigned FieldOffset = (unsigned)Layout.getFieldOffset(FieldNo: idx);
2381
2382	// If we found a field after the region we care about, then we're done.
2383	if (FieldOffset >= EndBit) break;
2384
2385	unsigned FieldStart = FieldOffset < StartBit ? StartBit-FieldOffset :0;
2386	if (!BitsContainNoUserData(i->getType(), FieldStart, EndBit-FieldOffset,
2387	Context))
2388	return false;
2389	}
2390
2391	// If nothing in this record overlapped the area of interest, then we're
2392	// clean.
2393	return true;
2394	}
2395
2396	return false;
2397	}
2398
2399	/// getFPTypeAtOffset - Return a floating point type at the specified offset.
2400	static llvm::Type *getFPTypeAtOffset(llvm::Type *IRType, unsigned IROffset,
2401	const llvm::DataLayout &TD) {
2402	if (IROffset == 0 && IRType->isFloatingPointTy())
2403	return IRType;
2404
2405	// If this is a struct, recurse into the field at the specified offset.
2406	if (llvm::StructType *STy = dyn_cast<llvm::StructType>(Val: IRType)) {
2407	if (!STy->getNumContainedTypes())
2408	return nullptr;
2409
2410	const llvm::StructLayout *SL = TD.getStructLayout(Ty: STy);
2411	unsigned Elt = SL->getElementContainingOffset(FixedOffset: IROffset);
2412	IROffset -= SL->getElementOffset(Idx: Elt);
2413	return getFPTypeAtOffset(IRType: STy->getElementType(N: Elt), IROffset, TD);
2414	}
2415
2416	// If this is an array, recurse into the field at the specified offset.
2417	if (llvm::ArrayType *ATy = dyn_cast<llvm::ArrayType>(Val: IRType)) {
2418	llvm::Type *EltTy = ATy->getElementType();
2419	unsigned EltSize = TD.getTypeAllocSize(Ty: EltTy);
2420	IROffset -= IROffset / EltSize * EltSize;
2421	return getFPTypeAtOffset(IRType: EltTy, IROffset, TD);
2422	}
2423
2424	return nullptr;
2425	}
2426
2427	/// GetSSETypeAtOffset - Return a type that will be passed by the backend in the
2428	/// low 8 bytes of an XMM register, corresponding to the SSE class.
2429	llvm::Type *X86_64ABIInfo::
2430	GetSSETypeAtOffset(llvm::Type *IRType, unsigned IROffset,
2431	QualType SourceTy, unsigned SourceOffset) const {
2432	const llvm::DataLayout &TD = getDataLayout();
2433	unsigned SourceSize =
2434	(unsigned)getContext().getTypeSize(T: SourceTy) / 8 - SourceOffset;
2435	llvm::Type *T0 = getFPTypeAtOffset(IRType, IROffset, TD);
2436	if (!T0 || T0->isDoubleTy())
2437	return llvm::Type::getDoubleTy(C&: getVMContext());
2438
2439	// Get the adjacent FP type.
2440	llvm::Type *T1 = nullptr;
2441	unsigned T0Size = TD.getTypeAllocSize(Ty: T0);
2442	if (SourceSize > T0Size)
2443	T1 = getFPTypeAtOffset(IRType, IROffset: IROffset + T0Size, TD);
2444	if (T1 == nullptr) {
2445	// Check if IRType is a half/bfloat + float. float type will be in IROffset+4 due
2446	// to its alignment.
2447	if (T0->is16bitFPTy() && SourceSize > 4)
2448	T1 = getFPTypeAtOffset(IRType, IROffset: IROffset + 4, TD);
2449	// If we can't get a second FP type, return a simple half or float.
2450	// avx512fp16-abi.c:pr51813_2 shows it works to return float for
2451	// {float, i8} too.
2452	if (T1 == nullptr)
2453	return T0;
2454	}
2455
2456	if (T0->isFloatTy() && T1->isFloatTy())
2457	return llvm::FixedVectorType::get(ElementType: T0, NumElts: 2);
2458
2459	if (T0->is16bitFPTy() && T1->is16bitFPTy()) {
2460	llvm::Type *T2 = nullptr;
2461	if (SourceSize > 4)
2462	T2 = getFPTypeAtOffset(IRType, IROffset: IROffset + 4, TD);
2463	if (T2 == nullptr)
2464	return llvm::FixedVectorType::get(ElementType: T0, NumElts: 2);
2465	return llvm::FixedVectorType::get(ElementType: T0, NumElts: 4);
2466	}
2467
2468	if (T0->is16bitFPTy() || T1->is16bitFPTy())
2469	return llvm::FixedVectorType::get(ElementType: llvm::Type::getHalfTy(C&: getVMContext()), NumElts: 4);
2470
2471	return llvm::Type::getDoubleTy(C&: getVMContext());
2472	}
2473
2474
2475	/// GetINTEGERTypeAtOffset - The ABI specifies that a value should be passed in
2476	/// an 8-byte GPR. This means that we either have a scalar or we are talking
2477	/// about the high or low part of an up-to-16-byte struct. This routine picks
2478	/// the best LLVM IR type to represent this, which may be i64 or may be anything
2479	/// else that the backend will pass in a GPR that works better (e.g. i8, %foo*,
2480	/// etc).
2481	///
2482	/// PrefType is an LLVM IR type that corresponds to (part of) the IR type for
2483	/// the source type. IROffset is an offset in bytes into the LLVM IR type that
2484	/// the 8-byte value references. PrefType may be null.
2485	///
2486	/// SourceTy is the source-level type for the entire argument. SourceOffset is
2487	/// an offset into this that we're processing (which is always either 0 or 8).
2488	///
2489	llvm::Type *X86_64ABIInfo::
2490	GetINTEGERTypeAtOffset(llvm::Type *IRType, unsigned IROffset,
2491	QualType SourceTy, unsigned SourceOffset) const {
2492	// If we're dealing with an un-offset LLVM IR type, then it means that we're
2493	// returning an 8-byte unit starting with it. See if we can safely use it.
2494	if (IROffset == 0) {
2495	// Pointers and int64's always fill the 8-byte unit.
2496	if ((isa<llvm::PointerType>(Val: IRType) && Has64BitPointers) ||
2497	IRType->isIntegerTy(Bitwidth: 64))
2498	return IRType;
2499
2500	// If we have a 1/2/4-byte integer, we can use it only if the rest of the
2501	// goodness in the source type is just tail padding. This is allowed to
2502	// kick in for struct {double,int} on the int, but not on
2503	// struct{double,int,int} because we wouldn't return the second int. We
2504	// have to do this analysis on the source type because we can't depend on
2505	// unions being lowered a specific way etc.
2506	if (IRType->isIntegerTy(Bitwidth: 8) || IRType->isIntegerTy(Bitwidth: 16) ||
2507	IRType->isIntegerTy(Bitwidth: 32) ||
2508	(isa<llvm::PointerType>(Val: IRType) && !Has64BitPointers)) {
2509	unsigned BitWidth = isa<llvm::PointerType>(Val: IRType) ? 32 :
2510	cast<llvm::IntegerType>(Val: IRType)->getBitWidth();
2511
2512	if (BitsContainNoUserData(Ty: SourceTy, StartBit: SourceOffset*8+BitWidth,
2513	EndBit: SourceOffset*8+64, Context&: getContext()))
2514	return IRType;
2515	}
2516	}
2517
2518	if (llvm::StructType *STy = dyn_cast<llvm::StructType>(Val: IRType)) {
2519	// If this is a struct, recurse into the field at the specified offset.
2520	const llvm::StructLayout *SL = getDataLayout().getStructLayout(Ty: STy);
2521	if (IROffset < SL->getSizeInBytes()) {
2522	unsigned FieldIdx = SL->getElementContainingOffset(FixedOffset: IROffset);
2523	IROffset -= SL->getElementOffset(Idx: FieldIdx);
2524
2525	return GetINTEGERTypeAtOffset(IRType: STy->getElementType(N: FieldIdx), IROffset,
2526	SourceTy, SourceOffset);
2527	}
2528	}
2529
2530	if (llvm::ArrayType *ATy = dyn_cast<llvm::ArrayType>(Val: IRType)) {
2531	llvm::Type *EltTy = ATy->getElementType();
2532	unsigned EltSize = getDataLayout().getTypeAllocSize(Ty: EltTy);
2533	unsigned EltOffset = IROffset/EltSize*EltSize;
2534	return GetINTEGERTypeAtOffset(IRType: EltTy, IROffset: IROffset-EltOffset, SourceTy,
2535	SourceOffset);
2536	}
2537
2538	// Okay, we don't have any better idea of what to pass, so we pass this in an
2539	// integer register that isn't too big to fit the rest of the struct.
2540	unsigned TySizeInBytes =
2541	(unsigned)getContext().getTypeSizeInChars(T: SourceTy).getQuantity();
2542
2543	assert(TySizeInBytes != SourceOffset && "Empty field?");
2544
2545	// It is always safe to classify this as an integer type up to i64 that
2546	// isn't larger than the structure.
2547	return llvm::IntegerType::get(C&: getVMContext(),
2548	NumBits: std::min(a: TySizeInBytes-SourceOffset, b: 8U)*8);
2549	}
2550
2551
2552	/// GetX86_64ByValArgumentPair - Given a high and low type that can ideally
2553	/// be used as elements of a two register pair to pass or return, return a
2554	/// first class aggregate to represent them. For example, if the low part of
2555	/// a by-value argument should be passed as i32* and the high part as float,
2556	/// return {i32*, float}.
2557	static llvm::Type *
2558	GetX86_64ByValArgumentPair(llvm::Type *Lo, llvm::Type *Hi,
2559	const llvm::DataLayout &TD) {
2560	// In order to correctly satisfy the ABI, we need to the high part to start
2561	// at offset 8. If the high and low parts we inferred are both 4-byte types
2562	// (e.g. i32 and i32) then the resultant struct type ({i32,i32}) won't have
2563	// the second element at offset 8. Check for this:
2564	unsigned LoSize = (unsigned)TD.getTypeAllocSize(Ty: Lo);
2565	llvm::Align HiAlign = TD.getABITypeAlign(Ty: Hi);
2566	unsigned HiStart = llvm::alignTo(Size: LoSize, A: HiAlign);
2567	assert(HiStart != 0 && HiStart <= 8 && "Invalid x86-64 argument pair!");
2568
2569	// To handle this, we have to increase the size of the low part so that the
2570	// second element will start at an 8 byte offset. We can't increase the size
2571	// of the second element because it might make us access off the end of the
2572	// struct.
2573	if (HiStart != 8) {
2574	// There are usually two sorts of types the ABI generation code can produce
2575	// for the low part of a pair that aren't 8 bytes in size: half, float or
2576	// i8/i16/i32. This can also include pointers when they are 32-bit (X32 and
2577	// NaCl).
2578	// Promote these to a larger type.
2579	if (Lo->isHalfTy() || Lo->isFloatTy())
2580	Lo = llvm::Type::getDoubleTy(C&: Lo->getContext());
2581	else {
2582	assert((Lo->isIntegerTy() || Lo->isPointerTy())
2583	&& "Invalid/unknown lo type");
2584	Lo = llvm::Type::getInt64Ty(C&: Lo->getContext());
2585	}
2586	}
2587
2588	llvm::StructType *Result = llvm::StructType::get(elt1: Lo, elts: Hi);
2589
2590	// Verify that the second element is at an 8-byte offset.
2591	assert(TD.getStructLayout(Result)->getElementOffset(1) == 8 &&
2592	"Invalid x86-64 argument pair!");
2593	return Result;
2594	}
2595
2596	ABIArgInfo X86_64ABIInfo::
2597	classifyReturnType(QualType RetTy) const {
2598	// AMD64-ABI 3.2.3p4: Rule 1. Classify the return type with the
2599	// classification algorithm.
2600	X86_64ABIInfo::Class Lo, Hi;
2601	classify(Ty: RetTy, OffsetBase: 0, Lo, Hi, /*isNamedArg*/ true);
2602
2603	// Check some invariants.
2604	assert((Hi != Memory || Lo == Memory) && "Invalid memory classification.");
2605	assert((Hi != SSEUp || Lo == SSE) && "Invalid SSEUp classification.");
2606
2607	llvm::Type *ResType = nullptr;
2608	switch (Lo) {
2609	case NoClass:
2610	if (Hi == NoClass)
2611	return ABIArgInfo::getIgnore();
2612	// If the low part is just padding, it takes no register, leave ResType
2613	// null.
2614	assert((Hi == SSE || Hi == Integer || Hi == X87Up) &&
2615	"Unknown missing lo part");
2616	break;
2617
2618	case SSEUp:
2619	case X87Up:
2620	llvm_unreachable("Invalid classification for lo word.");
2621
2622	// AMD64-ABI 3.2.3p4: Rule 2. Types of class memory are returned via
2623	// hidden argument.
2624	case Memory:
2625	return getIndirectReturnResult(Ty: RetTy);
2626
2627	// AMD64-ABI 3.2.3p4: Rule 3. If the class is INTEGER, the next
2628	// available register of the sequence %rax, %rdx is used.
2629	case Integer:
2630	ResType = GetINTEGERTypeAtOffset(IRType: CGT.ConvertType(T: RetTy), IROffset: 0, SourceTy: RetTy, SourceOffset: 0);
2631
2632	// If we have a sign or zero extended integer, make sure to return Extend
2633	// so that the parameter gets the right LLVM IR attributes.
2634	if (Hi == NoClass && isa<llvm::IntegerType>(Val: ResType)) {
2635	// Treat an enum type as its underlying type.
2636	if (const EnumType *EnumTy = RetTy->getAs<EnumType>())
2637	RetTy = EnumTy->getDecl()->getIntegerType();
2638
2639	if (RetTy->isIntegralOrEnumerationType() &&
2640	isPromotableIntegerTypeForABI(Ty: RetTy))
2641	return ABIArgInfo::getExtend(Ty: RetTy);
2642	}
2643	break;
2644
2645	// AMD64-ABI 3.2.3p4: Rule 4. If the class is SSE, the next
2646	// available SSE register of the sequence %xmm0, %xmm1 is used.
2647	case SSE:
2648	ResType = GetSSETypeAtOffset(IRType: CGT.ConvertType(T: RetTy), IROffset: 0, SourceTy: RetTy, SourceOffset: 0);
2649	break;
2650
2651	// AMD64-ABI 3.2.3p4: Rule 6. If the class is X87, the value is
2652	// returned on the X87 stack in %st0 as 80-bit x87 number.
2653	case X87:
2654	ResType = llvm::Type::getX86_FP80Ty(C&: getVMContext());
2655	break;
2656
2657	// AMD64-ABI 3.2.3p4: Rule 8. If the class is COMPLEX_X87, the real
2658	// part of the value is returned in %st0 and the imaginary part in
2659	// %st1.
2660	case ComplexX87:
2661	assert(Hi == ComplexX87 && "Unexpected ComplexX87 classification.");
2662	ResType = llvm::StructType::get(elt1: llvm::Type::getX86_FP80Ty(C&: getVMContext()),
2663	elts: llvm::Type::getX86_FP80Ty(C&: getVMContext()));
2664	break;
2665	}
2666
2667	llvm::Type *HighPart = nullptr;
2668	switch (Hi) {
2669	// Memory was handled previously and X87 should
2670	// never occur as a hi class.
2671	case Memory:
2672	case X87:
2673	llvm_unreachable("Invalid classification for hi word.");
2674
2675	case ComplexX87: // Previously handled.
2676	case NoClass:
2677	break;
2678
2679	case Integer:
2680	HighPart = GetINTEGERTypeAtOffset(IRType: CGT.ConvertType(T: RetTy), IROffset: 8, SourceTy: RetTy, SourceOffset: 8);
2681	if (Lo == NoClass) // Return HighPart at offset 8 in memory.
2682	return ABIArgInfo::getDirect(T: HighPart, Offset: 8);
2683	break;
2684	case SSE:
2685	HighPart = GetSSETypeAtOffset(IRType: CGT.ConvertType(T: RetTy), IROffset: 8, SourceTy: RetTy, SourceOffset: 8);
2686	if (Lo == NoClass) // Return HighPart at offset 8 in memory.
2687	return ABIArgInfo::getDirect(T: HighPart, Offset: 8);
2688	break;
2689
2690	// AMD64-ABI 3.2.3p4: Rule 5. If the class is SSEUP, the eightbyte
2691	// is passed in the next available eightbyte chunk if the last used
2692	// vector register.
2693	//
2694	// SSEUP should always be preceded by SSE, just widen.
2695	case SSEUp:
2696	assert(Lo == SSE && "Unexpected SSEUp classification.");
2697	ResType = GetByteVectorType(Ty: RetTy);
2698	break;
2699
2700	// AMD64-ABI 3.2.3p4: Rule 7. If the class is X87UP, the value is
2701	// returned together with the previous X87 value in %st0.
2702	case X87Up:
2703	// If X87Up is preceded by X87, we don't need to do
2704	// anything. However, in some cases with unions it may not be
2705	// preceded by X87. In such situations we follow gcc and pass the
2706	// extra bits in an SSE reg.
2707	if (Lo != X87) {
2708	HighPart = GetSSETypeAtOffset(IRType: CGT.ConvertType(T: RetTy), IROffset: 8, SourceTy: RetTy, SourceOffset: 8);
2709	if (Lo == NoClass) // Return HighPart at offset 8 in memory.
2710	return ABIArgInfo::getDirect(T: HighPart, Offset: 8);
2711	}
2712	break;
2713	}
2714
2715	// If a high part was specified, merge it together with the low part. It is
2716	// known to pass in the high eightbyte of the result. We do this by forming a
2717	// first class struct aggregate with the high and low part: {low, high}
2718	if (HighPart)
2719	ResType = GetX86_64ByValArgumentPair(Lo: ResType, Hi: HighPart, TD: getDataLayout());
2720
2721	return ABIArgInfo::getDirect(T: ResType);
2722	}
2723
2724	ABIArgInfo
2725	X86_64ABIInfo::classifyArgumentType(QualType Ty, unsigned freeIntRegs,
2726	unsigned &neededInt, unsigned &neededSSE,
2727	bool isNamedArg, bool IsRegCall) const {
2728	Ty = useFirstFieldIfTransparentUnion(Ty);
2729
2730	X86_64ABIInfo::Class Lo, Hi;
2731	classify(Ty, OffsetBase: 0, Lo, Hi, isNamedArg, IsRegCall);
2732
2733	// Check some invariants.
2734	// FIXME: Enforce these by construction.
2735	assert((Hi != Memory || Lo == Memory) && "Invalid memory classification.");
2736	assert((Hi != SSEUp || Lo == SSE) && "Invalid SSEUp classification.");
2737
2738	neededInt = 0;
2739	neededSSE = 0;
2740	llvm::Type *ResType = nullptr;
2741	switch (Lo) {
2742	case NoClass:
2743	if (Hi == NoClass)
2744	return ABIArgInfo::getIgnore();
2745	// If the low part is just padding, it takes no register, leave ResType
2746	// null.
2747	assert((Hi == SSE || Hi == Integer || Hi == X87Up) &&
2748	"Unknown missing lo part");
2749	break;
2750
2751	// AMD64-ABI 3.2.3p3: Rule 1. If the class is MEMORY, pass the argument
2752	// on the stack.
2753	case Memory:
2754
2755	// AMD64-ABI 3.2.3p3: Rule 5. If the class is X87, X87UP or
2756	// COMPLEX_X87, it is passed in memory.
2757	case X87:
2758	case ComplexX87:
2759	if (getRecordArgABI(T: Ty, CXXABI&: getCXXABI()) == CGCXXABI::RAA_Indirect)
2760	++neededInt;
2761	return getIndirectResult(Ty, freeIntRegs);
2762
2763	case SSEUp:
2764	case X87Up:
2765	llvm_unreachable("Invalid classification for lo word.");
2766
2767	// AMD64-ABI 3.2.3p3: Rule 2. If the class is INTEGER, the next
2768	// available register of the sequence %rdi, %rsi, %rdx, %rcx, %r8
2769	// and %r9 is used.
2770	case Integer:
2771	++neededInt;
2772
2773	// Pick an 8-byte type based on the preferred type.
2774	ResType = GetINTEGERTypeAtOffset(IRType: CGT.ConvertType(T: Ty), IROffset: 0, SourceTy: Ty, SourceOffset: 0);
2775
2776	// If we have a sign or zero extended integer, make sure to return Extend
2777	// so that the parameter gets the right LLVM IR attributes.
2778	if (Hi == NoClass && isa<llvm::IntegerType>(Val: ResType)) {
2779	// Treat an enum type as its underlying type.
2780	if (const EnumType *EnumTy = Ty->getAs<EnumType>())
2781	Ty = EnumTy->getDecl()->getIntegerType();
2782
2783	if (Ty->isIntegralOrEnumerationType() &&
2784	isPromotableIntegerTypeForABI(Ty))
2785	return ABIArgInfo::getExtend(Ty, T: CGT.ConvertType(T: Ty));
2786	}
2787
2788	break;
2789
2790	// AMD64-ABI 3.2.3p3: Rule 3. If the class is SSE, the next
2791	// available SSE register is used, the registers are taken in the
2792	// order from %xmm0 to %xmm7.
2793	case SSE: {
2794	llvm::Type *IRType = CGT.ConvertType(T: Ty);
2795	ResType = GetSSETypeAtOffset(IRType, IROffset: 0, SourceTy: Ty, SourceOffset: 0);
2796	++neededSSE;
2797	break;
2798	}
2799	}
2800
2801	llvm::Type *HighPart = nullptr;
2802	switch (Hi) {
2803	// Memory was handled previously, ComplexX87 and X87 should
2804	// never occur as hi classes, and X87Up must be preceded by X87,
2805	// which is passed in memory.
2806	case Memory:
2807	case X87:
2808	case ComplexX87:
2809	llvm_unreachable("Invalid classification for hi word.");
2810
2811	case NoClass: break;
2812
2813	case Integer:
2814	++neededInt;
2815	// Pick an 8-byte type based on the preferred type.
2816	HighPart = GetINTEGERTypeAtOffset(IRType: CGT.ConvertType(T: Ty), IROffset: 8, SourceTy: Ty, SourceOffset: 8);
2817
2818	if (Lo == NoClass) // Pass HighPart at offset 8 in memory.
2819	return ABIArgInfo::getDirect(T: HighPart, Offset: 8);
2820	break;
2821
2822	// X87Up generally doesn't occur here (long double is passed in
2823	// memory), except in situations involving unions.
2824	case X87Up:
2825	case SSE:
2826	++neededSSE;
2827	HighPart = GetSSETypeAtOffset(IRType: CGT.ConvertType(T: Ty), IROffset: 8, SourceTy: Ty, SourceOffset: 8);
2828
2829	if (Lo == NoClass) // Pass HighPart at offset 8 in memory.
2830	return ABIArgInfo::getDirect(T: HighPart, Offset: 8);
2831	break;
2832
2833	// AMD64-ABI 3.2.3p3: Rule 4. If the class is SSEUP, the
2834	// eightbyte is passed in the upper half of the last used SSE
2835	// register. This only happens when 128-bit vectors are passed.
2836	case SSEUp:
2837	assert(Lo == SSE && "Unexpected SSEUp classification");
2838	ResType = GetByteVectorType(Ty);
2839	break;
2840	}
2841
2842	// If a high part was specified, merge it together with the low part. It is
2843	// known to pass in the high eightbyte of the result. We do this by forming a
2844	// first class struct aggregate with the high and low part: {low, high}
2845	if (HighPart)
2846	ResType = GetX86_64ByValArgumentPair(Lo: ResType, Hi: HighPart, TD: getDataLayout());
2847
2848	return ABIArgInfo::getDirect(T: ResType);
2849	}
2850
2851	ABIArgInfo
2852	X86_64ABIInfo::classifyRegCallStructTypeImpl(QualType Ty, unsigned &NeededInt,
2853	unsigned &NeededSSE,
2854	unsigned &MaxVectorWidth) const {
2855	auto RT = Ty->getAs<RecordType>();
2856	assert(RT && "classifyRegCallStructType only valid with struct types");
2857
2858	if (RT->getDecl()->hasFlexibleArrayMember())
2859	return getIndirectReturnResult(Ty);
2860
2861	// Sum up bases
2862	if (auto CXXRD = dyn_cast<CXXRecordDecl>(Val: RT->getDecl())) {
2863	if (CXXRD->isDynamicClass()) {
2864	NeededInt = NeededSSE = 0;
2865	return getIndirectReturnResult(Ty);
2866	}
2867
2868	for (const auto &I : CXXRD->bases())
2869	if (classifyRegCallStructTypeImpl(Ty: I.getType(), NeededInt, NeededSSE,
2870	MaxVectorWidth)
2871	.isIndirect()) {
2872	NeededInt = NeededSSE = 0;
2873	return getIndirectReturnResult(Ty);
2874	}
2875	}
2876
2877	// Sum up members
2878	for (const auto *FD : RT->getDecl()->fields()) {
2879	QualType MTy = FD->getType();
2880	if (MTy->isRecordType() && !MTy->isUnionType()) {
2881	if (classifyRegCallStructTypeImpl(Ty: MTy, NeededInt, NeededSSE,
2882	MaxVectorWidth)
2883	.isIndirect()) {
2884	NeededInt = NeededSSE = 0;
2885	return getIndirectReturnResult(Ty);
2886	}
2887	} else {
2888	unsigned LocalNeededInt, LocalNeededSSE;
2889	if (classifyArgumentType(Ty: MTy, UINT_MAX, neededInt&: LocalNeededInt, neededSSE&: LocalNeededSSE,
2890	isNamedArg: true, IsRegCall: true)
2891	.isIndirect()) {
2892	NeededInt = NeededSSE = 0;
2893	return getIndirectReturnResult(Ty);
2894	}
2895	if (const auto *AT = getContext().getAsConstantArrayType(MTy))
2896	MTy = AT->getElementType();
2897	if (const auto *VT = MTy->getAs<VectorType>())
2898	if (getContext().getTypeSize(VT) > MaxVectorWidth)
2899	MaxVectorWidth = getContext().getTypeSize(VT);
2900	NeededInt += LocalNeededInt;
2901	NeededSSE += LocalNeededSSE;
2902	}
2903	}
2904
2905	return ABIArgInfo::getDirect();
2906	}
2907
2908	ABIArgInfo
2909	X86_64ABIInfo::classifyRegCallStructType(QualType Ty, unsigned &NeededInt,
2910	unsigned &NeededSSE,
2911	unsigned &MaxVectorWidth) const {
2912
2913	NeededInt = 0;
2914	NeededSSE = 0;
2915	MaxVectorWidth = 0;
2916
2917	return classifyRegCallStructTypeImpl(Ty, NeededInt, NeededSSE,
2918	MaxVectorWidth);
2919	}
2920
2921	void X86_64ABIInfo::computeInfo(CGFunctionInfo &FI) const {
2922
2923	const unsigned CallingConv = FI.getCallingConvention();
2924	// It is possible to force Win64 calling convention on any x86_64 target by
2925	// using __attribute__((ms_abi)). In such case to correctly emit Win64
2926	// compatible code delegate this call to WinX86_64ABIInfo::computeInfo.
2927	if (CallingConv == llvm::CallingConv::Win64) {
2928	WinX86_64ABIInfo Win64ABIInfo(CGT, AVXLevel);
2929	Win64ABIInfo.computeInfo(FI);
2930	return;
2931	}
2932
2933	bool IsRegCall = CallingConv == llvm::CallingConv::X86_RegCall;
2934
2935	// Keep track of the number of assigned registers.
2936	unsigned FreeIntRegs = IsRegCall ? 11 : 6;
2937	unsigned FreeSSERegs = IsRegCall ? 16 : 8;
2938	unsigned NeededInt = 0, NeededSSE = 0, MaxVectorWidth = 0;
2939
2940	if (!::classifyReturnType(CXXABI: getCXXABI(), FI, Info: *this)) {
2941	if (IsRegCall && FI.getReturnType()->getTypePtr()->isRecordType() &&
2942	!FI.getReturnType()->getTypePtr()->isUnionType()) {
2943	FI.getReturnInfo() = classifyRegCallStructType(
2944	Ty: FI.getReturnType(), NeededInt, NeededSSE, MaxVectorWidth);
2945	if (FreeIntRegs >= NeededInt && FreeSSERegs >= NeededSSE) {
2946	FreeIntRegs -= NeededInt;
2947	FreeSSERegs -= NeededSSE;
2948	} else {
2949	FI.getReturnInfo() = getIndirectReturnResult(Ty: FI.getReturnType());
2950	}
2951	} else if (IsRegCall && FI.getReturnType()->getAs<ComplexType>() &&
2952	getContext().getCanonicalType(FI.getReturnType()
2953	->getAs<ComplexType>()
2954	->getElementType()) ==
2955	getContext().LongDoubleTy)
2956	// Complex Long Double Type is passed in Memory when Regcall
2957	// calling convention is used.
2958	FI.getReturnInfo() = getIndirectReturnResult(Ty: FI.getReturnType());
2959	else
2960	FI.getReturnInfo() = classifyReturnType(RetTy: FI.getReturnType());
2961	}
2962
2963	// If the return value is indirect, then the hidden argument is consuming one
2964	// integer register.
2965	if (FI.getReturnInfo().isIndirect())
2966	--FreeIntRegs;
2967	else if (NeededSSE && MaxVectorWidth > 0)
2968	FI.setMaxVectorWidth(MaxVectorWidth);
2969
2970	// The chain argument effectively gives us another free register.
2971	if (FI.isChainCall())
2972	++FreeIntRegs;
2973
2974	unsigned NumRequiredArgs = FI.getNumRequiredArgs();
2975	// AMD64-ABI 3.2.3p3: Once arguments are classified, the registers
2976	// get assigned (in left-to-right order) for passing as follows...
2977	unsigned ArgNo = 0;
2978	for (CGFunctionInfo::arg_iterator it = FI.arg_begin(), ie = FI.arg_end();
2979	it != ie; ++it, ++ArgNo) {
2980	bool IsNamedArg = ArgNo < NumRequiredArgs;
2981
2982	if (IsRegCall && it->type->isStructureOrClassType())
2983	it->info = classifyRegCallStructType(Ty: it->type, NeededInt, NeededSSE,
2984	MaxVectorWidth);
2985	else
2986	it->info = classifyArgumentType(Ty: it->type, freeIntRegs: FreeIntRegs, neededInt&: NeededInt,
2987	neededSSE&: NeededSSE, isNamedArg: IsNamedArg);
2988
2989	// AMD64-ABI 3.2.3p3: If there are no registers available for any
2990	// eightbyte of an argument, the whole argument is passed on the
2991	// stack. If registers have already been assigned for some
2992	// eightbytes of such an argument, the assignments get reverted.
2993	if (FreeIntRegs >= NeededInt && FreeSSERegs >= NeededSSE) {
2994	FreeIntRegs -= NeededInt;
2995	FreeSSERegs -= NeededSSE;
2996	if (MaxVectorWidth > FI.getMaxVectorWidth())
2997	FI.setMaxVectorWidth(MaxVectorWidth);
2998	} else {
2999	it->info = getIndirectResult(Ty: it->type, freeIntRegs: FreeIntRegs);
3000	}
3001	}
3002	}
3003
3004	static Address EmitX86_64VAArgFromMemory(CodeGenFunction &CGF,
3005	Address VAListAddr, QualType Ty) {
3006	Address overflow_arg_area_p =
3007	CGF.Builder.CreateStructGEP(Addr: VAListAddr, Index: 2, Name: "overflow_arg_area_p");
3008	llvm::Value *overflow_arg_area =
3009	CGF.Builder.CreateLoad(Addr: overflow_arg_area_p, Name: "overflow_arg_area");
3010
3011	// AMD64-ABI 3.5.7p5: Step 7. Align l->overflow_arg_area upwards to a 16
3012	// byte boundary if alignment needed by type exceeds 8 byte boundary.
3013	// It isn't stated explicitly in the standard, but in practice we use
3014	// alignment greater than 16 where necessary.
3015	CharUnits Align = CGF.getContext().getTypeAlignInChars(T: Ty);
3016	if (Align > CharUnits::fromQuantity(Quantity: 8)) {
3017	overflow_arg_area = emitRoundPointerUpToAlignment(CGF, Ptr: overflow_arg_area,
3018	Align);
3019	}
3020
3021	// AMD64-ABI 3.5.7p5: Step 8. Fetch type from l->overflow_arg_area.
3022	llvm::Type *LTy = CGF.ConvertTypeForMem(T: Ty);
3023	llvm::Value *Res = overflow_arg_area;
3024
3025	// AMD64-ABI 3.5.7p5: Step 9. Set l->overflow_arg_area to:
3026	// l->overflow_arg_area + sizeof(type).
3027	// AMD64-ABI 3.5.7p5: Step 10. Align l->overflow_arg_area upwards to
3028	// an 8 byte boundary.
3029
3030	uint64_t SizeInBytes = (CGF.getContext().getTypeSize(T: Ty) + 7) / 8;
3031	llvm::Value *Offset =
3032	llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: (SizeInBytes + 7) & ~7);
3033	overflow_arg_area = CGF.Builder.CreateGEP(Ty: CGF.Int8Ty, Ptr: overflow_arg_area,
3034	IdxList: Offset, Name: "overflow_arg_area.next");
3035	CGF.Builder.CreateStore(Val: overflow_arg_area, Addr: overflow_arg_area_p);
3036
3037	// AMD64-ABI 3.5.7p5: Step 11. Return the fetched type.
3038	return Address(Res, LTy, Align);
3039	}
3040
3041	RValue X86_64ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
3042	QualType Ty, AggValueSlot Slot) const {
3043	// Assume that va_list type is correct; should be pointer to LLVM type:
3044	// struct {
3045	// i32 gp_offset;
3046	// i32 fp_offset;
3047	// i8* overflow_arg_area;
3048	// i8* reg_save_area;
3049	// };
3050	unsigned neededInt, neededSSE;
3051
3052	Ty = getContext().getCanonicalType(T: Ty);
3053	ABIArgInfo AI = classifyArgumentType(Ty, freeIntRegs: 0, neededInt, neededSSE,
3054	/*isNamedArg*/false);
3055
3056	// Empty records are ignored for parameter passing purposes.
3057	if (AI.isIgnore())
3058	return Slot.asRValue();
3059
3060	// AMD64-ABI 3.5.7p5: Step 1. Determine whether type may be passed
3061	// in the registers. If not go to step 7.
3062	if (!neededInt && !neededSSE)
3063	return CGF.EmitLoadOfAnyValue(
3064	V: CGF.MakeAddrLValue(Addr: EmitX86_64VAArgFromMemory(CGF, VAListAddr, Ty), T: Ty),
3065	Slot);
3066
3067	// AMD64-ABI 3.5.7p5: Step 2. Compute num_gp to hold the number of
3068	// general purpose registers needed to pass type and num_fp to hold
3069	// the number of floating point registers needed.
3070
3071	// AMD64-ABI 3.5.7p5: Step 3. Verify whether arguments fit into
3072	// registers. In the case: l->gp_offset > 48 - num_gp * 8 or
3073	// l->fp_offset > 304 - num_fp * 16 go to step 7.
3074	//
3075	// NOTE: 304 is a typo, there are (6 * 8 + 8 * 16) = 176 bytes of
3076	// register save space).
3077
3078	llvm::Value *InRegs = nullptr;
3079	Address gp_offset_p = Address::invalid(), fp_offset_p = Address::invalid();
3080	llvm::Value *gp_offset = nullptr, *fp_offset = nullptr;
3081	if (neededInt) {
3082	gp_offset_p = CGF.Builder.CreateStructGEP(Addr: VAListAddr, Index: 0, Name: "gp_offset_p");
3083	gp_offset = CGF.Builder.CreateLoad(Addr: gp_offset_p, Name: "gp_offset");
3084	InRegs = llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: 48 - neededInt * 8);
3085	InRegs = CGF.Builder.CreateICmpULE(LHS: gp_offset, RHS: InRegs, Name: "fits_in_gp");
3086	}
3087
3088	if (neededSSE) {
3089	fp_offset_p = CGF.Builder.CreateStructGEP(Addr: VAListAddr, Index: 1, Name: "fp_offset_p");
3090	fp_offset = CGF.Builder.CreateLoad(Addr: fp_offset_p, Name: "fp_offset");
3091	llvm::Value *FitsInFP =
3092	llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: 176 - neededSSE * 16);
3093	FitsInFP = CGF.Builder.CreateICmpULE(LHS: fp_offset, RHS: FitsInFP, Name: "fits_in_fp");
3094	InRegs = InRegs ? CGF.Builder.CreateAnd(LHS: InRegs, RHS: FitsInFP) : FitsInFP;
3095	}
3096
3097	llvm::BasicBlock *InRegBlock = CGF.createBasicBlock(name: "vaarg.in_reg");
3098	llvm::BasicBlock *InMemBlock = CGF.createBasicBlock(name: "vaarg.in_mem");
3099	llvm::BasicBlock *ContBlock = CGF.createBasicBlock(name: "vaarg.end");
3100	CGF.Builder.CreateCondBr(Cond: InRegs, True: InRegBlock, False: InMemBlock);
3101
3102	// Emit code to load the value if it was passed in registers.
3103
3104	CGF.EmitBlock(BB: InRegBlock);
3105
3106	// AMD64-ABI 3.5.7p5: Step 4. Fetch type from l->reg_save_area with
3107	// an offset of l->gp_offset and/or l->fp_offset. This may require
3108	// copying to a temporary location in case the parameter is passed
3109	// in different register classes or requires an alignment greater
3110	// than 8 for general purpose registers and 16 for XMM registers.
3111	//
3112	// FIXME: This really results in shameful code when we end up needing to
3113	// collect arguments from different places; often what should result in a
3114	// simple assembling of a structure from scattered addresses has many more
3115	// loads than necessary. Can we clean this up?
3116	llvm::Type *LTy = CGF.ConvertTypeForMem(T: Ty);
3117	llvm::Value *RegSaveArea = CGF.Builder.CreateLoad(
3118	Addr: CGF.Builder.CreateStructGEP(Addr: VAListAddr, Index: 3), Name: "reg_save_area");
3119
3120	Address RegAddr = Address::invalid();
3121	if (neededInt && neededSSE) {
3122	// FIXME: Cleanup.
3123	assert(AI.isDirect() && "Unexpected ABI info for mixed regs");
3124	llvm::StructType *ST = cast<llvm::StructType>(Val: AI.getCoerceToType());
3125	Address Tmp = CGF.CreateMemTemp(T: Ty);
3126	Tmp = Tmp.withElementType(ElemTy: ST);
3127	assert(ST->getNumElements() == 2 && "Unexpected ABI info for mixed regs");
3128	llvm::Type *TyLo = ST->getElementType(N: 0);
3129	llvm::Type *TyHi = ST->getElementType(N: 1);
3130	assert((TyLo->isFPOrFPVectorTy() ^ TyHi->isFPOrFPVectorTy()) &&
3131	"Unexpected ABI info for mixed regs");
3132	llvm::Value *GPAddr =
3133	CGF.Builder.CreateGEP(Ty: CGF.Int8Ty, Ptr: RegSaveArea, IdxList: gp_offset);
3134	llvm::Value *FPAddr =
3135	CGF.Builder.CreateGEP(Ty: CGF.Int8Ty, Ptr: RegSaveArea, IdxList: fp_offset);
3136	llvm::Value *RegLoAddr = TyLo->isFPOrFPVectorTy() ? FPAddr : GPAddr;
3137	llvm::Value *RegHiAddr = TyLo->isFPOrFPVectorTy() ? GPAddr : FPAddr;
3138
3139	// Copy the first element.
3140	// FIXME: Our choice of alignment here and below is probably pessimistic.
3141	llvm::Value *V = CGF.Builder.CreateAlignedLoad(
3142	Ty: TyLo, Addr: RegLoAddr,
3143	Align: CharUnits::fromQuantity(Quantity: getDataLayout().getABITypeAlign(Ty: TyLo)));
3144	CGF.Builder.CreateStore(Val: V, Addr: CGF.Builder.CreateStructGEP(Addr: Tmp, Index: 0));
3145
3146	// Copy the second element.
3147	V = CGF.Builder.CreateAlignedLoad(
3148	Ty: TyHi, Addr: RegHiAddr,
3149	Align: CharUnits::fromQuantity(Quantity: getDataLayout().getABITypeAlign(Ty: TyHi)));
3150	CGF.Builder.CreateStore(Val: V, Addr: CGF.Builder.CreateStructGEP(Addr: Tmp, Index: 1));
3151
3152	RegAddr = Tmp.withElementType(ElemTy: LTy);
3153	} else if (neededInt || neededSSE == 1) {
3154	// Copy to a temporary if necessary to ensure the appropriate alignment.
3155	auto TInfo = getContext().getTypeInfoInChars(T: Ty);
3156	uint64_t TySize = TInfo.Width.getQuantity();
3157	CharUnits TyAlign = TInfo.Align;
3158	llvm::Type *CoTy = nullptr;
3159	if (AI.isDirect())
3160	CoTy = AI.getCoerceToType();
3161
3162	llvm::Value *GpOrFpOffset = neededInt ? gp_offset : fp_offset;
3163	uint64_t Alignment = neededInt ? 8 : 16;
3164	uint64_t RegSize = neededInt ? neededInt * 8 : 16;
3165	// There are two cases require special handling:
3166	// 1)
3167	// ```
3168	// struct {
3169	// struct {} a[8];
3170	// int b;
3171	// };
3172	// ```
3173	// The lower 8 bytes of the structure are not stored,
3174	// so an 8-byte offset is needed when accessing the structure.
3175	// 2)
3176	// ```
3177	// struct {
3178	// long long a;
3179	// struct {} b;
3180	// };
3181	// ```
3182	// The stored size of this structure is smaller than its actual size,
3183	// which may lead to reading past the end of the register save area.
3184	if (CoTy && (AI.getDirectOffset() == 8 || RegSize < TySize)) {
3185	Address Tmp = CGF.CreateMemTemp(T: Ty);
3186	llvm::Value *Addr =
3187	CGF.Builder.CreateGEP(Ty: CGF.Int8Ty, Ptr: RegSaveArea, IdxList: GpOrFpOffset);
3188	llvm::Value *Src = CGF.Builder.CreateAlignedLoad(Ty: CoTy, Addr, Align: TyAlign);
3189	llvm::Value *PtrOffset =
3190	llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: AI.getDirectOffset());
3191	Address Dst = Address(
3192	CGF.Builder.CreateGEP(Ty: CGF.Int8Ty, Ptr: Tmp.getBasePointer(), IdxList: PtrOffset),
3193	LTy, TyAlign);
3194	CGF.Builder.CreateStore(Val: Src, Addr: Dst);
3195	RegAddr = Tmp.withElementType(ElemTy: LTy);
3196	} else {
3197	RegAddr =
3198	Address(CGF.Builder.CreateGEP(Ty: CGF.Int8Ty, Ptr: RegSaveArea, IdxList: GpOrFpOffset),
3199	LTy, CharUnits::fromQuantity(Quantity: Alignment));
3200
3201	// Copy into a temporary if the type is more aligned than the
3202	// register save area.
3203	if (neededInt && TyAlign.getQuantity() > 8) {
3204	Address Tmp = CGF.CreateMemTemp(T: Ty);
3205	CGF.Builder.CreateMemCpy(Dest: Tmp, Src: RegAddr, Size: TySize, IsVolatile: false);
3206	RegAddr = Tmp;
3207	}
3208	}
3209
3210	} else {
3211	assert(neededSSE == 2 && "Invalid number of needed registers!");
3212	// SSE registers are spaced 16 bytes apart in the register save
3213	// area, we need to collect the two eightbytes together.
3214	// The ABI isn't explicit about this, but it seems reasonable
3215	// to assume that the slots are 16-byte aligned, since the stack is
3216	// naturally 16-byte aligned and the prologue is expected to store
3217	// all the SSE registers to the RSA.
3218	Address RegAddrLo = Address(CGF.Builder.CreateGEP(Ty: CGF.Int8Ty, Ptr: RegSaveArea,
3219	IdxList: fp_offset),
3220	CGF.Int8Ty, CharUnits::fromQuantity(Quantity: 16));
3221	Address RegAddrHi =
3222	CGF.Builder.CreateConstInBoundsByteGEP(Addr: RegAddrLo,
3223	Offset: CharUnits::fromQuantity(Quantity: 16));
3224	llvm::Type *ST = AI.canHaveCoerceToType()
3225	? AI.getCoerceToType()
3226	: llvm::StructType::get(elt1: CGF.DoubleTy, elts: CGF.DoubleTy);
3227	llvm::Value *V;
3228	Address Tmp = CGF.CreateMemTemp(T: Ty);
3229	Tmp = Tmp.withElementType(ElemTy: ST);
3230	V = CGF.Builder.CreateLoad(
3231	Addr: RegAddrLo.withElementType(ElemTy: ST->getStructElementType(N: 0)));
3232	CGF.Builder.CreateStore(Val: V, Addr: CGF.Builder.CreateStructGEP(Addr: Tmp, Index: 0));
3233	V = CGF.Builder.CreateLoad(
3234	Addr: RegAddrHi.withElementType(ElemTy: ST->getStructElementType(N: 1)));
3235	CGF.Builder.CreateStore(Val: V, Addr: CGF.Builder.CreateStructGEP(Addr: Tmp, Index: 1));
3236
3237	RegAddr = Tmp.withElementType(ElemTy: LTy);
3238	}
3239
3240	// AMD64-ABI 3.5.7p5: Step 5. Set:
3241	// l->gp_offset = l->gp_offset + num_gp * 8
3242	// l->fp_offset = l->fp_offset + num_fp * 16.
3243	if (neededInt) {
3244	llvm::Value *Offset = llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: neededInt * 8);
3245	CGF.Builder.CreateStore(Val: CGF.Builder.CreateAdd(LHS: gp_offset, RHS: Offset),
3246	Addr: gp_offset_p);
3247	}
3248	if (neededSSE) {
3249	llvm::Value *Offset = llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: neededSSE * 16);
3250	CGF.Builder.CreateStore(Val: CGF.Builder.CreateAdd(LHS: fp_offset, RHS: Offset),
3251	Addr: fp_offset_p);
3252	}
3253	CGF.EmitBranch(Block: ContBlock);
3254
3255	// Emit code to load the value if it was passed in memory.
3256
3257	CGF.EmitBlock(BB: InMemBlock);
3258	Address MemAddr = EmitX86_64VAArgFromMemory(CGF, VAListAddr, Ty);
3259
3260	// Return the appropriate result.
3261
3262	CGF.EmitBlock(BB: ContBlock);
3263	Address ResAddr = emitMergePHI(CGF, Addr1: RegAddr, Block1: InRegBlock, Addr2: MemAddr, Block2: InMemBlock,
3264	Name: "vaarg.addr");
3265	return CGF.EmitLoadOfAnyValue(V: CGF.MakeAddrLValue(Addr: ResAddr, T: Ty), Slot);
3266	}
3267
3268	RValue X86_64ABIInfo::EmitMSVAArg(CodeGenFunction &CGF, Address VAListAddr,
3269	QualType Ty, AggValueSlot Slot) const {
3270	// MS x64 ABI requirement: "Any argument that doesn't fit in 8 bytes, or is
3271	// not 1, 2, 4, or 8 bytes, must be passed by reference."
3272	uint64_t Width = getContext().getTypeSize(T: Ty);
3273	bool IsIndirect = Width > 64 || !llvm::isPowerOf2_64(Value: Width);
3274
3275	return emitVoidPtrVAArg(CGF, VAListAddr, ValueTy: Ty, IsIndirect,
3276	ValueInfo: CGF.getContext().getTypeInfoInChars(T: Ty),
3277	SlotSizeAndAlign: CharUnits::fromQuantity(Quantity: 8),
3278	/*allowHigherAlign*/ AllowHigherAlign: false, Slot);
3279	}
3280
3281	ABIArgInfo WinX86_64ABIInfo::reclassifyHvaArgForVectorCall(
3282	QualType Ty, unsigned &FreeSSERegs, const ABIArgInfo &current) const {
3283	const Type *Base = nullptr;
3284	uint64_t NumElts = 0;
3285
3286	if (!Ty->isBuiltinType() && !Ty->isVectorType() &&
3287	isHomogeneousAggregate(Ty, Base, Members&: NumElts) && FreeSSERegs >= NumElts) {
3288	FreeSSERegs -= NumElts;
3289	return getDirectX86Hva();
3290	}
3291	return current;
3292	}
3293
3294	ABIArgInfo WinX86_64ABIInfo::classify(QualType Ty, unsigned &FreeSSERegs,
3295	bool IsReturnType, bool IsVectorCall,
3296	bool IsRegCall) const {
3297
3298	if (Ty->isVoidType())
3299	return ABIArgInfo::getIgnore();
3300
3301	if (const EnumType *EnumTy = Ty->getAs<EnumType>())
3302	Ty = EnumTy->getDecl()->getIntegerType();
3303
3304	TypeInfo Info = getContext().getTypeInfo(T: Ty);
3305	uint64_t Width = Info.Width;
3306	CharUnits Align = getContext().toCharUnitsFromBits(BitSize: Info.Align);
3307
3308	const RecordType *RT = Ty->getAs<RecordType>();
3309	if (RT) {
3310	if (!IsReturnType) {
3311	if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(RT, CXXABI&: getCXXABI()))
3312	return getNaturalAlignIndirect(Ty, AddrSpace: getDataLayout().getAllocaAddrSpace(),
3313	ByVal: RAA == CGCXXABI::RAA_DirectInMemory);
3314	}
3315
3316	if (RT->getDecl()->hasFlexibleArrayMember())
3317	return getNaturalAlignIndirect(Ty, AddrSpace: getDataLayout().getAllocaAddrSpace(),
3318	/*ByVal=*/false);
3319	}
3320
3321	const Type *Base = nullptr;
3322	uint64_t NumElts = 0;
3323	// vectorcall adds the concept of a homogenous vector aggregate, similar to
3324	// other targets.
3325	if ((IsVectorCall || IsRegCall) &&
3326	isHomogeneousAggregate(Ty, Base, Members&: NumElts)) {
3327	if (IsRegCall) {
3328	if (FreeSSERegs >= NumElts) {
3329	FreeSSERegs -= NumElts;
3330	if (IsReturnType || Ty->isBuiltinType() || Ty->isVectorType())
3331	return ABIArgInfo::getDirect();
3332	return ABIArgInfo::getExpand();
3333	}
3334	return ABIArgInfo::getIndirect(
3335	Alignment: Align, /*AddrSpace=*/getDataLayout().getAllocaAddrSpace(),
3336	/*ByVal=*/false);
3337	} else if (IsVectorCall) {
3338	if (FreeSSERegs >= NumElts &&
3339	(IsReturnType || Ty->isBuiltinType() || Ty->isVectorType())) {
3340	FreeSSERegs -= NumElts;
3341	return ABIArgInfo::getDirect();
3342	} else if (IsReturnType) {
3343	return ABIArgInfo::getExpand();
3344	} else if (!Ty->isBuiltinType() && !Ty->isVectorType()) {
3345	// HVAs are delayed and reclassified in the 2nd step.
3346	return ABIArgInfo::getIndirect(
3347	Alignment: Align, /*AddrSpace=*/getDataLayout().getAllocaAddrSpace(),
3348	/*ByVal=*/false);
3349	}
3350	}
3351	}
3352
3353	if (Ty->isMemberPointerType()) {
3354	// If the member pointer is represented by an LLVM int or ptr, pass it
3355	// directly.
3356	llvm::Type *LLTy = CGT.ConvertType(T: Ty);
3357	if (LLTy->isPointerTy() || LLTy->isIntegerTy())
3358	return ABIArgInfo::getDirect();
3359	}
3360
3361	if (RT || Ty->isAnyComplexType() || Ty->isMemberPointerType()) {
3362	// MS x64 ABI requirement: "Any argument that doesn't fit in 8 bytes, or is
3363	// not 1, 2, 4, or 8 bytes, must be passed by reference."
3364	if (Width > 64 || !llvm::isPowerOf2_64(Value: Width))
3365	return getNaturalAlignIndirect(Ty, AddrSpace: getDataLayout().getAllocaAddrSpace(),
3366	/*ByVal=*/false);
3367
3368	// Otherwise, coerce it to a small integer.
3369	return ABIArgInfo::getDirect(T: llvm::IntegerType::get(C&: getVMContext(), NumBits: Width));
3370	}
3371
3372	if (const BuiltinType *BT = Ty->getAs<BuiltinType>()) {
3373	switch (BT->getKind()) {
3374	case BuiltinType::Bool:
3375	// Bool type is always extended to the ABI, other builtin types are not
3376	// extended.
3377	return ABIArgInfo::getExtend(Ty);
3378
3379	case BuiltinType::LongDouble:
3380	// Mingw64 GCC uses the old 80 bit extended precision floating point
3381	// unit. It passes them indirectly through memory.
3382	if (IsMingw64) {
3383	const llvm::fltSemantics *LDF = &getTarget().getLongDoubleFormat();
3384	if (LDF == &llvm::APFloat::x87DoubleExtended())
3385	return ABIArgInfo::getIndirect(
3386	Alignment: Align, /*AddrSpace=*/getDataLayout().getAllocaAddrSpace(),
3387	/*ByVal=*/false);
3388	}
3389	break;
3390
3391	case BuiltinType::Int128:
3392	case BuiltinType::UInt128:
3393	case BuiltinType::Float128:
3394	// 128-bit float and integer types share the same ABI.
3395
3396	// If it's a parameter type, the normal ABI rule is that arguments larger
3397	// than 8 bytes are passed indirectly. GCC follows it. We follow it too,
3398	// even though it isn't particularly efficient.
3399	if (!IsReturnType)
3400	return ABIArgInfo::getIndirect(
3401	Alignment: Align, /*AddrSpace=*/getDataLayout().getAllocaAddrSpace(),
3402	/*ByVal=*/false);
3403
3404	// Mingw64 GCC returns i128 in XMM0. Coerce to v2i64 to handle that.
3405	// Clang matches them for compatibility.
3406	// NOTE: GCC actually returns f128 indirectly but will hopefully change.
3407	// See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=115054#c8.
3408	return ABIArgInfo::getDirect(T: llvm::FixedVectorType::get(
3409	ElementType: llvm::Type::getInt64Ty(C&: getVMContext()), NumElts: 2));
3410
3411	default:
3412	break;
3413	}
3414	}
3415
3416	if (Ty->isBitIntType()) {
3417	// MS x64 ABI requirement: "Any argument that doesn't fit in 8 bytes, or is
3418	// not 1, 2, 4, or 8 bytes, must be passed by reference."
3419	// However, non-power-of-two bit-precise integers will be passed as 1, 2, 4,
3420	// or 8 bytes anyway as long is it fits in them, so we don't have to check
3421	// the power of 2.
3422	if (Width <= 64)
3423	return ABIArgInfo::getDirect();
3424	return ABIArgInfo::getIndirect(
3425	Alignment: Align, /*AddrSpace=*/getDataLayout().getAllocaAddrSpace(),
3426	/*ByVal=*/false);
3427	}
3428
3429	return ABIArgInfo::getDirect();
3430	}
3431
3432	void WinX86_64ABIInfo::computeInfo(CGFunctionInfo &FI) const {
3433	const unsigned CC = FI.getCallingConvention();
3434	bool IsVectorCall = CC == llvm::CallingConv::X86_VectorCall;
3435	bool IsRegCall = CC == llvm::CallingConv::X86_RegCall;
3436
3437	// If __attribute__((sysv_abi)) is in use, use the SysV argument
3438	// classification rules.
3439	if (CC == llvm::CallingConv::X86_64_SysV) {
3440	X86_64ABIInfo SysVABIInfo(CGT, AVXLevel);
3441	SysVABIInfo.computeInfo(FI);
3442	return;
3443	}
3444
3445	unsigned FreeSSERegs = 0;
3446	if (IsVectorCall) {
3447	// We can use up to 4 SSE return registers with vectorcall.
3448	FreeSSERegs = 4;
3449	} else if (IsRegCall) {
3450	// RegCall gives us 16 SSE registers.
3451	FreeSSERegs = 16;
3452	}
3453
3454	if (!getCXXABI().classifyReturnType(FI))
3455	FI.getReturnInfo() = classify(Ty: FI.getReturnType(), FreeSSERegs, IsReturnType: true,
3456	IsVectorCall, IsRegCall);
3457
3458	if (IsVectorCall) {
3459	// We can use up to 6 SSE register parameters with vectorcall.
3460	FreeSSERegs = 6;
3461	} else if (IsRegCall) {
3462	// RegCall gives us 16 SSE registers, we can reuse the return registers.
3463	FreeSSERegs = 16;
3464	}
3465
3466	unsigned ArgNum = 0;
3467	unsigned ZeroSSERegs = 0;
3468	for (auto &I : FI.arguments()) {
3469	// Vectorcall in x64 only permits the first 6 arguments to be passed as
3470	// XMM/YMM registers. After the sixth argument, pretend no vector
3471	// registers are left.
3472	unsigned *MaybeFreeSSERegs =
3473	(IsVectorCall && ArgNum >= 6) ? &ZeroSSERegs : &FreeSSERegs;
3474	I.info =
3475	classify(Ty: I.type, FreeSSERegs&: *MaybeFreeSSERegs, IsReturnType: false, IsVectorCall, IsRegCall);
3476	++ArgNum;
3477	}
3478
3479	if (IsVectorCall) {
3480	// For vectorcall, assign aggregate HVAs to any free vector registers in a
3481	// second pass.
3482	for (auto &I : FI.arguments())
3483	I.info = reclassifyHvaArgForVectorCall(Ty: I.type, FreeSSERegs, current: I.info);
3484	}
3485	}
3486
3487	RValue WinX86_64ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
3488	QualType Ty, AggValueSlot Slot) const {
3489	// MS x64 ABI requirement: "Any argument that doesn't fit in 8 bytes, or is
3490	// not 1, 2, 4, or 8 bytes, must be passed by reference."
3491	uint64_t Width = getContext().getTypeSize(T: Ty);
3492	bool IsIndirect = Width > 64 || !llvm::isPowerOf2_64(Value: Width);
3493
3494	return emitVoidPtrVAArg(CGF, VAListAddr, ValueTy: Ty, IsIndirect,
3495	ValueInfo: CGF.getContext().getTypeInfoInChars(T: Ty),
3496	SlotSizeAndAlign: CharUnits::fromQuantity(Quantity: 8),
3497	/*allowHigherAlign*/ AllowHigherAlign: false, Slot);
3498	}
3499
3500	std::unique_ptr<TargetCodeGenInfo> CodeGen::createX86_32TargetCodeGenInfo(
3501	CodeGenModule &CGM, bool DarwinVectorABI, bool Win32StructABI,
3502	unsigned NumRegisterParameters, bool SoftFloatABI) {
3503	bool RetSmallStructInRegABI = X86_32TargetCodeGenInfo::isStructReturnInRegABI(
3504	Triple: CGM.getTriple(), Opts: CGM.getCodeGenOpts());
3505	return std::make_unique<X86_32TargetCodeGenInfo>(
3506	args&: CGM.getTypes(), args&: DarwinVectorABI, args&: RetSmallStructInRegABI, args&: Win32StructABI,
3507	args&: NumRegisterParameters, args&: SoftFloatABI);
3508	}
3509
3510	std::unique_ptr<TargetCodeGenInfo> CodeGen::createWinX86_32TargetCodeGenInfo(
3511	CodeGenModule &CGM, bool DarwinVectorABI, bool Win32StructABI,
3512	unsigned NumRegisterParameters) {
3513	bool RetSmallStructInRegABI = X86_32TargetCodeGenInfo::isStructReturnInRegABI(
3514	Triple: CGM.getTriple(), Opts: CGM.getCodeGenOpts());
3515	return std::make_unique<WinX86_32TargetCodeGenInfo>(
3516	args&: CGM.getTypes(), args&: DarwinVectorABI, args&: RetSmallStructInRegABI, args&: Win32StructABI,
3517	args&: NumRegisterParameters);
3518	}
3519
3520	std::unique_ptr<TargetCodeGenInfo>
3521	CodeGen::createX86_64TargetCodeGenInfo(CodeGenModule &CGM,
3522	X86AVXABILevel AVXLevel) {
3523	return std::make_unique<X86_64TargetCodeGenInfo>(args&: CGM.getTypes(), args&: AVXLevel);
3524	}
3525
3526	std::unique_ptr<TargetCodeGenInfo>
3527	CodeGen::createWinX86_64TargetCodeGenInfo(CodeGenModule &CGM,
3528	X86AVXABILevel AVXLevel) {
3529	return std::make_unique<WinX86_64TargetCodeGenInfo>(args&: CGM.getTypes(), args&: AVXLevel);
3530	}
3531