1	//===- X86.cpp ------------------------------------------------------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8
9	#include "ABIInfoImpl.h"
10	#include "TargetInfo.h"
11	#include "clang/Basic/DiagnosticFrontend.h"
12	#include "llvm/ADT/SmallBitVector.h"
13
14	using namespace clang;
15	using namespace clang::CodeGen;
16
17	namespace {
18
19	/// IsX86_MMXType - Return true if this is an MMX type.
20	bool IsX86_MMXType(llvm::Type *IRType) {
21	// Return true if the type is an MMX type <2 x i32>, <4 x i16>, or <8 x i8>.
22	return IRType->isVectorTy() && IRType->getPrimitiveSizeInBits() == 64 &&
23	cast<llvm::VectorType>(Val: IRType)->getElementType()->isIntegerTy() &&
24	IRType->getScalarSizeInBits() != 64;
25	}
26
27	static llvm::Type* X86AdjustInlineAsmType(CodeGen::CodeGenFunction &CGF,
28	StringRef Constraint,
29	llvm::Type* Ty) {
30	bool IsMMXCons = llvm::StringSwitch<bool>(Constraint)
31	.Cases(S0: "y", S1: "&y", S2: "^Ym", Value: true)
32	.Default(Value: false);
33	if (IsMMXCons && Ty->isVectorTy()) {
34	if (cast<llvm::VectorType>(Val: Ty)->getPrimitiveSizeInBits().getFixedValue() !=
35	64) {
36	// Invalid MMX constraint
37	return nullptr;
38	}
39
40	return llvm::Type::getX86_MMXTy(C&: CGF.getLLVMContext());
41	}
42
43	if (Constraint == "k") {
44	llvm::Type *Int1Ty = llvm::Type::getInt1Ty(C&: CGF.getLLVMContext());
45	return llvm::FixedVectorType::get(ElementType: Int1Ty, NumElts: Ty->getScalarSizeInBits());
46	}
47
48	// No operation needed
49	return Ty;
50	}
51
52	/// Returns true if this type can be passed in SSE registers with the
53	/// X86_VectorCall calling convention. Shared between x86_32 and x86_64.
54	static bool isX86VectorTypeForVectorCall(ASTContext &Context, QualType Ty) {
55	if (const BuiltinType *BT = Ty->getAs<BuiltinType>()) {
56	if (BT->isFloatingPoint() && BT->getKind() != BuiltinType::Half) {
57	if (BT->getKind() == BuiltinType::LongDouble) {
58	if (&Context.getTargetInfo().getLongDoubleFormat() ==
59	&llvm::APFloat::x87DoubleExtended())
60	return false;
61	}
62	return true;
63	}
64	} else if (const VectorType *VT = Ty->getAs<VectorType>()) {
65	// vectorcall can pass XMM, YMM, and ZMM vectors. We don't pass SSE1 MMX
66	// registers specially.
67	unsigned VecSize = Context.getTypeSize(VT);
68	if (VecSize == 128 || VecSize == 256 || VecSize == 512)
69	return true;
70	}
71	return false;
72	}
73
74	/// Returns true if this aggregate is small enough to be passed in SSE registers
75	/// in the X86_VectorCall calling convention. Shared between x86_32 and x86_64.
76	static bool isX86VectorCallAggregateSmallEnough(uint64_t NumMembers) {
77	return NumMembers <= 4;
78	}
79
80	/// Returns a Homogeneous Vector Aggregate ABIArgInfo, used in X86.
81	static ABIArgInfo getDirectX86Hva(llvm::Type* T = nullptr) {
82	auto AI = ABIArgInfo::getDirect(T);
83	AI.setInReg(true);
84	AI.setCanBeFlattened(false);
85	return AI;
86	}
87
88	//===----------------------------------------------------------------------===//
89	// X86-32 ABI Implementation
90	//===----------------------------------------------------------------------===//
91
92	/// Similar to llvm::CCState, but for Clang.
93	struct CCState {
94	CCState(CGFunctionInfo &FI)
95	: IsPreassigned(FI.arg_size()), CC(FI.getCallingConvention()),
96	Required(FI.getRequiredArgs()), IsDelegateCall(FI.isDelegateCall()) {}
97
98	llvm::SmallBitVector IsPreassigned;
99	unsigned CC = CallingConv::CC_C;
100	unsigned FreeRegs = 0;
101	unsigned FreeSSERegs = 0;
102	RequiredArgs Required;
103	bool IsDelegateCall = false;
104	};
105
106	/// X86_32ABIInfo - The X86-32 ABI information.
107	class X86_32ABIInfo : public ABIInfo {
108	enum Class {
109	Integer,
110	Float
111	};
112
113	static const unsigned MinABIStackAlignInBytes = 4;
114
115	bool IsDarwinVectorABI;
116	bool IsRetSmallStructInRegABI;
117	bool IsWin32StructABI;
118	bool IsSoftFloatABI;
119	bool IsMCUABI;
120	bool IsLinuxABI;
121	unsigned DefaultNumRegisterParameters;
122
123	static bool isRegisterSize(unsigned Size) {
124	return (Size == 8 || Size == 16 || Size == 32 || Size == 64);
125	}
126
127	bool isHomogeneousAggregateBaseType(QualType Ty) const override {
128	// FIXME: Assumes vectorcall is in use.
129	return isX86VectorTypeForVectorCall(Context&: getContext(), Ty);
130	}
131
132	bool isHomogeneousAggregateSmallEnough(const Type *Ty,
133	uint64_t NumMembers) const override {
134	// FIXME: Assumes vectorcall is in use.
135	return isX86VectorCallAggregateSmallEnough(NumMembers);
136	}
137
138	bool shouldReturnTypeInRegister(QualType Ty, ASTContext &Context) const;
139
140	/// getIndirectResult - Give a source type \arg Ty, return a suitable result
141	/// such that the argument will be passed in memory.
142	ABIArgInfo getIndirectResult(QualType Ty, bool ByVal, CCState &State) const;
143
144	ABIArgInfo getIndirectReturnResult(QualType Ty, CCState &State) const;
145
146	/// Return the alignment to use for the given type on the stack.
147	unsigned getTypeStackAlignInBytes(QualType Ty, unsigned Align) const;
148
149	Class classify(QualType Ty) const;
150	ABIArgInfo classifyReturnType(QualType RetTy, CCState &State) const;
151	ABIArgInfo classifyArgumentType(QualType RetTy, CCState &State,
152	unsigned ArgIndex) const;
153
154	/// Updates the number of available free registers, returns
155	/// true if any registers were allocated.
156	bool updateFreeRegs(QualType Ty, CCState &State) const;
157
158	bool shouldAggregateUseDirect(QualType Ty, CCState &State, bool &InReg,
159	bool &NeedsPadding) const;
160	bool shouldPrimitiveUseInReg(QualType Ty, CCState &State) const;
161
162	bool canExpandIndirectArgument(QualType Ty) const;
163
164	/// Rewrite the function info so that all memory arguments use
165	/// inalloca.
166	void rewriteWithInAlloca(CGFunctionInfo &FI) const;
167
168	void addFieldToArgStruct(SmallVector<llvm::Type *, 6> &FrameFields,
169	CharUnits &StackOffset, ABIArgInfo &Info,
170	QualType Type) const;
171	void runVectorCallFirstPass(CGFunctionInfo &FI, CCState &State) const;
172
173	public:
174
175	void computeInfo(CGFunctionInfo &FI) const override;
176	Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
177	QualType Ty) const override;
178
179	X86_32ABIInfo(CodeGen::CodeGenTypes &CGT, bool DarwinVectorABI,
180	bool RetSmallStructInRegABI, bool Win32StructABI,
181	unsigned NumRegisterParameters, bool SoftFloatABI)
182	: ABIInfo(CGT), IsDarwinVectorABI(DarwinVectorABI),
183	IsRetSmallStructInRegABI(RetSmallStructInRegABI),
184	IsWin32StructABI(Win32StructABI), IsSoftFloatABI(SoftFloatABI),
185	IsMCUABI(CGT.getTarget().getTriple().isOSIAMCU()),
186	IsLinuxABI(CGT.getTarget().getTriple().isOSLinux() ||
187	CGT.getTarget().getTriple().isOSCygMing()),
188	DefaultNumRegisterParameters(NumRegisterParameters) {}
189	};
190
191	class X86_32SwiftABIInfo : public SwiftABIInfo {
192	public:
193	explicit X86_32SwiftABIInfo(CodeGenTypes &CGT)
194	: SwiftABIInfo(CGT, /*SwiftErrorInRegister=*/false) {}
195
196	bool shouldPassIndirectly(ArrayRef<llvm::Type *> ComponentTys,
197	bool AsReturnValue) const override {
198	// LLVM's x86-32 lowering currently only assigns up to three
199	// integer registers and three fp registers. Oddly, it'll use up to
200	// four vector registers for vectors, but those can overlap with the
201	// scalar registers.
202	return occupiesMoreThan(scalarTypes: ComponentTys, /*total=*/maxAllRegisters: 3);
203	}
204	};
205
206	class X86_32TargetCodeGenInfo : public TargetCodeGenInfo {
207	public:
208	X86_32TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT, bool DarwinVectorABI,
209	bool RetSmallStructInRegABI, bool Win32StructABI,
210	unsigned NumRegisterParameters, bool SoftFloatABI)
211	: TargetCodeGenInfo(std::make_unique<X86_32ABIInfo>(
212	args&: CGT, args&: DarwinVectorABI, args&: RetSmallStructInRegABI, args&: Win32StructABI,
213	args&: NumRegisterParameters, args&: SoftFloatABI)) {
214	SwiftInfo = std::make_unique<X86_32SwiftABIInfo>(args&: CGT);
215	}
216
217	static bool isStructReturnInRegABI(
218	const llvm::Triple &Triple, const CodeGenOptions &Opts);
219
220	void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
221	CodeGen::CodeGenModule &CGM) const override;
222
223	int getDwarfEHStackPointer(CodeGen::CodeGenModule &CGM) const override {
224	// Darwin uses different dwarf register numbers for EH.
225	if (CGM.getTarget().getTriple().isOSDarwin()) return 5;
226	return 4;
227	}
228
229	bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
230	llvm::Value *Address) const override;
231
232	llvm::Type* adjustInlineAsmType(CodeGen::CodeGenFunction &CGF,
233	StringRef Constraint,
234	llvm::Type* Ty) const override {
235	return X86AdjustInlineAsmType(CGF, Constraint, Ty);
236	}
237
238	void addReturnRegisterOutputs(CodeGenFunction &CGF, LValue ReturnValue,
239	std::string &Constraints,
240	std::vector<llvm::Type *> &ResultRegTypes,
241	std::vector<llvm::Type *> &ResultTruncRegTypes,
242	std::vector<LValue> &ResultRegDests,
243	std::string &AsmString,
244	unsigned NumOutputs) const override;
245
246	StringRef getARCRetainAutoreleasedReturnValueMarker() const override {
247	return "movl\t%ebp, %ebp"
248	"\t\t// marker for objc_retainAutoreleaseReturnValue";
249	}
250	};
251
252	}
253
254	/// Rewrite input constraint references after adding some output constraints.
255	/// In the case where there is one output and one input and we add one output,
256	/// we need to replace all operand references greater than or equal to 1:
257	/// mov $0, $1
258	/// mov eax, $1
259	/// The result will be:
260	/// mov $0, $2
261	/// mov eax, $2
262	static void rewriteInputConstraintReferences(unsigned FirstIn,
263	unsigned NumNewOuts,
264	std::string &AsmString) {
265	std::string Buf;
266	llvm::raw_string_ostream OS(Buf);
267	size_t Pos = 0;
268	while (Pos < AsmString.size()) {
269	size_t DollarStart = AsmString.find(c: '$', pos: Pos);
270	if (DollarStart == std::string::npos)
271	DollarStart = AsmString.size();
272	size_t DollarEnd = AsmString.find_first_not_of(c: '$', pos: DollarStart);
273	if (DollarEnd == std::string::npos)
274	DollarEnd = AsmString.size();
275	OS << StringRef(&AsmString[Pos], DollarEnd - Pos);
276	Pos = DollarEnd;
277	size_t NumDollars = DollarEnd - DollarStart;
278	if (NumDollars % 2 != 0 && Pos < AsmString.size()) {
279	// We have an operand reference.
280	size_t DigitStart = Pos;
281	if (AsmString[DigitStart] == '{') {
282	OS << '{';
283	++DigitStart;
284	}
285	size_t DigitEnd = AsmString.find_first_not_of(s: "0123456789", pos: DigitStart);
286	if (DigitEnd == std::string::npos)
287	DigitEnd = AsmString.size();
288	StringRef OperandStr(&AsmString[DigitStart], DigitEnd - DigitStart);
289	unsigned OperandIndex;
290	if (!OperandStr.getAsInteger(Radix: 10, Result&: OperandIndex)) {
291	if (OperandIndex >= FirstIn)
292	OperandIndex += NumNewOuts;
293	OS << OperandIndex;
294	} else {
295	OS << OperandStr;
296	}
297	Pos = DigitEnd;
298	}
299	}
300	AsmString = std::move(OS.str());
301	}
302
303	/// Add output constraints for EAX:EDX because they are return registers.
304	void X86_32TargetCodeGenInfo::addReturnRegisterOutputs(
305	CodeGenFunction &CGF, LValue ReturnSlot, std::string &Constraints,
306	std::vector<llvm::Type *> &ResultRegTypes,
307	std::vector<llvm::Type *> &ResultTruncRegTypes,
308	std::vector<LValue> &ResultRegDests, std::string &AsmString,
309	unsigned NumOutputs) const {
310	uint64_t RetWidth = CGF.getContext().getTypeSize(T: ReturnSlot.getType());
311
312	// Use the EAX constraint if the width is 32 or smaller and EAX:EDX if it is
313	// larger.
314	if (!Constraints.empty())
315	Constraints += ',';
316	if (RetWidth <= 32) {
317	Constraints += "={eax}";
318	ResultRegTypes.push_back(x: CGF.Int32Ty);
319	} else {
320	// Use the 'A' constraint for EAX:EDX.
321	Constraints += "=A";
322	ResultRegTypes.push_back(x: CGF.Int64Ty);
323	}
324
325	// Truncate EAX or EAX:EDX to an integer of the appropriate size.
326	llvm::Type *CoerceTy = llvm::IntegerType::get(C&: CGF.getLLVMContext(), NumBits: RetWidth);
327	ResultTruncRegTypes.push_back(x: CoerceTy);
328
329	// Coerce the integer by bitcasting the return slot pointer.
330	ReturnSlot.setAddress(ReturnSlot.getAddress(CGF).withElementType(ElemTy: CoerceTy));
331	ResultRegDests.push_back(x: ReturnSlot);
332
333	rewriteInputConstraintReferences(FirstIn: NumOutputs, NumNewOuts: 1, AsmString);
334	}
335
336	/// shouldReturnTypeInRegister - Determine if the given type should be
337	/// returned in a register (for the Darwin and MCU ABI).
338	bool X86_32ABIInfo::shouldReturnTypeInRegister(QualType Ty,
339	ASTContext &Context) const {
340	uint64_t Size = Context.getTypeSize(T: Ty);
341
342	// For i386, type must be register sized.
343	// For the MCU ABI, it only needs to be <= 8-byte
344	if ((IsMCUABI && Size > 64) || (!IsMCUABI && !isRegisterSize(Size)))
345	return false;
346
347	if (Ty->isVectorType()) {
348	// 64- and 128- bit vectors inside structures are not returned in
349	// registers.
350	if (Size == 64 || Size == 128)
351	return false;
352
353	return true;
354	}
355
356	// If this is a builtin, pointer, enum, complex type, member pointer, or
357	// member function pointer it is ok.
358	if (Ty->getAs<BuiltinType>() || Ty->hasPointerRepresentation() ||
359	Ty->isAnyComplexType() || Ty->isEnumeralType() ||
360	Ty->isBlockPointerType() || Ty->isMemberPointerType())
361	return true;
362
363	// Arrays are treated like records.
364	if (const ConstantArrayType *AT = Context.getAsConstantArrayType(T: Ty))
365	return shouldReturnTypeInRegister(Ty: AT->getElementType(), Context);
366
367	// Otherwise, it must be a record type.
368	const RecordType *RT = Ty->getAs<RecordType>();
369	if (!RT) return false;
370
371	// FIXME: Traverse bases here too.
372
373	// Structure types are passed in register if all fields would be
374	// passed in a register.
375	for (const auto *FD : RT->getDecl()->fields()) {
376	// Empty fields are ignored.
377	if (isEmptyField(Context, FD, AllowArrays: true))
378	continue;
379
380	// Check fields recursively.
381	if (!shouldReturnTypeInRegister(Ty: FD->getType(), Context))
382	return false;
383	}
384	return true;
385	}
386
387	static bool is32Or64BitBasicType(QualType Ty, ASTContext &Context) {
388	// Treat complex types as the element type.
389	if (const ComplexType *CTy = Ty->getAs<ComplexType>())
390	Ty = CTy->getElementType();
391
392	// Check for a type which we know has a simple scalar argument-passing
393	// convention without any padding. (We're specifically looking for 32
394	// and 64-bit integer and integer-equivalents, float, and double.)
395	if (!Ty->getAs<BuiltinType>() && !Ty->hasPointerRepresentation() &&
396	!Ty->isEnumeralType() && !Ty->isBlockPointerType())
397	return false;
398
399	uint64_t Size = Context.getTypeSize(T: Ty);
400	return Size == 32 || Size == 64;
401	}
402
403	static bool addFieldSizes(ASTContext &Context, const RecordDecl *RD,
404	uint64_t &Size) {
405	for (const auto *FD : RD->fields()) {
406	// Scalar arguments on the stack get 4 byte alignment on x86. If the
407	// argument is smaller than 32-bits, expanding the struct will create
408	// alignment padding.
409	if (!is32Or64BitBasicType(FD->getType(), Context))
410	return false;
411
412	// FIXME: Reject bit-fields wholesale; there are two problems, we don't know
413	// how to expand them yet, and the predicate for telling if a bitfield still
414	// counts as "basic" is more complicated than what we were doing previously.
415	if (FD->isBitField())
416	return false;
417
418	Size += Context.getTypeSize(FD->getType());
419	}
420	return true;
421	}
422
423	static bool addBaseAndFieldSizes(ASTContext &Context, const CXXRecordDecl *RD,
424	uint64_t &Size) {
425	// Don't do this if there are any non-empty bases.
426	for (const CXXBaseSpecifier &Base : RD->bases()) {
427	if (!addBaseAndFieldSizes(Context, RD: Base.getType()->getAsCXXRecordDecl(),
428	Size))
429	return false;
430	}
431	if (!addFieldSizes(Context, RD, Size))
432	return false;
433	return true;
434	}
435
436	/// Test whether an argument type which is to be passed indirectly (on the
437	/// stack) would have the equivalent layout if it was expanded into separate
438	/// arguments. If so, we prefer to do the latter to avoid inhibiting
439	/// optimizations.
440	bool X86_32ABIInfo::canExpandIndirectArgument(QualType Ty) const {
441	// We can only expand structure types.
442	const RecordType *RT = Ty->getAs<RecordType>();
443	if (!RT)
444	return false;
445	const RecordDecl *RD = RT->getDecl();
446	uint64_t Size = 0;
447	if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(Val: RD)) {
448	if (!IsWin32StructABI) {
449	// On non-Windows, we have to conservatively match our old bitcode
450	// prototypes in order to be ABI-compatible at the bitcode level.
451	if (!CXXRD->isCLike())
452	return false;
453	} else {
454	// Don't do this for dynamic classes.
455	if (CXXRD->isDynamicClass())
456	return false;
457	}
458	if (!addBaseAndFieldSizes(Context&: getContext(), RD: CXXRD, Size))
459	return false;
460	} else {
461	if (!addFieldSizes(Context&: getContext(), RD, Size))
462	return false;
463	}
464
465	// We can do this if there was no alignment padding.
466	return Size == getContext().getTypeSize(T: Ty);
467	}
468
469	ABIArgInfo X86_32ABIInfo::getIndirectReturnResult(QualType RetTy, CCState &State) const {
470	// If the return value is indirect, then the hidden argument is consuming one
471	// integer register.
472	if (State.FreeRegs) {
473	--State.FreeRegs;
474	if (!IsMCUABI)
475	return getNaturalAlignIndirectInReg(Ty: RetTy);
476	}
477	return getNaturalAlignIndirect(Ty: RetTy, /*ByVal=*/false);
478	}
479
480	ABIArgInfo X86_32ABIInfo::classifyReturnType(QualType RetTy,
481	CCState &State) const {
482	if (RetTy->isVoidType())
483	return ABIArgInfo::getIgnore();
484
485	const Type *Base = nullptr;
486	uint64_t NumElts = 0;
487	if ((State.CC == llvm::CallingConv::X86_VectorCall ||
488	State.CC == llvm::CallingConv::X86_RegCall) &&
489	isHomogeneousAggregate(Ty: RetTy, Base, Members&: NumElts)) {
490	// The LLVM struct type for such an aggregate should lower properly.
491	return ABIArgInfo::getDirect();
492	}
493
494	if (const VectorType *VT = RetTy->getAs<VectorType>()) {
495	// On Darwin, some vectors are returned in registers.
496	if (IsDarwinVectorABI) {
497	uint64_t Size = getContext().getTypeSize(T: RetTy);
498
499	// 128-bit vectors are a special case; they are returned in
500	// registers and we need to make sure to pick a type the LLVM
501	// backend will like.
502	if (Size == 128)
503	return ABIArgInfo::getDirect(T: llvm::FixedVectorType::get(
504	ElementType: llvm::Type::getInt64Ty(C&: getVMContext()), NumElts: 2));
505
506	// Always return in register if it fits in a general purpose
507	// register, or if it is 64 bits and has a single element.
508	if ((Size == 8 || Size == 16 || Size == 32) ||
509	(Size == 64 && VT->getNumElements() == 1))
510	return ABIArgInfo::getDirect(T: llvm::IntegerType::get(C&: getVMContext(),
511	NumBits: Size));
512
513	return getIndirectReturnResult(RetTy, State);
514	}
515
516	return ABIArgInfo::getDirect();
517	}
518
519	if (isAggregateTypeForABI(T: RetTy)) {
520	if (const RecordType *RT = RetTy->getAs<RecordType>()) {
521	// Structures with flexible arrays are always indirect.
522	if (RT->getDecl()->hasFlexibleArrayMember())
523	return getIndirectReturnResult(RetTy, State);
524	}
525
526	// If specified, structs and unions are always indirect.
527	if (!IsRetSmallStructInRegABI && !RetTy->isAnyComplexType())
528	return getIndirectReturnResult(RetTy, State);
529
530	// Ignore empty structs/unions.
531	if (isEmptyRecord(Context&: getContext(), T: RetTy, AllowArrays: true))
532	return ABIArgInfo::getIgnore();
533
534	// Return complex of _Float16 as <2 x half> so the backend will use xmm0.
535	if (const ComplexType *CT = RetTy->getAs<ComplexType>()) {
536	QualType ET = getContext().getCanonicalType(T: CT->getElementType());
537	if (ET->isFloat16Type())
538	return ABIArgInfo::getDirect(T: llvm::FixedVectorType::get(
539	ElementType: llvm::Type::getHalfTy(C&: getVMContext()), NumElts: 2));
540	}
541
542	// Small structures which are register sized are generally returned
543	// in a register.
544	if (shouldReturnTypeInRegister(Ty: RetTy, Context&: getContext())) {
545	uint64_t Size = getContext().getTypeSize(T: RetTy);
546
547	// As a special-case, if the struct is a "single-element" struct, and
548	// the field is of type "float" or "double", return it in a
549	// floating-point register. (MSVC does not apply this special case.)
550	// We apply a similar transformation for pointer types to improve the
551	// quality of the generated IR.
552	if (const Type *SeltTy = isSingleElementStruct(T: RetTy, Context&: getContext()))
553	if ((!IsWin32StructABI && SeltTy->isRealFloatingType())
554	|| SeltTy->hasPointerRepresentation())
555	return ABIArgInfo::getDirect(T: CGT.ConvertType(T: QualType(SeltTy, 0)));
556
557	// FIXME: We should be able to narrow this integer in cases with dead
558	// padding.
559	return ABIArgInfo::getDirect(T: llvm::IntegerType::get(C&: getVMContext(),NumBits: Size));
560	}
561
562	return getIndirectReturnResult(RetTy, State);
563	}
564
565	// Treat an enum type as its underlying type.
566	if (const EnumType *EnumTy = RetTy->getAs<EnumType>())
567	RetTy = EnumTy->getDecl()->getIntegerType();
568
569	if (const auto *EIT = RetTy->getAs<BitIntType>())
570	if (EIT->getNumBits() > 64)
571	return getIndirectReturnResult(RetTy, State);
572
573	return (isPromotableIntegerTypeForABI(Ty: RetTy) ? ABIArgInfo::getExtend(Ty: RetTy)
574	: ABIArgInfo::getDirect());
575	}
576
577	unsigned X86_32ABIInfo::getTypeStackAlignInBytes(QualType Ty,
578	unsigned Align) const {
579	// Otherwise, if the alignment is less than or equal to the minimum ABI
580	// alignment, just use the default; the backend will handle this.
581	if (Align <= MinABIStackAlignInBytes)
582	return 0; // Use default alignment.
583
584	if (IsLinuxABI) {
585	// Exclude other System V OS (e.g Darwin, PS4 and FreeBSD) since we don't
586	// want to spend any effort dealing with the ramifications of ABI breaks.
587	//
588	// If the vector type is __m128/__m256/__m512, return the default alignment.
589	if (Ty->isVectorType() && (Align == 16 || Align == 32 || Align == 64))
590	return Align;
591	}
592	// On non-Darwin, the stack type alignment is always 4.
593	if (!IsDarwinVectorABI) {
594	// Set explicit alignment, since we may need to realign the top.
595	return MinABIStackAlignInBytes;
596	}
597
598	// Otherwise, if the type contains an SSE vector type, the alignment is 16.
599	if (Align >= 16 && (isSIMDVectorType(Context&: getContext(), Ty) ||
600	isRecordWithSIMDVectorType(Context&: getContext(), Ty)))
601	return 16;
602
603	return MinABIStackAlignInBytes;
604	}
605
606	ABIArgInfo X86_32ABIInfo::getIndirectResult(QualType Ty, bool ByVal,
607	CCState &State) const {
608	if (!ByVal) {
609	if (State.FreeRegs) {
610	--State.FreeRegs; // Non-byval indirects just use one pointer.
611	if (!IsMCUABI)
612	return getNaturalAlignIndirectInReg(Ty);
613	}
614	return getNaturalAlignIndirect(Ty, ByVal: false);
615	}
616
617	// Compute the byval alignment.
618	unsigned TypeAlign = getContext().getTypeAlign(T: Ty) / 8;
619	unsigned StackAlign = getTypeStackAlignInBytes(Ty, Align: TypeAlign);
620	if (StackAlign == 0)
621	return ABIArgInfo::getIndirect(Alignment: CharUnits::fromQuantity(Quantity: 4), /*ByVal=*/true);
622
623	// If the stack alignment is less than the type alignment, realign the
624	// argument.
625	bool Realign = TypeAlign > StackAlign;
626	return ABIArgInfo::getIndirect(Alignment: CharUnits::fromQuantity(Quantity: StackAlign),
627	/*ByVal=*/true, Realign);
628	}
629
630	X86_32ABIInfo::Class X86_32ABIInfo::classify(QualType Ty) const {
631	const Type *T = isSingleElementStruct(T: Ty, Context&: getContext());
632	if (!T)
633	T = Ty.getTypePtr();
634
635	if (const BuiltinType *BT = T->getAs<BuiltinType>()) {
636	BuiltinType::Kind K = BT->getKind();
637	if (K == BuiltinType::Float || K == BuiltinType::Double)
638	return Float;
639	}
640	return Integer;
641	}
642
643	bool X86_32ABIInfo::updateFreeRegs(QualType Ty, CCState &State) const {
644	if (!IsSoftFloatABI) {
645	Class C = classify(Ty);
646	if (C == Float)
647	return false;
648	}
649
650	unsigned Size = getContext().getTypeSize(T: Ty);
651	unsigned SizeInRegs = (Size + 31) / 32;
652
653	if (SizeInRegs == 0)
654	return false;
655
656	if (!IsMCUABI) {
657	if (SizeInRegs > State.FreeRegs) {
658	State.FreeRegs = 0;
659	return false;
660	}
661	} else {
662	// The MCU psABI allows passing parameters in-reg even if there are
663	// earlier parameters that are passed on the stack. Also,
664	// it does not allow passing >8-byte structs in-register,
665	// even if there are 3 free registers available.
666	if (SizeInRegs > State.FreeRegs || SizeInRegs > 2)
667	return false;
668	}
669
670	State.FreeRegs -= SizeInRegs;
671	return true;
672	}
673
674	bool X86_32ABIInfo::shouldAggregateUseDirect(QualType Ty, CCState &State,
675	bool &InReg,
676	bool &NeedsPadding) const {
677	// On Windows, aggregates other than HFAs are never passed in registers, and
678	// they do not consume register slots. Homogenous floating-point aggregates
679	// (HFAs) have already been dealt with at this point.
680	if (IsWin32StructABI && isAggregateTypeForABI(T: Ty))
681	return false;
682
683	NeedsPadding = false;
684	InReg = !IsMCUABI;
685
686	if (!updateFreeRegs(Ty, State))
687	return false;
688
689	if (IsMCUABI)
690	return true;
691
692	if (State.CC == llvm::CallingConv::X86_FastCall ||
693	State.CC == llvm::CallingConv::X86_VectorCall ||
694	State.CC == llvm::CallingConv::X86_RegCall) {
695	if (getContext().getTypeSize(T: Ty) <= 32 && State.FreeRegs)
696	NeedsPadding = true;
697
698	return false;
699	}
700
701	return true;
702	}
703
704	bool X86_32ABIInfo::shouldPrimitiveUseInReg(QualType Ty, CCState &State) const {
705	bool IsPtrOrInt = (getContext().getTypeSize(T: Ty) <= 32) &&
706	(Ty->isIntegralOrEnumerationType() || Ty->isPointerType() ||
707	Ty->isReferenceType());
708
709	if (!IsPtrOrInt && (State.CC == llvm::CallingConv::X86_FastCall ||
710	State.CC == llvm::CallingConv::X86_VectorCall))
711	return false;
712
713	if (!updateFreeRegs(Ty, State))
714	return false;
715
716	if (!IsPtrOrInt && State.CC == llvm::CallingConv::X86_RegCall)
717	return false;
718
719	// Return true to apply inreg to all legal parameters except for MCU targets.
720	return !IsMCUABI;
721	}
722
723	void X86_32ABIInfo::runVectorCallFirstPass(CGFunctionInfo &FI, CCState &State) const {
724	// Vectorcall x86 works subtly different than in x64, so the format is
725	// a bit different than the x64 version. First, all vector types (not HVAs)
726	// are assigned, with the first 6 ending up in the [XYZ]MM0-5 registers.
727	// This differs from the x64 implementation, where the first 6 by INDEX get
728	// registers.
729	// In the second pass over the arguments, HVAs are passed in the remaining
730	// vector registers if possible, or indirectly by address. The address will be
731	// passed in ECX/EDX if available. Any other arguments are passed according to
732	// the usual fastcall rules.
733	MutableArrayRef<CGFunctionInfoArgInfo> Args = FI.arguments();
734	for (int I = 0, E = Args.size(); I < E; ++I) {
735	const Type *Base = nullptr;
736	uint64_t NumElts = 0;
737	const QualType &Ty = Args[I].type;
738	if ((Ty->isVectorType() || Ty->isBuiltinType()) &&
739	isHomogeneousAggregate(Ty, Base, Members&: NumElts)) {
740	if (State.FreeSSERegs >= NumElts) {
741	State.FreeSSERegs -= NumElts;
742	Args[I].info = ABIArgInfo::getDirectInReg();
743	State.IsPreassigned.set(I);
744	}
745	}
746	}
747	}
748
749	ABIArgInfo X86_32ABIInfo::classifyArgumentType(QualType Ty, CCState &State,
750	unsigned ArgIndex) const {
751	// FIXME: Set alignment on indirect arguments.
752	bool IsFastCall = State.CC == llvm::CallingConv::X86_FastCall;
753	bool IsRegCall = State.CC == llvm::CallingConv::X86_RegCall;
754	bool IsVectorCall = State.CC == llvm::CallingConv::X86_VectorCall;
755
756	Ty = useFirstFieldIfTransparentUnion(Ty);
757	TypeInfo TI = getContext().getTypeInfo(T: Ty);
758
759	// Check with the C++ ABI first.
760	const RecordType *RT = Ty->getAs<RecordType>();
761	if (RT) {
762	CGCXXABI::RecordArgABI RAA = getRecordArgABI(RT, CXXABI&: getCXXABI());
763	if (RAA == CGCXXABI::RAA_Indirect) {
764	return getIndirectResult(Ty, ByVal: false, State);
765	} else if (State.IsDelegateCall) {
766	// Avoid having different alignments on delegate call args by always
767	// setting the alignment to 4, which is what we do for inallocas.
768	ABIArgInfo Res = getIndirectResult(Ty, ByVal: false, State);
769	Res.setIndirectAlign(CharUnits::fromQuantity(Quantity: 4));
770	return Res;
771	} else if (RAA == CGCXXABI::RAA_DirectInMemory) {
772	// The field index doesn't matter, we'll fix it up later.
773	return ABIArgInfo::getInAlloca(/*FieldIndex=*/0);
774	}
775	}
776
777	// Regcall uses the concept of a homogenous vector aggregate, similar
778	// to other targets.
779	const Type *Base = nullptr;
780	uint64_t NumElts = 0;
781	if ((IsRegCall || IsVectorCall) &&
782	isHomogeneousAggregate(Ty, Base, Members&: NumElts)) {
783	if (State.FreeSSERegs >= NumElts) {
784	State.FreeSSERegs -= NumElts;
785
786	// Vectorcall passes HVAs directly and does not flatten them, but regcall
787	// does.
788	if (IsVectorCall)
789	return getDirectX86Hva();
790
791	if (Ty->isBuiltinType() || Ty->isVectorType())
792	return ABIArgInfo::getDirect();
793	return ABIArgInfo::getExpand();
794	}
795	return getIndirectResult(Ty, /*ByVal=*/false, State);
796	}
797
798	if (isAggregateTypeForABI(T: Ty)) {
799	// Structures with flexible arrays are always indirect.
800	// FIXME: This should not be byval!
801	if (RT && RT->getDecl()->hasFlexibleArrayMember())
802	return getIndirectResult(Ty, ByVal: true, State);
803
804	// Ignore empty structs/unions on non-Windows.
805	if (!IsWin32StructABI && isEmptyRecord(Context&: getContext(), T: Ty, AllowArrays: true))
806	return ABIArgInfo::getIgnore();
807
808	llvm::LLVMContext &LLVMContext = getVMContext();
809	llvm::IntegerType *Int32 = llvm::Type::getInt32Ty(C&: LLVMContext);
810	bool NeedsPadding = false;
811	bool InReg;
812	if (shouldAggregateUseDirect(Ty, State, InReg, NeedsPadding)) {
813	unsigned SizeInRegs = (TI.Width + 31) / 32;
814	SmallVector<llvm::Type*, 3> Elements(SizeInRegs, Int32);
815	llvm::Type *Result = llvm::StructType::get(Context&: LLVMContext, Elements);
816	if (InReg)
817	return ABIArgInfo::getDirectInReg(T: Result);
818	else
819	return ABIArgInfo::getDirect(T: Result);
820	}
821	llvm::IntegerType *PaddingType = NeedsPadding ? Int32 : nullptr;
822
823	// Pass over-aligned aggregates to non-variadic functions on Windows
824	// indirectly. This behavior was added in MSVC 2015. Use the required
825	// alignment from the record layout, since that may be less than the
826	// regular type alignment, and types with required alignment of less than 4
827	// bytes are not passed indirectly.
828	if (IsWin32StructABI && State.Required.isRequiredArg(argIdx: ArgIndex)) {
829	unsigned AlignInBits = 0;
830	if (RT) {
831	const ASTRecordLayout &Layout =
832	getContext().getASTRecordLayout(D: RT->getDecl());
833	AlignInBits = getContext().toBits(CharSize: Layout.getRequiredAlignment());
834	} else if (TI.isAlignRequired()) {
835	AlignInBits = TI.Align;
836	}
837	if (AlignInBits > 32)
838	return getIndirectResult(Ty, /*ByVal=*/false, State);
839	}
840
841	// Expand small (<= 128-bit) record types when we know that the stack layout
842	// of those arguments will match the struct. This is important because the
843	// LLVM backend isn't smart enough to remove byval, which inhibits many
844	// optimizations.
845	// Don't do this for the MCU if there are still free integer registers
846	// (see X86_64 ABI for full explanation).
847	if (TI.Width <= 4 * 32 && (!IsMCUABI || State.FreeRegs == 0) &&
848	canExpandIndirectArgument(Ty))
849	return ABIArgInfo::getExpandWithPadding(
850	PaddingInReg: IsFastCall || IsVectorCall || IsRegCall, Padding: PaddingType);
851
852	return getIndirectResult(Ty, ByVal: true, State);
853	}
854
855	if (const VectorType *VT = Ty->getAs<VectorType>()) {
856	// On Windows, vectors are passed directly if registers are available, or
857	// indirectly if not. This avoids the need to align argument memory. Pass
858	// user-defined vector types larger than 512 bits indirectly for simplicity.
859	if (IsWin32StructABI) {
860	if (TI.Width <= 512 && State.FreeSSERegs > 0) {
861	--State.FreeSSERegs;
862	return ABIArgInfo::getDirectInReg();
863	}
864	return getIndirectResult(Ty, /*ByVal=*/false, State);
865	}
866
867	// On Darwin, some vectors are passed in memory, we handle this by passing
868	// it as an i8/i16/i32/i64.
869	if (IsDarwinVectorABI) {
870	if ((TI.Width == 8 || TI.Width == 16 || TI.Width == 32) ||
871	(TI.Width == 64 && VT->getNumElements() == 1))
872	return ABIArgInfo::getDirect(
873	T: llvm::IntegerType::get(C&: getVMContext(), NumBits: TI.Width));
874	}
875
876	if (IsX86_MMXType(IRType: CGT.ConvertType(T: Ty)))
877	return ABIArgInfo::getDirect(T: llvm::IntegerType::get(C&: getVMContext(), NumBits: 64));
878
879	return ABIArgInfo::getDirect();
880	}
881
882
883	if (const EnumType *EnumTy = Ty->getAs<EnumType>())
884	Ty = EnumTy->getDecl()->getIntegerType();
885
886	bool InReg = shouldPrimitiveUseInReg(Ty, State);
887
888	if (isPromotableIntegerTypeForABI(Ty)) {
889	if (InReg)
890	return ABIArgInfo::getExtendInReg(Ty);
891	return ABIArgInfo::getExtend(Ty);
892	}
893
894	if (const auto *EIT = Ty->getAs<BitIntType>()) {
895	if (EIT->getNumBits() <= 64) {
896	if (InReg)
897	return ABIArgInfo::getDirectInReg();
898	return ABIArgInfo::getDirect();
899	}
900	return getIndirectResult(Ty, /*ByVal=*/false, State);
901	}
902
903	if (InReg)
904	return ABIArgInfo::getDirectInReg();
905	return ABIArgInfo::getDirect();
906	}
907
908	void X86_32ABIInfo::computeInfo(CGFunctionInfo &FI) const {
909	CCState State(FI);
910	if (IsMCUABI)
911	State.FreeRegs = 3;
912	else if (State.CC == llvm::CallingConv::X86_FastCall) {
913	State.FreeRegs = 2;
914	State.FreeSSERegs = 3;
915	} else if (State.CC == llvm::CallingConv::X86_VectorCall) {
916	State.FreeRegs = 2;
917	State.FreeSSERegs = 6;
918	} else if (FI.getHasRegParm())
919	State.FreeRegs = FI.getRegParm();
920	else if (State.CC == llvm::CallingConv::X86_RegCall) {
921	State.FreeRegs = 5;
922	State.FreeSSERegs = 8;
923	} else if (IsWin32StructABI) {
924	// Since MSVC 2015, the first three SSE vectors have been passed in
925	// registers. The rest are passed indirectly.
926	State.FreeRegs = DefaultNumRegisterParameters;
927	State.FreeSSERegs = 3;
928	} else
929	State.FreeRegs = DefaultNumRegisterParameters;
930
931	if (!::classifyReturnType(CXXABI: getCXXABI(), FI, Info: *this)) {
932	FI.getReturnInfo() = classifyReturnType(RetTy: FI.getReturnType(), State);
933	} else if (FI.getReturnInfo().isIndirect()) {
934	// The C++ ABI is not aware of register usage, so we have to check if the
935	// return value was sret and put it in a register ourselves if appropriate.
936	if (State.FreeRegs) {
937	--State.FreeRegs; // The sret parameter consumes a register.
938	if (!IsMCUABI)
939	FI.getReturnInfo().setInReg(true);
940	}
941	}
942
943	// The chain argument effectively gives us another free register.
944	if (FI.isChainCall())
945	++State.FreeRegs;
946
947	// For vectorcall, do a first pass over the arguments, assigning FP and vector
948	// arguments to XMM registers as available.
949	if (State.CC == llvm::CallingConv::X86_VectorCall)
950	runVectorCallFirstPass(FI, State);
951
952	bool UsedInAlloca = false;
953	MutableArrayRef<CGFunctionInfoArgInfo> Args = FI.arguments();
954	for (unsigned I = 0, E = Args.size(); I < E; ++I) {
955	// Skip arguments that have already been assigned.
956	if (State.IsPreassigned.test(Idx: I))
957	continue;
958
959	Args[I].info =
960	classifyArgumentType(Ty: Args[I].type, State, ArgIndex: I);
961	UsedInAlloca |= (Args[I].info.getKind() == ABIArgInfo::InAlloca);
962	}
963
964	// If we needed to use inalloca for any argument, do a second pass and rewrite
965	// all the memory arguments to use inalloca.
966	if (UsedInAlloca)
967	rewriteWithInAlloca(FI);
968	}
969
970	void
971	X86_32ABIInfo::addFieldToArgStruct(SmallVector<llvm::Type *, 6> &FrameFields,
972	CharUnits &StackOffset, ABIArgInfo &Info,
973	QualType Type) const {
974	// Arguments are always 4-byte-aligned.
975	CharUnits WordSize = CharUnits::fromQuantity(Quantity: 4);
976	assert(StackOffset.isMultipleOf(WordSize) && "unaligned inalloca struct");
977
978	// sret pointers and indirect things will require an extra pointer
979	// indirection, unless they are byval. Most things are byval, and will not
980	// require this indirection.
981	bool IsIndirect = false;
982	if (Info.isIndirect() && !Info.getIndirectByVal())
983	IsIndirect = true;
984	Info = ABIArgInfo::getInAlloca(FieldIndex: FrameFields.size(), Indirect: IsIndirect);
985	llvm::Type *LLTy = CGT.ConvertTypeForMem(T: Type);
986	if (IsIndirect)
987	LLTy = llvm::PointerType::getUnqual(C&: getVMContext());
988	FrameFields.push_back(Elt: LLTy);
989	StackOffset += IsIndirect ? WordSize : getContext().getTypeSizeInChars(T: Type);
990
991	// Insert padding bytes to respect alignment.
992	CharUnits FieldEnd = StackOffset;
993	StackOffset = FieldEnd.alignTo(Align: WordSize);
994	if (StackOffset != FieldEnd) {
995	CharUnits NumBytes = StackOffset - FieldEnd;
996	llvm::Type *Ty = llvm::Type::getInt8Ty(C&: getVMContext());
997	Ty = llvm::ArrayType::get(ElementType: Ty, NumElements: NumBytes.getQuantity());
998	FrameFields.push_back(Elt: Ty);
999	}
1000	}
1001
1002	static bool isArgInAlloca(const ABIArgInfo &Info) {
1003	// Leave ignored and inreg arguments alone.
1004	switch (Info.getKind()) {
1005	case ABIArgInfo::InAlloca:
1006	return true;
1007	case ABIArgInfo::Ignore:
1008	case ABIArgInfo::IndirectAliased:
1009	return false;
1010	case ABIArgInfo::Indirect:
1011	case ABIArgInfo::Direct:
1012	case ABIArgInfo::Extend:
1013	return !Info.getInReg();
1014	case ABIArgInfo::Expand:
1015	case ABIArgInfo::CoerceAndExpand:
1016	// These are aggregate types which are never passed in registers when
1017	// inalloca is involved.
1018	return true;
1019	}
1020	llvm_unreachable("invalid enum");
1021	}
1022
1023	void X86_32ABIInfo::rewriteWithInAlloca(CGFunctionInfo &FI) const {
1024	assert(IsWin32StructABI && "inalloca only supported on win32");
1025
1026	// Build a packed struct type for all of the arguments in memory.
1027	SmallVector<llvm::Type *, 6> FrameFields;
1028
1029	// The stack alignment is always 4.
1030	CharUnits StackAlign = CharUnits::fromQuantity(Quantity: 4);
1031
1032	CharUnits StackOffset;
1033	CGFunctionInfo::arg_iterator I = FI.arg_begin(), E = FI.arg_end();
1034
1035	// Put 'this' into the struct before 'sret', if necessary.
1036	bool IsThisCall =
1037	FI.getCallingConvention() == llvm::CallingConv::X86_ThisCall;
1038	ABIArgInfo &Ret = FI.getReturnInfo();
1039	if (Ret.isIndirect() && Ret.isSRetAfterThis() && !IsThisCall &&
1040	isArgInAlloca(Info: I->info)) {
1041	addFieldToArgStruct(FrameFields, StackOffset, Info&: I->info, Type: I->type);
1042	++I;
1043	}
1044
1045	// Put the sret parameter into the inalloca struct if it's in memory.
1046	if (Ret.isIndirect() && !Ret.getInReg()) {
1047	addFieldToArgStruct(FrameFields, StackOffset, Info&: Ret, Type: FI.getReturnType());
1048	// On Windows, the hidden sret parameter is always returned in eax.
1049	Ret.setInAllocaSRet(IsWin32StructABI);
1050	}
1051
1052	// Skip the 'this' parameter in ecx.
1053	if (IsThisCall)
1054	++I;
1055
1056	// Put arguments passed in memory into the struct.
1057	for (; I != E; ++I) {
1058	if (isArgInAlloca(Info: I->info))
1059	addFieldToArgStruct(FrameFields, StackOffset, Info&: I->info, Type: I->type);
1060	}
1061
1062	FI.setArgStruct(Ty: llvm::StructType::get(Context&: getVMContext(), Elements: FrameFields,
1063	/*isPacked=*/true),
1064	Align: StackAlign);
1065	}
1066
1067	Address X86_32ABIInfo::EmitVAArg(CodeGenFunction &CGF,
1068	Address VAListAddr, QualType Ty) const {
1069
1070	auto TypeInfo = getContext().getTypeInfoInChars(T: Ty);
1071
1072	CCState State(*const_cast<CGFunctionInfo *>(CGF.CurFnInfo));
1073	ABIArgInfo AI = classifyArgumentType(Ty, State, /*ArgIndex*/ 0);
1074	// Empty records are ignored for parameter passing purposes.
1075	if (AI.isIgnore())
1076	return CGF.CreateMemTemp(T: Ty);
1077
1078	// x86-32 changes the alignment of certain arguments on the stack.
1079	//
1080	// Just messing with TypeInfo like this works because we never pass
1081	// anything indirectly.
1082	TypeInfo.Align = CharUnits::fromQuantity(
1083	Quantity: getTypeStackAlignInBytes(Ty, Align: TypeInfo.Align.getQuantity()));
1084
1085	return emitVoidPtrVAArg(CGF, VAListAddr, ValueTy: Ty, /*Indirect*/ IsIndirect: false,
1086	ValueInfo: TypeInfo, SlotSizeAndAlign: CharUnits::fromQuantity(Quantity: 4),
1087	/*AllowHigherAlign*/ true);
1088	}
1089
1090	bool X86_32TargetCodeGenInfo::isStructReturnInRegABI(
1091	const llvm::Triple &Triple, const CodeGenOptions &Opts) {
1092	assert(Triple.getArch() == llvm::Triple::x86);
1093
1094	switch (Opts.getStructReturnConvention()) {
1095	case CodeGenOptions::SRCK_Default:
1096	break;
1097	case CodeGenOptions::SRCK_OnStack: // -fpcc-struct-return
1098	return false;
1099	case CodeGenOptions::SRCK_InRegs: // -freg-struct-return
1100	return true;
1101	}
1102
1103	if (Triple.isOSDarwin() || Triple.isOSIAMCU())
1104	return true;
1105
1106	switch (Triple.getOS()) {
1107	case llvm::Triple::DragonFly:
1108	case llvm::Triple::FreeBSD:
1109	case llvm::Triple::OpenBSD:
1110	case llvm::Triple::Win32:
1111	return true;
1112	default:
1113	return false;
1114	}
1115	}
1116
1117	static void addX86InterruptAttrs(const FunctionDecl *FD, llvm::GlobalValue *GV,
1118	CodeGen::CodeGenModule &CGM) {
1119	if (!FD->hasAttr<AnyX86InterruptAttr>())
1120	return;
1121
1122	llvm::Function *Fn = cast<llvm::Function>(Val: GV);
1123	Fn->setCallingConv(llvm::CallingConv::X86_INTR);
1124	if (FD->getNumParams() == 0)
1125	return;
1126
1127	auto PtrTy = cast<PointerType>(FD->getParamDecl(i: 0)->getType());
1128	llvm::Type *ByValTy = CGM.getTypes().ConvertType(T: PtrTy->getPointeeType());
1129	llvm::Attribute NewAttr = llvm::Attribute::getWithByValType(
1130	Context&: Fn->getContext(), Ty: ByValTy);
1131	Fn->addParamAttr(ArgNo: 0, Attr: NewAttr);
1132	}
1133
1134	void X86_32TargetCodeGenInfo::setTargetAttributes(
1135	const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &CGM) const {
1136	if (GV->isDeclaration())
1137	return;
1138	if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(Val: D)) {
1139	if (FD->hasAttr<X86ForceAlignArgPointerAttr>()) {
1140	llvm::Function *Fn = cast<llvm::Function>(Val: GV);
1141	Fn->addFnAttr(Kind: "stackrealign");
1142	}
1143
1144	addX86InterruptAttrs(FD, GV, CGM);
1145	}
1146	}
1147
1148	bool X86_32TargetCodeGenInfo::initDwarfEHRegSizeTable(
1149	CodeGen::CodeGenFunction &CGF,
1150	llvm::Value *Address) const {
1151	CodeGen::CGBuilderTy &Builder = CGF.Builder;
1152
1153	llvm::Value *Four8 = llvm::ConstantInt::get(Ty: CGF.Int8Ty, V: 4);
1154
1155	// 0-7 are the eight integer registers; the order is different
1156	// on Darwin (for EH), but the range is the same.
1157	// 8 is %eip.
1158	AssignToArrayRange(Builder, Array: Address, Value: Four8, FirstIndex: 0, LastIndex: 8);
1159
1160	if (CGF.CGM.getTarget().getTriple().isOSDarwin()) {
1161	// 12-16 are st(0..4). Not sure why we stop at 4.
1162	// These have size 16, which is sizeof(long double) on
1163	// platforms with 8-byte alignment for that type.
1164	llvm::Value *Sixteen8 = llvm::ConstantInt::get(Ty: CGF.Int8Ty, V: 16);
1165	AssignToArrayRange(Builder, Array: Address, Value: Sixteen8, FirstIndex: 12, LastIndex: 16);
1166
1167	} else {
1168	// 9 is %eflags, which doesn't get a size on Darwin for some
1169	// reason.
1170	Builder.CreateAlignedStore(
1171	Val: Four8, Addr: Builder.CreateConstInBoundsGEP1_32(Ty: CGF.Int8Ty, Ptr: Address, Idx0: 9),
1172	Align: CharUnits::One());
1173
1174	// 11-16 are st(0..5). Not sure why we stop at 5.
1175	// These have size 12, which is sizeof(long double) on
1176	// platforms with 4-byte alignment for that type.
1177	llvm::Value *Twelve8 = llvm::ConstantInt::get(Ty: CGF.Int8Ty, V: 12);
1178	AssignToArrayRange(Builder, Array: Address, Value: Twelve8, FirstIndex: 11, LastIndex: 16);
1179	}
1180
1181	return false;
1182	}
1183
1184	//===----------------------------------------------------------------------===//
1185	// X86-64 ABI Implementation
1186	//===----------------------------------------------------------------------===//
1187
1188
1189	namespace {
1190
1191	/// \p returns the size in bits of the largest (native) vector for \p AVXLevel.
1192	static unsigned getNativeVectorSizeForAVXABI(X86AVXABILevel AVXLevel) {
1193	switch (AVXLevel) {
1194	case X86AVXABILevel::AVX512:
1195	return 512;
1196	case X86AVXABILevel::AVX:
1197	return 256;
1198	case X86AVXABILevel::None:
1199	return 128;
1200	}
1201	llvm_unreachable("Unknown AVXLevel");
1202	}
1203
1204	/// X86_64ABIInfo - The X86_64 ABI information.
1205	class X86_64ABIInfo : public ABIInfo {
1206	enum Class {
1207	Integer = 0,
1208	SSE,
1209	SSEUp,
1210	X87,
1211	X87Up,
1212	ComplexX87,
1213	NoClass,
1214	Memory
1215	};
1216
1217	/// merge - Implement the X86_64 ABI merging algorithm.
1218	///
1219	/// Merge an accumulating classification \arg Accum with a field
1220	/// classification \arg Field.
1221	///
1222	/// \param Accum - The accumulating classification. This should
1223	/// always be either NoClass or the result of a previous merge
1224	/// call. In addition, this should never be Memory (the caller
1225	/// should just return Memory for the aggregate).
1226	static Class merge(Class Accum, Class Field);
1227
1228	/// postMerge - Implement the X86_64 ABI post merging algorithm.
1229	///
1230	/// Post merger cleanup, reduces a malformed Hi and Lo pair to
1231	/// final MEMORY or SSE classes when necessary.
1232	///
1233	/// \param AggregateSize - The size of the current aggregate in
1234	/// the classification process.
1235	///
1236	/// \param Lo - The classification for the parts of the type
1237	/// residing in the low word of the containing object.
1238	///
1239	/// \param Hi - The classification for the parts of the type
1240	/// residing in the higher words of the containing object.
1241	///
1242	void postMerge(unsigned AggregateSize, Class &Lo, Class &Hi) const;
1243
1244	/// classify - Determine the x86_64 register classes in which the
1245	/// given type T should be passed.
1246	///
1247	/// \param Lo - The classification for the parts of the type
1248	/// residing in the low word of the containing object.
1249	///
1250	/// \param Hi - The classification for the parts of the type
1251	/// residing in the high word of the containing object.
1252	///
1253	/// \param OffsetBase - The bit offset of this type in the
1254	/// containing object. Some parameters are classified different
1255	/// depending on whether they straddle an eightbyte boundary.
1256	///
1257	/// \param isNamedArg - Whether the argument in question is a "named"
1258	/// argument, as used in AMD64-ABI 3.5.7.
1259	///
1260	/// \param IsRegCall - Whether the calling conversion is regcall.
1261	///
1262	/// If a word is unused its result will be NoClass; if a type should
1263	/// be passed in Memory then at least the classification of \arg Lo
1264	/// will be Memory.
1265	///
1266	/// The \arg Lo class will be NoClass iff the argument is ignored.
1267	///
1268	/// If the \arg Lo class is ComplexX87, then the \arg Hi class will
1269	/// also be ComplexX87.
1270	void classify(QualType T, uint64_t OffsetBase, Class &Lo, Class &Hi,
1271	bool isNamedArg, bool IsRegCall = false) const;
1272
1273	llvm::Type *GetByteVectorType(QualType Ty) const;
1274	llvm::Type *GetSSETypeAtOffset(llvm::Type *IRType,
1275	unsigned IROffset, QualType SourceTy,
1276	unsigned SourceOffset) const;
1277	llvm::Type *GetINTEGERTypeAtOffset(llvm::Type *IRType,
1278	unsigned IROffset, QualType SourceTy,
1279	unsigned SourceOffset) const;
1280
1281	/// getIndirectResult - Give a source type \arg Ty, return a suitable result
1282	/// such that the argument will be returned in memory.
1283	ABIArgInfo getIndirectReturnResult(QualType Ty) const;
1284
1285	/// getIndirectResult - Give a source type \arg Ty, return a suitable result
1286	/// such that the argument will be passed in memory.
1287	///
1288	/// \param freeIntRegs - The number of free integer registers remaining
1289	/// available.
1290	ABIArgInfo getIndirectResult(QualType Ty, unsigned freeIntRegs) const;
1291
1292	ABIArgInfo classifyReturnType(QualType RetTy) const;
1293
1294	ABIArgInfo classifyArgumentType(QualType Ty, unsigned freeIntRegs,
1295	unsigned &neededInt, unsigned &neededSSE,
1296	bool isNamedArg,
1297	bool IsRegCall = false) const;
1298
1299	ABIArgInfo classifyRegCallStructType(QualType Ty, unsigned &NeededInt,
1300	unsigned &NeededSSE,
1301	unsigned &MaxVectorWidth) const;
1302
1303	ABIArgInfo classifyRegCallStructTypeImpl(QualType Ty, unsigned &NeededInt,
1304	unsigned &NeededSSE,
1305	unsigned &MaxVectorWidth) const;
1306
1307	bool IsIllegalVectorType(QualType Ty) const;
1308
1309	/// The 0.98 ABI revision clarified a lot of ambiguities,
1310	/// unfortunately in ways that were not always consistent with
1311	/// certain previous compilers. In particular, platforms which
1312	/// required strict binary compatibility with older versions of GCC
1313	/// may need to exempt themselves.
1314	bool honorsRevision0_98() const {
1315	return !getTarget().getTriple().isOSDarwin();
1316	}
1317
1318	/// GCC classifies <1 x long long> as SSE but some platform ABIs choose to
1319	/// classify it as INTEGER (for compatibility with older clang compilers).
1320	bool classifyIntegerMMXAsSSE() const {
1321	// Clang <= 3.8 did not do this.
1322	if (getContext().getLangOpts().getClangABICompat() <=
1323	LangOptions::ClangABI::Ver3_8)
1324	return false;
1325
1326	const llvm::Triple &Triple = getTarget().getTriple();
1327	if (Triple.isOSDarwin() || Triple.isPS() || Triple.isOSFreeBSD())
1328	return false;
1329	return true;
1330	}
1331
1332	// GCC classifies vectors of __int128 as memory.
1333	bool passInt128VectorsInMem() const {
1334	// Clang <= 9.0 did not do this.
1335	if (getContext().getLangOpts().getClangABICompat() <=
1336	LangOptions::ClangABI::Ver9)
1337	return false;
1338
1339	const llvm::Triple &T = getTarget().getTriple();
1340	return T.isOSLinux() || T.isOSNetBSD();
1341	}
1342
1343	X86AVXABILevel AVXLevel;
1344	// Some ABIs (e.g. X32 ABI and Native Client OS) use 32 bit pointers on
1345	// 64-bit hardware.
1346	bool Has64BitPointers;
1347
1348	public:
1349	X86_64ABIInfo(CodeGen::CodeGenTypes &CGT, X86AVXABILevel AVXLevel)
1350	: ABIInfo(CGT), AVXLevel(AVXLevel),
1351	Has64BitPointers(CGT.getDataLayout().getPointerSize(AS: 0) == 8) {}
1352
1353	bool isPassedUsingAVXType(QualType type) const {
1354	unsigned neededInt, neededSSE;
1355	// The freeIntRegs argument doesn't matter here.
1356	ABIArgInfo info = classifyArgumentType(Ty: type, freeIntRegs: 0, neededInt, neededSSE,
1357	/*isNamedArg*/true);
1358	if (info.isDirect()) {
1359	llvm::Type *ty = info.getCoerceToType();
1360	if (llvm::VectorType *vectorTy = dyn_cast_or_null<llvm::VectorType>(Val: ty))
1361	return vectorTy->getPrimitiveSizeInBits().getFixedValue() > 128;
1362	}
1363	return false;
1364	}
1365
1366	void computeInfo(CGFunctionInfo &FI) const override;
1367
1368	Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
1369	QualType Ty) const override;
1370	Address EmitMSVAArg(CodeGenFunction &CGF, Address VAListAddr,
1371	QualType Ty) const override;
1372
1373	bool has64BitPointers() const {
1374	return Has64BitPointers;
1375	}
1376	};
1377
1378	/// WinX86_64ABIInfo - The Windows X86_64 ABI information.
1379	class WinX86_64ABIInfo : public ABIInfo {
1380	public:
1381	WinX86_64ABIInfo(CodeGen::CodeGenTypes &CGT, X86AVXABILevel AVXLevel)
1382	: ABIInfo(CGT), AVXLevel(AVXLevel),
1383	IsMingw64(getTarget().getTriple().isWindowsGNUEnvironment()) {}
1384
1385	void computeInfo(CGFunctionInfo &FI) const override;
1386
1387	Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
1388	QualType Ty) const override;
1389
1390	bool isHomogeneousAggregateBaseType(QualType Ty) const override {
1391	// FIXME: Assumes vectorcall is in use.
1392	return isX86VectorTypeForVectorCall(Context&: getContext(), Ty);
1393	}
1394
1395	bool isHomogeneousAggregateSmallEnough(const Type *Ty,
1396	uint64_t NumMembers) const override {
1397	// FIXME: Assumes vectorcall is in use.
1398	return isX86VectorCallAggregateSmallEnough(NumMembers);
1399	}
1400
1401	private:
1402	ABIArgInfo classify(QualType Ty, unsigned &FreeSSERegs, bool IsReturnType,
1403	bool IsVectorCall, bool IsRegCall) const;
1404	ABIArgInfo reclassifyHvaArgForVectorCall(QualType Ty, unsigned &FreeSSERegs,
1405	const ABIArgInfo &current) const;
1406
1407	X86AVXABILevel AVXLevel;
1408
1409	bool IsMingw64;
1410	};
1411
1412	class X86_64TargetCodeGenInfo : public TargetCodeGenInfo {
1413	public:
1414	X86_64TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT, X86AVXABILevel AVXLevel)
1415	: TargetCodeGenInfo(std::make_unique<X86_64ABIInfo>(args&: CGT, args&: AVXLevel)) {
1416	SwiftInfo =
1417	std::make_unique<SwiftABIInfo>(args&: CGT, /*SwiftErrorInRegister=*/args: true);
1418	}
1419
1420	/// Disable tail call on x86-64. The epilogue code before the tail jump blocks
1421	/// autoreleaseRV/retainRV and autoreleaseRV/unsafeClaimRV optimizations.
1422	bool markARCOptimizedReturnCallsAsNoTail() const override { return true; }
1423
1424	int getDwarfEHStackPointer(CodeGen::CodeGenModule &CGM) const override {
1425	return 7;
1426	}
1427
1428	bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
1429	llvm::Value *Address) const override {
1430	llvm::Value *Eight8 = llvm::ConstantInt::get(Ty: CGF.Int8Ty, V: 8);
1431
1432	// 0-15 are the 16 integer registers.
1433	// 16 is %rip.
1434	AssignToArrayRange(Builder&: CGF.Builder, Array: Address, Value: Eight8, FirstIndex: 0, LastIndex: 16);
1435	return false;
1436	}
1437
1438	llvm::Type* adjustInlineAsmType(CodeGen::CodeGenFunction &CGF,
1439	StringRef Constraint,
1440	llvm::Type* Ty) const override {
1441	return X86AdjustInlineAsmType(CGF, Constraint, Ty);
1442	}
1443
1444	bool isNoProtoCallVariadic(const CallArgList &args,
1445	const FunctionNoProtoType *fnType) const override {
1446	// The default CC on x86-64 sets %al to the number of SSA
1447	// registers used, and GCC sets this when calling an unprototyped
1448	// function, so we override the default behavior. However, don't do
1449	// that when AVX types are involved: the ABI explicitly states it is
1450	// undefined, and it doesn't work in practice because of how the ABI
1451	// defines varargs anyway.
1452	if (fnType->getCallConv() == CC_C) {
1453	bool HasAVXType = false;
1454	for (CallArgList::const_iterator
1455	it = args.begin(), ie = args.end(); it != ie; ++it) {
1456	if (getABIInfo<X86_64ABIInfo>().isPassedUsingAVXType(type: it->Ty)) {
1457	HasAVXType = true;
1458	break;
1459	}
1460	}
1461
1462	if (!HasAVXType)
1463	return true;
1464	}
1465
1466	return TargetCodeGenInfo::isNoProtoCallVariadic(args, fnType);
1467	}
1468
1469	void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
1470	CodeGen::CodeGenModule &CGM) const override {
1471	if (GV->isDeclaration())
1472	return;
1473	if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(Val: D)) {
1474	if (FD->hasAttr<X86ForceAlignArgPointerAttr>()) {
1475	llvm::Function *Fn = cast<llvm::Function>(Val: GV);
1476	Fn->addFnAttr(Kind: "stackrealign");
1477	}
1478
1479	addX86InterruptAttrs(FD, GV, CGM);
1480	}
1481	}
1482
1483	void checkFunctionCallABI(CodeGenModule &CGM, SourceLocation CallLoc,
1484	const FunctionDecl *Caller,
1485	const FunctionDecl *Callee,
1486	const CallArgList &Args) const override;
1487	};
1488	} // namespace
1489
1490	static void initFeatureMaps(const ASTContext &Ctx,
1491	llvm::StringMap<bool> &CallerMap,
1492	const FunctionDecl *Caller,
1493	llvm::StringMap<bool> &CalleeMap,
1494	const FunctionDecl *Callee) {
1495	if (CalleeMap.empty() && CallerMap.empty()) {
1496	// The caller is potentially nullptr in the case where the call isn't in a
1497	// function. In this case, the getFunctionFeatureMap ensures we just get
1498	// the TU level setting (since it cannot be modified by 'target'..
1499	Ctx.getFunctionFeatureMap(FeatureMap&: CallerMap, Caller);
1500	Ctx.getFunctionFeatureMap(FeatureMap&: CalleeMap, Callee);
1501	}
1502	}
1503
1504	static bool checkAVXParamFeature(DiagnosticsEngine &Diag,
1505	SourceLocation CallLoc,
1506	const llvm::StringMap<bool> &CallerMap,
1507	const llvm::StringMap<bool> &CalleeMap,
1508	QualType Ty, StringRef Feature,
1509	bool IsArgument) {
1510	bool CallerHasFeat = CallerMap.lookup(Key: Feature);
1511	bool CalleeHasFeat = CalleeMap.lookup(Key: Feature);
1512	if (!CallerHasFeat && !CalleeHasFeat)
1513	return Diag.Report(CallLoc, diag::warn_avx_calling_convention)
1514	<< IsArgument << Ty << Feature;
1515
1516	// Mixing calling conventions here is very clearly an error.
1517	if (!CallerHasFeat || !CalleeHasFeat)
1518	return Diag.Report(CallLoc, diag::err_avx_calling_convention)
1519	<< IsArgument << Ty << Feature;
1520
1521	// Else, both caller and callee have the required feature, so there is no need
1522	// to diagnose.
1523	return false;
1524	}
1525
1526	static bool checkAVX512ParamFeature(DiagnosticsEngine &Diag,
1527	SourceLocation CallLoc,
1528	const llvm::StringMap<bool> &CallerMap,
1529	const llvm::StringMap<bool> &CalleeMap,
1530	QualType Ty, bool IsArgument) {
1531	bool Caller256 = CallerMap.lookup(Key: "avx512f") && !CallerMap.lookup(Key: "evex512");
1532	bool Callee256 = CalleeMap.lookup(Key: "avx512f") && !CalleeMap.lookup(Key: "evex512");
1533
1534	// Forbid 512-bit or larger vector pass or return when we disabled ZMM
1535	// instructions.
1536	if (Caller256 || Callee256)
1537	return Diag.Report(CallLoc, diag::err_avx_calling_convention)
1538	<< IsArgument << Ty << "evex512";
1539
1540	return checkAVXParamFeature(Diag, CallLoc, CallerMap, CalleeMap, Ty,
1541	Feature: "avx512f", IsArgument);
1542	}
1543
1544	static bool checkAVXParam(DiagnosticsEngine &Diag, ASTContext &Ctx,
1545	SourceLocation CallLoc,
1546	const llvm::StringMap<bool> &CallerMap,
1547	const llvm::StringMap<bool> &CalleeMap, QualType Ty,
1548	bool IsArgument) {
1549	uint64_t Size = Ctx.getTypeSize(T: Ty);
1550	if (Size > 256)
1551	return checkAVX512ParamFeature(Diag, CallLoc, CallerMap, CalleeMap, Ty,
1552	IsArgument);
1553
1554	if (Size > 128)
1555	return checkAVXParamFeature(Diag, CallLoc, CallerMap, CalleeMap, Ty, Feature: "avx",
1556	IsArgument);
1557
1558	return false;
1559	}
1560
1561	void X86_64TargetCodeGenInfo::checkFunctionCallABI(
1562	CodeGenModule &CGM, SourceLocation CallLoc, const FunctionDecl *Caller,
1563	const FunctionDecl *Callee, const CallArgList &Args) const {
1564	llvm::StringMap<bool> CallerMap;
1565	llvm::StringMap<bool> CalleeMap;
1566	unsigned ArgIndex = 0;
1567
1568	// We need to loop through the actual call arguments rather than the
1569	// function's parameters, in case this variadic.
1570	for (const CallArg &Arg : Args) {
1571	// The "avx" feature changes how vectors >128 in size are passed. "avx512f"
1572	// additionally changes how vectors >256 in size are passed. Like GCC, we
1573	// warn when a function is called with an argument where this will change.
1574	// Unlike GCC, we also error when it is an obvious ABI mismatch, that is,
1575	// the caller and callee features are mismatched.
1576	// Unfortunately, we cannot do this diagnostic in SEMA, since the callee can
1577	// change its ABI with attribute-target after this call.
1578	if (Arg.getType()->isVectorType() &&
1579	CGM.getContext().getTypeSize(T: Arg.getType()) > 128) {
1580	initFeatureMaps(Ctx: CGM.getContext(), CallerMap, Caller, CalleeMap, Callee);
1581	QualType Ty = Arg.getType();
1582	// The CallArg seems to have desugared the type already, so for clearer
1583	// diagnostics, replace it with the type in the FunctionDecl if possible.
1584	if (ArgIndex < Callee->getNumParams())
1585	Ty = Callee->getParamDecl(i: ArgIndex)->getType();
1586
1587	if (checkAVXParam(Diag&: CGM.getDiags(), Ctx&: CGM.getContext(), CallLoc, CallerMap,
1588	CalleeMap, Ty, /*IsArgument*/ true))
1589	return;
1590	}
1591	++ArgIndex;
1592	}
1593
1594	// Check return always, as we don't have a good way of knowing in codegen
1595	// whether this value is used, tail-called, etc.
1596	if (Callee->getReturnType()->isVectorType() &&
1597	CGM.getContext().getTypeSize(T: Callee->getReturnType()) > 128) {
1598	initFeatureMaps(Ctx: CGM.getContext(), CallerMap, Caller, CalleeMap, Callee);
1599	checkAVXParam(Diag&: CGM.getDiags(), Ctx&: CGM.getContext(), CallLoc, CallerMap,
1600	CalleeMap, Ty: Callee->getReturnType(),
1601	/*IsArgument*/ false);
1602	}
1603	}
1604
1605	std::string TargetCodeGenInfo::qualifyWindowsLibrary(StringRef Lib) {
1606	// If the argument does not end in .lib, automatically add the suffix.
1607	// If the argument contains a space, enclose it in quotes.
1608	// This matches the behavior of MSVC.
1609	bool Quote = Lib.contains(C: ' ');
1610	std::string ArgStr = Quote ? "\"" : "";
1611	ArgStr += Lib;
1612	if (!Lib.ends_with_insensitive(Suffix: ".lib") && !Lib.ends_with_insensitive(Suffix: ".a"))
1613	ArgStr += ".lib";
1614	ArgStr += Quote ? "\"" : "";
1615	return ArgStr;
1616	}
1617
1618	namespace {
1619	class WinX86_32TargetCodeGenInfo : public X86_32TargetCodeGenInfo {
1620	public:
1621	WinX86_32TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT,
1622	bool DarwinVectorABI, bool RetSmallStructInRegABI, bool Win32StructABI,
1623	unsigned NumRegisterParameters)
1624	: X86_32TargetCodeGenInfo(CGT, DarwinVectorABI, RetSmallStructInRegABI,
1625	Win32StructABI, NumRegisterParameters, false) {}
1626
1627	void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
1628	CodeGen::CodeGenModule &CGM) const override;
1629
1630	void getDependentLibraryOption(llvm::StringRef Lib,
1631	llvm::SmallString<24> &Opt) const override {
1632	Opt = "/DEFAULTLIB:";
1633	Opt += qualifyWindowsLibrary(Lib);
1634	}
1635
1636	void getDetectMismatchOption(llvm::StringRef Name,
1637	llvm::StringRef Value,
1638	llvm::SmallString<32> &Opt) const override {
1639	Opt = "/FAILIFMISMATCH:\"" + Name.str() + "=" + Value.str() + "\"";
1640	}
1641	};
1642	} // namespace
1643
1644	void WinX86_32TargetCodeGenInfo::setTargetAttributes(
1645	const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &CGM) const {
1646	X86_32TargetCodeGenInfo::setTargetAttributes(D, GV, CGM);
1647	if (GV->isDeclaration())
1648	return;
1649	addStackProbeTargetAttributes(D, GV, CGM);
1650	}
1651
1652	namespace {
1653	class WinX86_64TargetCodeGenInfo : public TargetCodeGenInfo {
1654	public:
1655	WinX86_64TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT,
1656	X86AVXABILevel AVXLevel)
1657	: TargetCodeGenInfo(std::make_unique<WinX86_64ABIInfo>(args&: CGT, args&: AVXLevel)) {
1658	SwiftInfo =
1659	std::make_unique<SwiftABIInfo>(args&: CGT, /*SwiftErrorInRegister=*/args: true);
1660	}
1661
1662	void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
1663	CodeGen::CodeGenModule &CGM) const override;
1664
1665	int getDwarfEHStackPointer(CodeGen::CodeGenModule &CGM) const override {
1666	return 7;
1667	}
1668
1669	bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
1670	llvm::Value *Address) const override {
1671	llvm::Value *Eight8 = llvm::ConstantInt::get(Ty: CGF.Int8Ty, V: 8);
1672
1673	// 0-15 are the 16 integer registers.
1674	// 16 is %rip.
1675	AssignToArrayRange(Builder&: CGF.Builder, Array: Address, Value: Eight8, FirstIndex: 0, LastIndex: 16);
1676	return false;
1677	}
1678
1679	void getDependentLibraryOption(llvm::StringRef Lib,
1680	llvm::SmallString<24> &Opt) const override {
1681	Opt = "/DEFAULTLIB:";
1682	Opt += qualifyWindowsLibrary(Lib);
1683	}
1684
1685	void getDetectMismatchOption(llvm::StringRef Name,
1686	llvm::StringRef Value,
1687	llvm::SmallString<32> &Opt) const override {
1688	Opt = "/FAILIFMISMATCH:\"" + Name.str() + "=" + Value.str() + "\"";
1689	}
1690	};
1691	} // namespace
1692
1693	void WinX86_64TargetCodeGenInfo::setTargetAttributes(
1694	const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &CGM) const {
1695	TargetCodeGenInfo::setTargetAttributes(D, GV, M&: CGM);
1696	if (GV->isDeclaration())
1697	return;
1698	if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(Val: D)) {
1699	if (FD->hasAttr<X86ForceAlignArgPointerAttr>()) {
1700	llvm::Function *Fn = cast<llvm::Function>(Val: GV);
1701	Fn->addFnAttr(Kind: "stackrealign");
1702	}
1703
1704	addX86InterruptAttrs(FD, GV, CGM);
1705	}
1706
1707	addStackProbeTargetAttributes(D, GV, CGM);
1708	}
1709
1710	void X86_64ABIInfo::postMerge(unsigned AggregateSize, Class &Lo,
1711	Class &Hi) const {
1712	// AMD64-ABI 3.2.3p2: Rule 5. Then a post merger cleanup is done:
1713	//
1714	// (a) If one of the classes is Memory, the whole argument is passed in
1715	// memory.
1716	//
1717	// (b) If X87UP is not preceded by X87, the whole argument is passed in
1718	// memory.
1719	//
1720	// (c) If the size of the aggregate exceeds two eightbytes and the first
1721	// eightbyte isn't SSE or any other eightbyte isn't SSEUP, the whole
1722	// argument is passed in memory. NOTE: This is necessary to keep the
1723	// ABI working for processors that don't support the __m256 type.
1724	//
1725	// (d) If SSEUP is not preceded by SSE or SSEUP, it is converted to SSE.
1726	//
1727	// Some of these are enforced by the merging logic. Others can arise
1728	// only with unions; for example:
1729	// union { _Complex double; unsigned; }
1730	//
1731	// Note that clauses (b) and (c) were added in 0.98.
1732	//
1733	if (Hi == Memory)
1734	Lo = Memory;
1735	if (Hi == X87Up && Lo != X87 && honorsRevision0_98())
1736	Lo = Memory;
1737	if (AggregateSize > 128 && (Lo != SSE || Hi != SSEUp))
1738	Lo = Memory;
1739	if (Hi == SSEUp && Lo != SSE)
1740	Hi = SSE;
1741	}
1742
1743	X86_64ABIInfo::Class X86_64ABIInfo::merge(Class Accum, Class Field) {
1744	// AMD64-ABI 3.2.3p2: Rule 4. Each field of an object is
1745	// classified recursively so that always two fields are
1746	// considered. The resulting class is calculated according to
1747	// the classes of the fields in the eightbyte:
1748	//
1749	// (a) If both classes are equal, this is the resulting class.
1750	//
1751	// (b) If one of the classes is NO_CLASS, the resulting class is
1752	// the other class.
1753	//
1754	// (c) If one of the classes is MEMORY, the result is the MEMORY
1755	// class.
1756	//
1757	// (d) If one of the classes is INTEGER, the result is the
1758	// INTEGER.
1759	//
1760	// (e) If one of the classes is X87, X87UP, COMPLEX_X87 class,
1761	// MEMORY is used as class.
1762	//
1763	// (f) Otherwise class SSE is used.
1764
1765	// Accum should never be memory (we should have returned) or
1766	// ComplexX87 (because this cannot be passed in a structure).
1767	assert((Accum != Memory && Accum != ComplexX87) &&
1768	"Invalid accumulated classification during merge.");
1769	if (Accum == Field || Field == NoClass)
1770	return Accum;
1771	if (Field == Memory)
1772	return Memory;
1773	if (Accum == NoClass)
1774	return Field;
1775	if (Accum == Integer || Field == Integer)
1776	return Integer;
1777	if (Field == X87 || Field == X87Up || Field == ComplexX87 ||
1778	Accum == X87 || Accum == X87Up)
1779	return Memory;
1780	return SSE;
1781	}
1782
1783	void X86_64ABIInfo::classify(QualType Ty, uint64_t OffsetBase, Class &Lo,
1784	Class &Hi, bool isNamedArg, bool IsRegCall) const {
1785	// FIXME: This code can be simplified by introducing a simple value class for
1786	// Class pairs with appropriate constructor methods for the various
1787	// situations.
1788
1789	// FIXME: Some of the split computations are wrong; unaligned vectors
1790	// shouldn't be passed in registers for example, so there is no chance they
1791	// can straddle an eightbyte. Verify & simplify.
1792
1793	Lo = Hi = NoClass;
1794
1795	Class &Current = OffsetBase < 64 ? Lo : Hi;
1796	Current = Memory;
1797
1798	if (const BuiltinType *BT = Ty->getAs<BuiltinType>()) {
1799	BuiltinType::Kind k = BT->getKind();
1800
1801	if (k == BuiltinType::Void) {
1802	Current = NoClass;
1803	} else if (k == BuiltinType::Int128 || k == BuiltinType::UInt128) {
1804	Lo = Integer;
1805	Hi = Integer;
1806	} else if (k >= BuiltinType::Bool && k <= BuiltinType::LongLong) {
1807	Current = Integer;
1808	} else if (k == BuiltinType::Float || k == BuiltinType::Double ||
1809	k == BuiltinType::Float16 || k == BuiltinType::BFloat16) {
1810	Current = SSE;
1811	} else if (k == BuiltinType::Float128) {
1812	Lo = SSE;
1813	Hi = SSEUp;
1814	} else if (k == BuiltinType::LongDouble) {
1815	const llvm::fltSemantics *LDF = &getTarget().getLongDoubleFormat();
1816	if (LDF == &llvm::APFloat::IEEEquad()) {
1817	Lo = SSE;
1818	Hi = SSEUp;
1819	} else if (LDF == &llvm::APFloat::x87DoubleExtended()) {
1820	Lo = X87;
1821	Hi = X87Up;
1822	} else if (LDF == &llvm::APFloat::IEEEdouble()) {
1823	Current = SSE;
1824	} else
1825	llvm_unreachable("unexpected long double representation!");
1826	}
1827	// FIXME: _Decimal32 and _Decimal64 are SSE.
1828	// FIXME: _float128 and _Decimal128 are (SSE, SSEUp).
1829	return;
1830	}
1831
1832	if (const EnumType *ET = Ty->getAs<EnumType>()) {
1833	// Classify the underlying integer type.
1834	classify(Ty: ET->getDecl()->getIntegerType(), OffsetBase, Lo, Hi, isNamedArg);
1835	return;
1836	}
1837
1838	if (Ty->hasPointerRepresentation()) {
1839	Current = Integer;
1840	return;
1841	}
1842
1843	if (Ty->isMemberPointerType()) {
1844	if (Ty->isMemberFunctionPointerType()) {
1845	if (Has64BitPointers) {
1846	// If Has64BitPointers, this is an {i64, i64}, so classify both
1847	// Lo and Hi now.
1848	Lo = Hi = Integer;
1849	} else {
1850	// Otherwise, with 32-bit pointers, this is an {i32, i32}. If that
1851	// straddles an eightbyte boundary, Hi should be classified as well.
1852	uint64_t EB_FuncPtr = (OffsetBase) / 64;
1853	uint64_t EB_ThisAdj = (OffsetBase + 64 - 1) / 64;
1854	if (EB_FuncPtr != EB_ThisAdj) {
1855	Lo = Hi = Integer;
1856	} else {
1857	Current = Integer;
1858	}
1859	}
1860	} else {
1861	Current = Integer;
1862	}
1863	return;
1864	}
1865
1866	if (const VectorType *VT = Ty->getAs<VectorType>()) {
1867	uint64_t Size = getContext().getTypeSize(VT);
1868	if (Size == 1 || Size == 8 || Size == 16 || Size == 32) {
1869	// gcc passes the following as integer:
1870	// 4 bytes - <4 x char>, <2 x short>, <1 x int>, <1 x float>
1871	// 2 bytes - <2 x char>, <1 x short>
1872	// 1 byte - <1 x char>
1873	Current = Integer;
1874
1875	// If this type crosses an eightbyte boundary, it should be
1876	// split.
1877	uint64_t EB_Lo = (OffsetBase) / 64;
1878	uint64_t EB_Hi = (OffsetBase + Size - 1) / 64;
1879	if (EB_Lo != EB_Hi)
1880	Hi = Lo;
1881	} else if (Size == 64) {
1882	QualType ElementType = VT->getElementType();
1883
1884	// gcc passes <1 x double> in memory. :(
1885	if (ElementType->isSpecificBuiltinType(K: BuiltinType::Double))
1886	return;
1887
1888	// gcc passes <1 x long long> as SSE but clang used to unconditionally
1889	// pass them as integer. For platforms where clang is the de facto
1890	// platform compiler, we must continue to use integer.
1891	if (!classifyIntegerMMXAsSSE() &&
1892	(ElementType->isSpecificBuiltinType(K: BuiltinType::LongLong) ||
1893	ElementType->isSpecificBuiltinType(K: BuiltinType::ULongLong) ||
1894	ElementType->isSpecificBuiltinType(K: BuiltinType::Long) ||
1895	ElementType->isSpecificBuiltinType(K: BuiltinType::ULong)))
1896	Current = Integer;
1897	else
1898	Current = SSE;
1899
1900	// If this type crosses an eightbyte boundary, it should be
1901	// split.
1902	if (OffsetBase && OffsetBase != 64)
1903	Hi = Lo;
1904	} else if (Size == 128 ||
1905	(isNamedArg && Size <= getNativeVectorSizeForAVXABI(AVXLevel))) {
1906	QualType ElementType = VT->getElementType();
1907
1908	// gcc passes 256 and 512 bit <X x __int128> vectors in memory. :(
1909	if (passInt128VectorsInMem() && Size != 128 &&
1910	(ElementType->isSpecificBuiltinType(K: BuiltinType::Int128) ||
1911	ElementType->isSpecificBuiltinType(K: BuiltinType::UInt128)))
1912	return;
1913
1914	// Arguments of 256-bits are split into four eightbyte chunks. The
1915	// least significant one belongs to class SSE and all the others to class
1916	// SSEUP. The original Lo and Hi design considers that types can't be
1917	// greater than 128-bits, so a 64-bit split in Hi and Lo makes sense.
1918	// This design isn't correct for 256-bits, but since there're no cases
1919	// where the upper parts would need to be inspected, avoid adding
1920	// complexity and just consider Hi to match the 64-256 part.
1921	//
1922	// Note that per 3.5.7 of AMD64-ABI, 256-bit args are only passed in
1923	// registers if they are "named", i.e. not part of the "..." of a
1924	// variadic function.
1925	//
1926	// Similarly, per 3.2.3. of the AVX512 draft, 512-bits ("named") args are
1927	// split into eight eightbyte chunks, one SSE and seven SSEUP.
1928	Lo = SSE;
1929	Hi = SSEUp;
1930	}
1931	return;
1932	}
1933
1934	if (const ComplexType *CT = Ty->getAs<ComplexType>()) {
1935	QualType ET = getContext().getCanonicalType(T: CT->getElementType());
1936
1937	uint64_t Size = getContext().getTypeSize(T: Ty);
1938	if (ET->isIntegralOrEnumerationType()) {
1939	if (Size <= 64)
1940	Current = Integer;
1941	else if (Size <= 128)
1942	Lo = Hi = Integer;
1943	} else if (ET->isFloat16Type() || ET == getContext().FloatTy ||
1944	ET->isBFloat16Type()) {
1945	Current = SSE;
1946	} else if (ET == getContext().DoubleTy) {
1947	Lo = Hi = SSE;
1948	} else if (ET == getContext().LongDoubleTy) {
1949	const llvm::fltSemantics *LDF = &getTarget().getLongDoubleFormat();
1950	if (LDF == &llvm::APFloat::IEEEquad())
1951	Current = Memory;
1952	else if (LDF == &llvm::APFloat::x87DoubleExtended())
1953	Current = ComplexX87;
1954	else if (LDF == &llvm::APFloat::IEEEdouble())
1955	Lo = Hi = SSE;
1956	else
1957	llvm_unreachable("unexpected long double representation!");
1958	}
1959
1960	// If this complex type crosses an eightbyte boundary then it
1961	// should be split.
1962	uint64_t EB_Real = (OffsetBase) / 64;
1963	uint64_t EB_Imag = (OffsetBase + getContext().getTypeSize(T: ET)) / 64;
1964	if (Hi == NoClass && EB_Real != EB_Imag)
1965	Hi = Lo;
1966
1967	return;
1968	}
1969
1970	if (const auto *EITy = Ty->getAs<BitIntType>()) {
1971	if (EITy->getNumBits() <= 64)
1972	Current = Integer;
1973	else if (EITy->getNumBits() <= 128)
1974	Lo = Hi = Integer;
1975	// Larger values need to get passed in memory.
1976	return;
1977	}
1978
1979	if (const ConstantArrayType *AT = getContext().getAsConstantArrayType(T: Ty)) {
1980	// Arrays are treated like structures.
1981
1982	uint64_t Size = getContext().getTypeSize(T: Ty);
1983
1984	// AMD64-ABI 3.2.3p2: Rule 1. If the size of an object is larger
1985	// than eight eightbytes, ..., it has class MEMORY.
1986	// regcall ABI doesn't have limitation to an object. The only limitation
1987	// is the free registers, which will be checked in computeInfo.
1988	if (!IsRegCall && Size > 512)
1989	return;
1990
1991	// AMD64-ABI 3.2.3p2: Rule 1. If ..., or it contains unaligned
1992	// fields, it has class MEMORY.
1993	//
1994	// Only need to check alignment of array base.
1995	if (OffsetBase % getContext().getTypeAlign(AT->getElementType()))
1996	return;
1997
1998	// Otherwise implement simplified merge. We could be smarter about
1999	// this, but it isn't worth it and would be harder to verify.
2000	Current = NoClass;
2001	uint64_t EltSize = getContext().getTypeSize(AT->getElementType());
2002	uint64_t ArraySize = AT->getZExtSize();
2003
2004	// The only case a 256-bit wide vector could be used is when the array
2005	// contains a single 256-bit element. Since Lo and Hi logic isn't extended
2006	// to work for sizes wider than 128, early check and fallback to memory.
2007	//
2008	if (Size > 128 &&
2009	(Size != EltSize || Size > getNativeVectorSizeForAVXABI(AVXLevel)))
2010	return;
2011
2012	for (uint64_t i=0, Offset=OffsetBase; i<ArraySize; ++i, Offset += EltSize) {
2013	Class FieldLo, FieldHi;
2014	classify(Ty: AT->getElementType(), OffsetBase: Offset, Lo&: FieldLo, Hi&: FieldHi, isNamedArg);
2015	Lo = merge(Accum: Lo, Field: FieldLo);
2016	Hi = merge(Accum: Hi, Field: FieldHi);
2017	if (Lo == Memory || Hi == Memory)
2018	break;
2019	}
2020
2021	postMerge(AggregateSize: Size, Lo, Hi);
2022	assert((Hi != SSEUp || Lo == SSE) && "Invalid SSEUp array classification.");
2023	return;
2024	}
2025
2026	if (const RecordType *RT = Ty->getAs<RecordType>()) {
2027	uint64_t Size = getContext().getTypeSize(T: Ty);
2028
2029	// AMD64-ABI 3.2.3p2: Rule 1. If the size of an object is larger
2030	// than eight eightbytes, ..., it has class MEMORY.
2031	if (Size > 512)
2032	return;
2033
2034	// AMD64-ABI 3.2.3p2: Rule 2. If a C++ object has either a non-trivial
2035	// copy constructor or a non-trivial destructor, it is passed by invisible
2036	// reference.
2037	if (getRecordArgABI(RT, CXXABI&: getCXXABI()))
2038	return;
2039
2040	const RecordDecl *RD = RT->getDecl();
2041
2042	// Assume variable sized types are passed in memory.
2043	if (RD->hasFlexibleArrayMember())
2044	return;
2045
2046	const ASTRecordLayout &Layout = getContext().getASTRecordLayout(D: RD);
2047
2048	// Reset Lo class, this will be recomputed.
2049	Current = NoClass;
2050
2051	// If this is a C++ record, classify the bases first.
2052	if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(Val: RD)) {
2053	for (const auto &I : CXXRD->bases()) {
2054	assert(!I.isVirtual() && !I.getType()->isDependentType() &&
2055	"Unexpected base class!");
2056	const auto *Base =
2057	cast<CXXRecordDecl>(Val: I.getType()->castAs<RecordType>()->getDecl());
2058
2059	// Classify this field.
2060	//
2061	// AMD64-ABI 3.2.3p2: Rule 3. If the size of the aggregate exceeds a
2062	// single eightbyte, each is classified separately. Each eightbyte gets
2063	// initialized to class NO_CLASS.
2064	Class FieldLo, FieldHi;
2065	uint64_t Offset =
2066	OffsetBase + getContext().toBits(CharSize: Layout.getBaseClassOffset(Base));
2067	classify(Ty: I.getType(), OffsetBase: Offset, Lo&: FieldLo, Hi&: FieldHi, isNamedArg);
2068	Lo = merge(Accum: Lo, Field: FieldLo);
2069	Hi = merge(Accum: Hi, Field: FieldHi);
2070	if (Lo == Memory || Hi == Memory) {
2071	postMerge(AggregateSize: Size, Lo, Hi);
2072	return;
2073	}
2074	}
2075	}
2076
2077	// Classify the fields one at a time, merging the results.
2078	unsigned idx = 0;
2079	bool UseClang11Compat = getContext().getLangOpts().getClangABICompat() <=
2080	LangOptions::ClangABI::Ver11 ||
2081	getContext().getTargetInfo().getTriple().isPS();
2082	bool IsUnion = RT->isUnionType() && !UseClang11Compat;
2083
2084	for (RecordDecl::field_iterator i = RD->field_begin(), e = RD->field_end();
2085	i != e; ++i, ++idx) {
2086	uint64_t Offset = OffsetBase + Layout.getFieldOffset(FieldNo: idx);
2087	bool BitField = i->isBitField();
2088
2089	// Ignore padding bit-fields.
2090	if (BitField && i->isUnnamedBitField())
2091	continue;
2092
2093	// AMD64-ABI 3.2.3p2: Rule 1. If the size of an object is larger than
2094	// eight eightbytes, or it contains unaligned fields, it has class MEMORY.
2095	//
2096	// The only case a 256-bit or a 512-bit wide vector could be used is when
2097	// the struct contains a single 256-bit or 512-bit element. Early check
2098	// and fallback to memory.
2099	//
2100	// FIXME: Extended the Lo and Hi logic properly to work for size wider
2101	// than 128.
2102	if (Size > 128 &&
2103	((!IsUnion && Size != getContext().getTypeSize(i->getType())) ||
2104	Size > getNativeVectorSizeForAVXABI(AVXLevel))) {
2105	Lo = Memory;
2106	postMerge(AggregateSize: Size, Lo, Hi);
2107	return;
2108	}
2109
2110	bool IsInMemory =
2111	Offset % getContext().getTypeAlign(i->getType().getCanonicalType());
2112	// Note, skip this test for bit-fields, see below.
2113	if (!BitField && IsInMemory) {
2114	Lo = Memory;
2115	postMerge(AggregateSize: Size, Lo, Hi);
2116	return;
2117	}
2118
2119	// Classify this field.
2120	//
2121	// AMD64-ABI 3.2.3p2: Rule 3. If the size of the aggregate
2122	// exceeds a single eightbyte, each is classified
2123	// separately. Each eightbyte gets initialized to class
2124	// NO_CLASS.
2125	Class FieldLo, FieldHi;
2126
2127	// Bit-fields require special handling, they do not force the
2128	// structure to be passed in memory even if unaligned, and
2129	// therefore they can straddle an eightbyte.
2130	if (BitField) {
2131	assert(!i->isUnnamedBitField());
2132	uint64_t Offset = OffsetBase + Layout.getFieldOffset(FieldNo: idx);
2133	uint64_t Size = i->getBitWidthValue(Ctx: getContext());
2134
2135	uint64_t EB_Lo = Offset / 64;
2136	uint64_t EB_Hi = (Offset + Size - 1) / 64;
2137
2138	if (EB_Lo) {
2139	assert(EB_Hi == EB_Lo && "Invalid classification, type > 16 bytes.");
2140	FieldLo = NoClass;
2141	FieldHi = Integer;
2142	} else {
2143	FieldLo = Integer;
2144	FieldHi = EB_Hi ? Integer : NoClass;
2145	}
2146	} else
2147	classify(Ty: i->getType(), OffsetBase: Offset, Lo&: FieldLo, Hi&: FieldHi, isNamedArg);
2148	Lo = merge(Accum: Lo, Field: FieldLo);
2149	Hi = merge(Accum: Hi, Field: FieldHi);
2150	if (Lo == Memory || Hi == Memory)
2151	break;
2152	}
2153
2154	postMerge(AggregateSize: Size, Lo, Hi);
2155	}
2156	}
2157
2158	ABIArgInfo X86_64ABIInfo::getIndirectReturnResult(QualType Ty) const {
2159	// If this is a scalar LLVM value then assume LLVM will pass it in the right
2160	// place naturally.
2161	if (!isAggregateTypeForABI(T: Ty)) {
2162	// Treat an enum type as its underlying type.
2163	if (const EnumType *EnumTy = Ty->getAs<EnumType>())
2164	Ty = EnumTy->getDecl()->getIntegerType();
2165
2166	if (Ty->isBitIntType())
2167	return getNaturalAlignIndirect(Ty);
2168
2169	return (isPromotableIntegerTypeForABI(Ty) ? ABIArgInfo::getExtend(Ty)
2170	: ABIArgInfo::getDirect());
2171	}
2172
2173	return getNaturalAlignIndirect(Ty);
2174	}
2175
2176	bool X86_64ABIInfo::IsIllegalVectorType(QualType Ty) const {
2177	if (const VectorType *VecTy = Ty->getAs<VectorType>()) {
2178	uint64_t Size = getContext().getTypeSize(VecTy);
2179	unsigned LargestVector = getNativeVectorSizeForAVXABI(AVXLevel);
2180	if (Size <= 64 || Size > LargestVector)
2181	return true;
2182	QualType EltTy = VecTy->getElementType();
2183	if (passInt128VectorsInMem() &&
2184	(EltTy->isSpecificBuiltinType(K: BuiltinType::Int128) ||
2185	EltTy->isSpecificBuiltinType(K: BuiltinType::UInt128)))
2186	return true;
2187	}
2188
2189	return false;
2190	}
2191
2192	ABIArgInfo X86_64ABIInfo::getIndirectResult(QualType Ty,
2193	unsigned freeIntRegs) const {
2194	// If this is a scalar LLVM value then assume LLVM will pass it in the right
2195	// place naturally.
2196	//
2197	// This assumption is optimistic, as there could be free registers available
2198	// when we need to pass this argument in memory, and LLVM could try to pass
2199	// the argument in the free register. This does not seem to happen currently,
2200	// but this code would be much safer if we could mark the argument with
2201	// 'onstack'. See PR12193.
2202	if (!isAggregateTypeForABI(T: Ty) && !IsIllegalVectorType(Ty) &&
2203	!Ty->isBitIntType()) {
2204	// Treat an enum type as its underlying type.
2205	if (const EnumType *EnumTy = Ty->getAs<EnumType>())
2206	Ty = EnumTy->getDecl()->getIntegerType();
2207
2208	return (isPromotableIntegerTypeForABI(Ty) ? ABIArgInfo::getExtend(Ty)
2209	: ABIArgInfo::getDirect());
2210	}
2211
2212	if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(T: Ty, CXXABI&: getCXXABI()))
2213	return getNaturalAlignIndirect(Ty, ByVal: RAA == CGCXXABI::RAA_DirectInMemory);
2214
2215	// Compute the byval alignment. We specify the alignment of the byval in all
2216	// cases so that the mid-level optimizer knows the alignment of the byval.
2217	unsigned Align = std::max(a: getContext().getTypeAlign(T: Ty) / 8, b: 8U);
2218
2219	// Attempt to avoid passing indirect results using byval when possible. This
2220	// is important for good codegen.
2221	//
2222	// We do this by coercing the value into a scalar type which the backend can
2223	// handle naturally (i.e., without using byval).
2224	//
2225	// For simplicity, we currently only do this when we have exhausted all of the
2226	// free integer registers. Doing this when there are free integer registers
2227	// would require more care, as we would have to ensure that the coerced value
2228	// did not claim the unused register. That would require either reording the
2229	// arguments to the function (so that any subsequent inreg values came first),
2230	// or only doing this optimization when there were no following arguments that
2231	// might be inreg.
2232	//
2233	// We currently expect it to be rare (particularly in well written code) for
2234	// arguments to be passed on the stack when there are still free integer
2235	// registers available (this would typically imply large structs being passed
2236	// by value), so this seems like a fair tradeoff for now.
2237	//
2238	// We can revisit this if the backend grows support for 'onstack' parameter
2239	// attributes. See PR12193.
2240	if (freeIntRegs == 0) {
2241	uint64_t Size = getContext().getTypeSize(T: Ty);
2242
2243	// If this type fits in an eightbyte, coerce it into the matching integral
2244	// type, which will end up on the stack (with alignment 8).
2245	if (Align == 8 && Size <= 64)
2246	return ABIArgInfo::getDirect(T: llvm::IntegerType::get(C&: getVMContext(),
2247	NumBits: Size));
2248	}
2249
2250	return ABIArgInfo::getIndirect(Alignment: CharUnits::fromQuantity(Quantity: Align));
2251	}
2252
2253	/// The ABI specifies that a value should be passed in a full vector XMM/YMM
2254	/// register. Pick an LLVM IR type that will be passed as a vector register.
2255	llvm::Type *X86_64ABIInfo::GetByteVectorType(QualType Ty) const {
2256	// Wrapper structs/arrays that only contain vectors are passed just like
2257	// vectors; strip them off if present.
2258	if (const Type *InnerTy = isSingleElementStruct(T: Ty, Context&: getContext()))
2259	Ty = QualType(InnerTy, 0);
2260
2261	llvm::Type *IRType = CGT.ConvertType(T: Ty);
2262	if (isa<llvm::VectorType>(Val: IRType)) {
2263	// Don't pass vXi128 vectors in their native type, the backend can't
2264	// legalize them.
2265	if (passInt128VectorsInMem() &&
2266	cast<llvm::VectorType>(Val: IRType)->getElementType()->isIntegerTy(Bitwidth: 128)) {
2267	// Use a vXi64 vector.
2268	uint64_t Size = getContext().getTypeSize(T: Ty);
2269	return llvm::FixedVectorType::get(ElementType: llvm::Type::getInt64Ty(C&: getVMContext()),
2270	NumElts: Size / 64);
2271	}
2272
2273	return IRType;
2274	}
2275
2276	if (IRType->getTypeID() == llvm::Type::FP128TyID)
2277	return IRType;
2278
2279	// We couldn't find the preferred IR vector type for 'Ty'.
2280	uint64_t Size = getContext().getTypeSize(T: Ty);
2281	assert((Size == 128 || Size == 256 || Size == 512) && "Invalid type found!");
2282
2283
2284	// Return a LLVM IR vector type based on the size of 'Ty'.
2285	return llvm::FixedVectorType::get(ElementType: llvm::Type::getDoubleTy(C&: getVMContext()),
2286	NumElts: Size / 64);
2287	}
2288
2289	/// BitsContainNoUserData - Return true if the specified [start,end) bit range
2290	/// is known to either be off the end of the specified type or being in
2291	/// alignment padding. The user type specified is known to be at most 128 bits
2292	/// in size, and have passed through X86_64ABIInfo::classify with a successful
2293	/// classification that put one of the two halves in the INTEGER class.
2294	///
2295	/// It is conservatively correct to return false.
2296	static bool BitsContainNoUserData(QualType Ty, unsigned StartBit,
2297	unsigned EndBit, ASTContext &Context) {
2298	// If the bytes being queried are off the end of the type, there is no user
2299	// data hiding here. This handles analysis of builtins, vectors and other
2300	// types that don't contain interesting padding.
2301	unsigned TySize = (unsigned)Context.getTypeSize(T: Ty);
2302	if (TySize <= StartBit)
2303	return true;
2304
2305	if (const ConstantArrayType *AT = Context.getAsConstantArrayType(T: Ty)) {
2306	unsigned EltSize = (unsigned)Context.getTypeSize(AT->getElementType());
2307	unsigned NumElts = (unsigned)AT->getZExtSize();
2308
2309	// Check each element to see if the element overlaps with the queried range.
2310	for (unsigned i = 0; i != NumElts; ++i) {
2311	// If the element is after the span we care about, then we're done..
2312	unsigned EltOffset = i*EltSize;
2313	if (EltOffset >= EndBit) break;
2314
2315	unsigned EltStart = EltOffset < StartBit ? StartBit-EltOffset :0;
2316	if (!BitsContainNoUserData(AT->getElementType(), EltStart,
2317	EndBit-EltOffset, Context))
2318	return false;
2319	}
2320	// If it overlaps no elements, then it is safe to process as padding.
2321	return true;
2322	}
2323
2324	if (const RecordType *RT = Ty->getAs<RecordType>()) {
2325	const RecordDecl *RD = RT->getDecl();
2326	const ASTRecordLayout &Layout = Context.getASTRecordLayout(D: RD);
2327
2328	// If this is a C++ record, check the bases first.
2329	if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(Val: RD)) {
2330	for (const auto &I : CXXRD->bases()) {
2331	assert(!I.isVirtual() && !I.getType()->isDependentType() &&
2332	"Unexpected base class!");
2333	const auto *Base =
2334	cast<CXXRecordDecl>(Val: I.getType()->castAs<RecordType>()->getDecl());
2335
2336	// If the base is after the span we care about, ignore it.
2337	unsigned BaseOffset = Context.toBits(CharSize: Layout.getBaseClassOffset(Base));
2338	if (BaseOffset >= EndBit) continue;
2339
2340	unsigned BaseStart = BaseOffset < StartBit ? StartBit-BaseOffset :0;
2341	if (!BitsContainNoUserData(Ty: I.getType(), StartBit: BaseStart,
2342	EndBit: EndBit-BaseOffset, Context))
2343	return false;
2344	}
2345	}
2346
2347	// Verify that no field has data that overlaps the region of interest. Yes
2348	// this could be sped up a lot by being smarter about queried fields,
2349	// however we're only looking at structs up to 16 bytes, so we don't care
2350	// much.
2351	unsigned idx = 0;
2352	for (RecordDecl::field_iterator i = RD->field_begin(), e = RD->field_end();
2353	i != e; ++i, ++idx) {
2354	unsigned FieldOffset = (unsigned)Layout.getFieldOffset(FieldNo: idx);
2355
2356	// If we found a field after the region we care about, then we're done.
2357	if (FieldOffset >= EndBit) break;
2358
2359	unsigned FieldStart = FieldOffset < StartBit ? StartBit-FieldOffset :0;
2360	if (!BitsContainNoUserData(i->getType(), FieldStart, EndBit-FieldOffset,
2361	Context))
2362	return false;
2363	}
2364
2365	// If nothing in this record overlapped the area of interest, then we're
2366	// clean.
2367	return true;
2368	}
2369
2370	return false;
2371	}
2372
2373	/// getFPTypeAtOffset - Return a floating point type at the specified offset.
2374	static llvm::Type *getFPTypeAtOffset(llvm::Type *IRType, unsigned IROffset,
2375	const llvm::DataLayout &TD) {
2376	if (IROffset == 0 && IRType->isFloatingPointTy())
2377	return IRType;
2378
2379	// If this is a struct, recurse into the field at the specified offset.
2380	if (llvm::StructType *STy = dyn_cast<llvm::StructType>(Val: IRType)) {
2381	if (!STy->getNumContainedTypes())
2382	return nullptr;
2383
2384	const llvm::StructLayout *SL = TD.getStructLayout(Ty: STy);
2385	unsigned Elt = SL->getElementContainingOffset(FixedOffset: IROffset);
2386	IROffset -= SL->getElementOffset(Idx: Elt);
2387	return getFPTypeAtOffset(IRType: STy->getElementType(N: Elt), IROffset, TD);
2388	}
2389
2390	// If this is an array, recurse into the field at the specified offset.
2391	if (llvm::ArrayType *ATy = dyn_cast<llvm::ArrayType>(Val: IRType)) {
2392	llvm::Type *EltTy = ATy->getElementType();
2393	unsigned EltSize = TD.getTypeAllocSize(Ty: EltTy);
2394	IROffset -= IROffset / EltSize * EltSize;
2395	return getFPTypeAtOffset(IRType: EltTy, IROffset, TD);
2396	}
2397
2398	return nullptr;
2399	}
2400
2401	/// GetSSETypeAtOffset - Return a type that will be passed by the backend in the
2402	/// low 8 bytes of an XMM register, corresponding to the SSE class.
2403	llvm::Type *X86_64ABIInfo::
2404	GetSSETypeAtOffset(llvm::Type *IRType, unsigned IROffset,
2405	QualType SourceTy, unsigned SourceOffset) const {
2406	const llvm::DataLayout &TD = getDataLayout();
2407	unsigned SourceSize =
2408	(unsigned)getContext().getTypeSize(T: SourceTy) / 8 - SourceOffset;
2409	llvm::Type *T0 = getFPTypeAtOffset(IRType, IROffset, TD);
2410	if (!T0 || T0->isDoubleTy())
2411	return llvm::Type::getDoubleTy(C&: getVMContext());
2412
2413	// Get the adjacent FP type.
2414	llvm::Type *T1 = nullptr;
2415	unsigned T0Size = TD.getTypeAllocSize(Ty: T0);
2416	if (SourceSize > T0Size)
2417	T1 = getFPTypeAtOffset(IRType, IROffset: IROffset + T0Size, TD);
2418	if (T1 == nullptr) {
2419	// Check if IRType is a half/bfloat + float. float type will be in IROffset+4 due
2420	// to its alignment.
2421	if (T0->is16bitFPTy() && SourceSize > 4)
2422	T1 = getFPTypeAtOffset(IRType, IROffset: IROffset + 4, TD);
2423	// If we can't get a second FP type, return a simple half or float.
2424	// avx512fp16-abi.c:pr51813_2 shows it works to return float for
2425	// {float, i8} too.
2426	if (T1 == nullptr)
2427	return T0;
2428	}
2429
2430	if (T0->isFloatTy() && T1->isFloatTy())
2431	return llvm::FixedVectorType::get(ElementType: T0, NumElts: 2);
2432
2433	if (T0->is16bitFPTy() && T1->is16bitFPTy()) {
2434	llvm::Type *T2 = nullptr;
2435	if (SourceSize > 4)
2436	T2 = getFPTypeAtOffset(IRType, IROffset: IROffset + 4, TD);
2437	if (T2 == nullptr)
2438	return llvm::FixedVectorType::get(ElementType: T0, NumElts: 2);
2439	return llvm::FixedVectorType::get(ElementType: T0, NumElts: 4);
2440	}
2441
2442	if (T0->is16bitFPTy() || T1->is16bitFPTy())
2443	return llvm::FixedVectorType::get(ElementType: llvm::Type::getHalfTy(C&: getVMContext()), NumElts: 4);
2444
2445	return llvm::Type::getDoubleTy(C&: getVMContext());
2446	}
2447
2448
2449	/// GetINTEGERTypeAtOffset - The ABI specifies that a value should be passed in
2450	/// an 8-byte GPR. This means that we either have a scalar or we are talking
2451	/// about the high or low part of an up-to-16-byte struct. This routine picks
2452	/// the best LLVM IR type to represent this, which may be i64 or may be anything
2453	/// else that the backend will pass in a GPR that works better (e.g. i8, %foo*,
2454	/// etc).
2455	///
2456	/// PrefType is an LLVM IR type that corresponds to (part of) the IR type for
2457	/// the source type. IROffset is an offset in bytes into the LLVM IR type that
2458	/// the 8-byte value references. PrefType may be null.
2459	///
2460	/// SourceTy is the source-level type for the entire argument. SourceOffset is
2461	/// an offset into this that we're processing (which is always either 0 or 8).
2462	///
2463	llvm::Type *X86_64ABIInfo::
2464	GetINTEGERTypeAtOffset(llvm::Type *IRType, unsigned IROffset,
2465	QualType SourceTy, unsigned SourceOffset) const {
2466	// If we're dealing with an un-offset LLVM IR type, then it means that we're
2467	// returning an 8-byte unit starting with it. See if we can safely use it.
2468	if (IROffset == 0) {
2469	// Pointers and int64's always fill the 8-byte unit.
2470	if ((isa<llvm::PointerType>(Val: IRType) && Has64BitPointers) ||
2471	IRType->isIntegerTy(Bitwidth: 64))
2472	return IRType;
2473
2474	// If we have a 1/2/4-byte integer, we can use it only if the rest of the
2475	// goodness in the source type is just tail padding. This is allowed to
2476	// kick in for struct {double,int} on the int, but not on
2477	// struct{double,int,int} because we wouldn't return the second int. We
2478	// have to do this analysis on the source type because we can't depend on
2479	// unions being lowered a specific way etc.
2480	if (IRType->isIntegerTy(Bitwidth: 8) || IRType->isIntegerTy(Bitwidth: 16) ||
2481	IRType->isIntegerTy(Bitwidth: 32) ||
2482	(isa<llvm::PointerType>(Val: IRType) && !Has64BitPointers)) {
2483	unsigned BitWidth = isa<llvm::PointerType>(Val: IRType) ? 32 :
2484	cast<llvm::IntegerType>(Val: IRType)->getBitWidth();
2485
2486	if (BitsContainNoUserData(Ty: SourceTy, StartBit: SourceOffset*8+BitWidth,
2487	EndBit: SourceOffset*8+64, Context&: getContext()))
2488	return IRType;
2489	}
2490	}
2491
2492	if (llvm::StructType *STy = dyn_cast<llvm::StructType>(Val: IRType)) {
2493	// If this is a struct, recurse into the field at the specified offset.
2494	const llvm::StructLayout *SL = getDataLayout().getStructLayout(Ty: STy);
2495	if (IROffset < SL->getSizeInBytes()) {
2496	unsigned FieldIdx = SL->getElementContainingOffset(FixedOffset: IROffset);
2497	IROffset -= SL->getElementOffset(Idx: FieldIdx);
2498
2499	return GetINTEGERTypeAtOffset(IRType: STy->getElementType(N: FieldIdx), IROffset,
2500	SourceTy, SourceOffset);
2501	}
2502	}
2503
2504	if (llvm::ArrayType *ATy = dyn_cast<llvm::ArrayType>(Val: IRType)) {
2505	llvm::Type *EltTy = ATy->getElementType();
2506	unsigned EltSize = getDataLayout().getTypeAllocSize(Ty: EltTy);
2507	unsigned EltOffset = IROffset/EltSize*EltSize;
2508	return GetINTEGERTypeAtOffset(IRType: EltTy, IROffset: IROffset-EltOffset, SourceTy,
2509	SourceOffset);
2510	}
2511
2512	// Okay, we don't have any better idea of what to pass, so we pass this in an
2513	// integer register that isn't too big to fit the rest of the struct.
2514	unsigned TySizeInBytes =
2515	(unsigned)getContext().getTypeSizeInChars(T: SourceTy).getQuantity();
2516
2517	assert(TySizeInBytes != SourceOffset && "Empty field?");
2518
2519	// It is always safe to classify this as an integer type up to i64 that
2520	// isn't larger than the structure.
2521	return llvm::IntegerType::get(C&: getVMContext(),
2522	NumBits: std::min(a: TySizeInBytes-SourceOffset, b: 8U)*8);
2523	}
2524
2525
2526	/// GetX86_64ByValArgumentPair - Given a high and low type that can ideally
2527	/// be used as elements of a two register pair to pass or return, return a
2528	/// first class aggregate to represent them. For example, if the low part of
2529	/// a by-value argument should be passed as i32* and the high part as float,
2530	/// return {i32*, float}.
2531	static llvm::Type *
2532	GetX86_64ByValArgumentPair(llvm::Type *Lo, llvm::Type *Hi,
2533	const llvm::DataLayout &TD) {
2534	// In order to correctly satisfy the ABI, we need to the high part to start
2535	// at offset 8. If the high and low parts we inferred are both 4-byte types
2536	// (e.g. i32 and i32) then the resultant struct type ({i32,i32}) won't have
2537	// the second element at offset 8. Check for this:
2538	unsigned LoSize = (unsigned)TD.getTypeAllocSize(Ty: Lo);
2539	llvm::Align HiAlign = TD.getABITypeAlign(Ty: Hi);
2540	unsigned HiStart = llvm::alignTo(Size: LoSize, A: HiAlign);
2541	assert(HiStart != 0 && HiStart <= 8 && "Invalid x86-64 argument pair!");
2542
2543	// To handle this, we have to increase the size of the low part so that the
2544	// second element will start at an 8 byte offset. We can't increase the size
2545	// of the second element because it might make us access off the end of the
2546	// struct.
2547	if (HiStart != 8) {
2548	// There are usually two sorts of types the ABI generation code can produce
2549	// for the low part of a pair that aren't 8 bytes in size: half, float or
2550	// i8/i16/i32. This can also include pointers when they are 32-bit (X32 and
2551	// NaCl).
2552	// Promote these to a larger type.
2553	if (Lo->isHalfTy() || Lo->isFloatTy())
2554	Lo = llvm::Type::getDoubleTy(C&: Lo->getContext());
2555	else {
2556	assert((Lo->isIntegerTy() || Lo->isPointerTy())
2557	&& "Invalid/unknown lo type");
2558	Lo = llvm::Type::getInt64Ty(C&: Lo->getContext());
2559	}
2560	}
2561
2562	llvm::StructType *Result = llvm::StructType::get(elt1: Lo, elts: Hi);
2563
2564	// Verify that the second element is at an 8-byte offset.
2565	assert(TD.getStructLayout(Result)->getElementOffset(1) == 8 &&
2566	"Invalid x86-64 argument pair!");
2567	return Result;
2568	}
2569
2570	ABIArgInfo X86_64ABIInfo::
2571	classifyReturnType(QualType RetTy) const {
2572	// AMD64-ABI 3.2.3p4: Rule 1. Classify the return type with the
2573	// classification algorithm.
2574	X86_64ABIInfo::Class Lo, Hi;
2575	classify(Ty: RetTy, OffsetBase: 0, Lo, Hi, /*isNamedArg*/ true);
2576
2577	// Check some invariants.
2578	assert((Hi != Memory || Lo == Memory) && "Invalid memory classification.");
2579	assert((Hi != SSEUp || Lo == SSE) && "Invalid SSEUp classification.");
2580
2581	llvm::Type *ResType = nullptr;
2582	switch (Lo) {
2583	case NoClass:
2584	if (Hi == NoClass)
2585	return ABIArgInfo::getIgnore();
2586	// If the low part is just padding, it takes no register, leave ResType
2587	// null.
2588	assert((Hi == SSE || Hi == Integer || Hi == X87Up) &&
2589	"Unknown missing lo part");
2590	break;
2591
2592	case SSEUp:
2593	case X87Up:
2594	llvm_unreachable("Invalid classification for lo word.");
2595
2596	// AMD64-ABI 3.2.3p4: Rule 2. Types of class memory are returned via
2597	// hidden argument.
2598	case Memory:
2599	return getIndirectReturnResult(Ty: RetTy);
2600
2601	// AMD64-ABI 3.2.3p4: Rule 3. If the class is INTEGER, the next
2602	// available register of the sequence %rax, %rdx is used.
2603	case Integer:
2604	ResType = GetINTEGERTypeAtOffset(IRType: CGT.ConvertType(T: RetTy), IROffset: 0, SourceTy: RetTy, SourceOffset: 0);
2605
2606	// If we have a sign or zero extended integer, make sure to return Extend
2607	// so that the parameter gets the right LLVM IR attributes.
2608	if (Hi == NoClass && isa<llvm::IntegerType>(Val: ResType)) {
2609	// Treat an enum type as its underlying type.
2610	if (const EnumType *EnumTy = RetTy->getAs<EnumType>())
2611	RetTy = EnumTy->getDecl()->getIntegerType();
2612
2613	if (RetTy->isIntegralOrEnumerationType() &&
2614	isPromotableIntegerTypeForABI(Ty: RetTy))
2615	return ABIArgInfo::getExtend(Ty: RetTy);
2616	}
2617	break;
2618
2619	// AMD64-ABI 3.2.3p4: Rule 4. If the class is SSE, the next
2620	// available SSE register of the sequence %xmm0, %xmm1 is used.
2621	case SSE:
2622	ResType = GetSSETypeAtOffset(IRType: CGT.ConvertType(T: RetTy), IROffset: 0, SourceTy: RetTy, SourceOffset: 0);
2623	break;
2624
2625	// AMD64-ABI 3.2.3p4: Rule 6. If the class is X87, the value is
2626	// returned on the X87 stack in %st0 as 80-bit x87 number.
2627	case X87:
2628	ResType = llvm::Type::getX86_FP80Ty(C&: getVMContext());
2629	break;
2630
2631	// AMD64-ABI 3.2.3p4: Rule 8. If the class is COMPLEX_X87, the real
2632	// part of the value is returned in %st0 and the imaginary part in
2633	// %st1.
2634	case ComplexX87:
2635	assert(Hi == ComplexX87 && "Unexpected ComplexX87 classification.");
2636	ResType = llvm::StructType::get(elt1: llvm::Type::getX86_FP80Ty(C&: getVMContext()),
2637	elts: llvm::Type::getX86_FP80Ty(C&: getVMContext()));
2638	break;
2639	}
2640
2641	llvm::Type *HighPart = nullptr;
2642	switch (Hi) {
2643	// Memory was handled previously and X87 should
2644	// never occur as a hi class.
2645	case Memory:
2646	case X87:
2647	llvm_unreachable("Invalid classification for hi word.");
2648
2649	case ComplexX87: // Previously handled.
2650	case NoClass:
2651	break;
2652
2653	case Integer:
2654	HighPart = GetINTEGERTypeAtOffset(IRType: CGT.ConvertType(T: RetTy), IROffset: 8, SourceTy: RetTy, SourceOffset: 8);
2655	if (Lo == NoClass) // Return HighPart at offset 8 in memory.
2656	return ABIArgInfo::getDirect(T: HighPart, Offset: 8);
2657	break;
2658	case SSE:
2659	HighPart = GetSSETypeAtOffset(IRType: CGT.ConvertType(T: RetTy), IROffset: 8, SourceTy: RetTy, SourceOffset: 8);
2660	if (Lo == NoClass) // Return HighPart at offset 8 in memory.
2661	return ABIArgInfo::getDirect(T: HighPart, Offset: 8);
2662	break;
2663
2664	// AMD64-ABI 3.2.3p4: Rule 5. If the class is SSEUP, the eightbyte
2665	// is passed in the next available eightbyte chunk if the last used
2666	// vector register.
2667	//
2668	// SSEUP should always be preceded by SSE, just widen.
2669	case SSEUp:
2670	assert(Lo == SSE && "Unexpected SSEUp classification.");
2671	ResType = GetByteVectorType(Ty: RetTy);
2672	break;
2673
2674	// AMD64-ABI 3.2.3p4: Rule 7. If the class is X87UP, the value is
2675	// returned together with the previous X87 value in %st0.
2676	case X87Up:
2677	// If X87Up is preceded by X87, we don't need to do
2678	// anything. However, in some cases with unions it may not be
2679	// preceded by X87. In such situations we follow gcc and pass the
2680	// extra bits in an SSE reg.
2681	if (Lo != X87) {
2682	HighPart = GetSSETypeAtOffset(IRType: CGT.ConvertType(T: RetTy), IROffset: 8, SourceTy: RetTy, SourceOffset: 8);
2683	if (Lo == NoClass) // Return HighPart at offset 8 in memory.
2684	return ABIArgInfo::getDirect(T: HighPart, Offset: 8);
2685	}
2686	break;
2687	}
2688
2689	// If a high part was specified, merge it together with the low part. It is
2690	// known to pass in the high eightbyte of the result. We do this by forming a
2691	// first class struct aggregate with the high and low part: {low, high}
2692	if (HighPart)
2693	ResType = GetX86_64ByValArgumentPair(Lo: ResType, Hi: HighPart, TD: getDataLayout());
2694
2695	return ABIArgInfo::getDirect(T: ResType);
2696	}
2697
2698	ABIArgInfo
2699	X86_64ABIInfo::classifyArgumentType(QualType Ty, unsigned freeIntRegs,
2700	unsigned &neededInt, unsigned &neededSSE,
2701	bool isNamedArg, bool IsRegCall) const {
2702	Ty = useFirstFieldIfTransparentUnion(Ty);
2703
2704	X86_64ABIInfo::Class Lo, Hi;
2705	classify(Ty, OffsetBase: 0, Lo, Hi, isNamedArg, IsRegCall);
2706
2707	// Check some invariants.
2708	// FIXME: Enforce these by construction.
2709	assert((Hi != Memory || Lo == Memory) && "Invalid memory classification.");
2710	assert((Hi != SSEUp || Lo == SSE) && "Invalid SSEUp classification.");
2711
2712	neededInt = 0;
2713	neededSSE = 0;
2714	llvm::Type *ResType = nullptr;
2715	switch (Lo) {
2716	case NoClass:
2717	if (Hi == NoClass)
2718	return ABIArgInfo::getIgnore();
2719	// If the low part is just padding, it takes no register, leave ResType
2720	// null.
2721	assert((Hi == SSE || Hi == Integer || Hi == X87Up) &&
2722	"Unknown missing lo part");
2723	break;
2724
2725	// AMD64-ABI 3.2.3p3: Rule 1. If the class is MEMORY, pass the argument
2726	// on the stack.
2727	case Memory:
2728
2729	// AMD64-ABI 3.2.3p3: Rule 5. If the class is X87, X87UP or
2730	// COMPLEX_X87, it is passed in memory.
2731	case X87:
2732	case ComplexX87:
2733	if (getRecordArgABI(T: Ty, CXXABI&: getCXXABI()) == CGCXXABI::RAA_Indirect)
2734	++neededInt;
2735	return getIndirectResult(Ty, freeIntRegs);
2736
2737	case SSEUp:
2738	case X87Up:
2739	llvm_unreachable("Invalid classification for lo word.");
2740
2741	// AMD64-ABI 3.2.3p3: Rule 2. If the class is INTEGER, the next
2742	// available register of the sequence %rdi, %rsi, %rdx, %rcx, %r8
2743	// and %r9 is used.
2744	case Integer:
2745	++neededInt;
2746
2747	// Pick an 8-byte type based on the preferred type.
2748	ResType = GetINTEGERTypeAtOffset(IRType: CGT.ConvertType(T: Ty), IROffset: 0, SourceTy: Ty, SourceOffset: 0);
2749
2750	// If we have a sign or zero extended integer, make sure to return Extend
2751	// so that the parameter gets the right LLVM IR attributes.
2752	if (Hi == NoClass && isa<llvm::IntegerType>(Val: ResType)) {
2753	// Treat an enum type as its underlying type.
2754	if (const EnumType *EnumTy = Ty->getAs<EnumType>())
2755	Ty = EnumTy->getDecl()->getIntegerType();
2756
2757	if (Ty->isIntegralOrEnumerationType() &&
2758	isPromotableIntegerTypeForABI(Ty))
2759	return ABIArgInfo::getExtend(Ty);
2760	}
2761
2762	break;
2763
2764	// AMD64-ABI 3.2.3p3: Rule 3. If the class is SSE, the next
2765	// available SSE register is used, the registers are taken in the
2766	// order from %xmm0 to %xmm7.
2767	case SSE: {
2768	llvm::Type *IRType = CGT.ConvertType(T: Ty);
2769	ResType = GetSSETypeAtOffset(IRType, IROffset: 0, SourceTy: Ty, SourceOffset: 0);
2770	++neededSSE;
2771	break;
2772	}
2773	}
2774
2775	llvm::Type *HighPart = nullptr;
2776	switch (Hi) {
2777	// Memory was handled previously, ComplexX87 and X87 should
2778	// never occur as hi classes, and X87Up must be preceded by X87,
2779	// which is passed in memory.
2780	case Memory:
2781	case X87:
2782	case ComplexX87:
2783	llvm_unreachable("Invalid classification for hi word.");
2784
2785	case NoClass: break;
2786
2787	case Integer:
2788	++neededInt;
2789	// Pick an 8-byte type based on the preferred type.
2790	HighPart = GetINTEGERTypeAtOffset(IRType: CGT.ConvertType(T: Ty), IROffset: 8, SourceTy: Ty, SourceOffset: 8);
2791
2792	if (Lo == NoClass) // Pass HighPart at offset 8 in memory.
2793	return ABIArgInfo::getDirect(T: HighPart, Offset: 8);
2794	break;
2795
2796	// X87Up generally doesn't occur here (long double is passed in
2797	// memory), except in situations involving unions.
2798	case X87Up:
2799	case SSE:
2800	++neededSSE;
2801	HighPart = GetSSETypeAtOffset(IRType: CGT.ConvertType(T: Ty), IROffset: 8, SourceTy: Ty, SourceOffset: 8);
2802
2803	if (Lo == NoClass) // Pass HighPart at offset 8 in memory.
2804	return ABIArgInfo::getDirect(T: HighPart, Offset: 8);
2805	break;
2806
2807	// AMD64-ABI 3.2.3p3: Rule 4. If the class is SSEUP, the
2808	// eightbyte is passed in the upper half of the last used SSE
2809	// register. This only happens when 128-bit vectors are passed.
2810	case SSEUp:
2811	assert(Lo == SSE && "Unexpected SSEUp classification");
2812	ResType = GetByteVectorType(Ty);
2813	break;
2814	}
2815
2816	// If a high part was specified, merge it together with the low part. It is
2817	// known to pass in the high eightbyte of the result. We do this by forming a
2818	// first class struct aggregate with the high and low part: {low, high}
2819	if (HighPart)
2820	ResType = GetX86_64ByValArgumentPair(Lo: ResType, Hi: HighPart, TD: getDataLayout());
2821
2822	return ABIArgInfo::getDirect(T: ResType);
2823	}
2824
2825	ABIArgInfo
2826	X86_64ABIInfo::classifyRegCallStructTypeImpl(QualType Ty, unsigned &NeededInt,
2827	unsigned &NeededSSE,
2828	unsigned &MaxVectorWidth) const {
2829	auto RT = Ty->getAs<RecordType>();
2830	assert(RT && "classifyRegCallStructType only valid with struct types");
2831
2832	if (RT->getDecl()->hasFlexibleArrayMember())
2833	return getIndirectReturnResult(Ty);
2834
2835	// Sum up bases
2836	if (auto CXXRD = dyn_cast<CXXRecordDecl>(Val: RT->getDecl())) {
2837	if (CXXRD->isDynamicClass()) {
2838	NeededInt = NeededSSE = 0;
2839	return getIndirectReturnResult(Ty);
2840	}
2841
2842	for (const auto &I : CXXRD->bases())
2843	if (classifyRegCallStructTypeImpl(Ty: I.getType(), NeededInt, NeededSSE,
2844	MaxVectorWidth)
2845	.isIndirect()) {
2846	NeededInt = NeededSSE = 0;
2847	return getIndirectReturnResult(Ty);
2848	}
2849	}
2850
2851	// Sum up members
2852	for (const auto *FD : RT->getDecl()->fields()) {
2853	QualType MTy = FD->getType();
2854	if (MTy->isRecordType() && !MTy->isUnionType()) {
2855	if (classifyRegCallStructTypeImpl(Ty: MTy, NeededInt, NeededSSE,
2856	MaxVectorWidth)
2857	.isIndirect()) {
2858	NeededInt = NeededSSE = 0;
2859	return getIndirectReturnResult(Ty);
2860	}
2861	} else {
2862	unsigned LocalNeededInt, LocalNeededSSE;
2863	if (classifyArgumentType(Ty: MTy, UINT_MAX, neededInt&: LocalNeededInt, neededSSE&: LocalNeededSSE,
2864	isNamedArg: true, IsRegCall: true)
2865	.isIndirect()) {
2866	NeededInt = NeededSSE = 0;
2867	return getIndirectReturnResult(Ty);
2868	}
2869	if (const auto *AT = getContext().getAsConstantArrayType(MTy))
2870	MTy = AT->getElementType();
2871	if (const auto *VT = MTy->getAs<VectorType>())
2872	if (getContext().getTypeSize(VT) > MaxVectorWidth)
2873	MaxVectorWidth = getContext().getTypeSize(VT);
2874	NeededInt += LocalNeededInt;
2875	NeededSSE += LocalNeededSSE;
2876	}
2877	}
2878
2879	return ABIArgInfo::getDirect();
2880	}
2881
2882	ABIArgInfo
2883	X86_64ABIInfo::classifyRegCallStructType(QualType Ty, unsigned &NeededInt,
2884	unsigned &NeededSSE,
2885	unsigned &MaxVectorWidth) const {
2886
2887	NeededInt = 0;
2888	NeededSSE = 0;
2889	MaxVectorWidth = 0;
2890
2891	return classifyRegCallStructTypeImpl(Ty, NeededInt, NeededSSE,
2892	MaxVectorWidth);
2893	}
2894
2895	void X86_64ABIInfo::computeInfo(CGFunctionInfo &FI) const {
2896
2897	const unsigned CallingConv = FI.getCallingConvention();
2898	// It is possible to force Win64 calling convention on any x86_64 target by
2899	// using __attribute__((ms_abi)). In such case to correctly emit Win64
2900	// compatible code delegate this call to WinX86_64ABIInfo::computeInfo.
2901	if (CallingConv == llvm::CallingConv::Win64) {
2902	WinX86_64ABIInfo Win64ABIInfo(CGT, AVXLevel);
2903	Win64ABIInfo.computeInfo(FI);
2904	return;
2905	}
2906
2907	bool IsRegCall = CallingConv == llvm::CallingConv::X86_RegCall;
2908
2909	// Keep track of the number of assigned registers.
2910	unsigned FreeIntRegs = IsRegCall ? 11 : 6;
2911	unsigned FreeSSERegs = IsRegCall ? 16 : 8;
2912	unsigned NeededInt = 0, NeededSSE = 0, MaxVectorWidth = 0;
2913
2914	if (!::classifyReturnType(CXXABI: getCXXABI(), FI, Info: *this)) {
2915	if (IsRegCall && FI.getReturnType()->getTypePtr()->isRecordType() &&
2916	!FI.getReturnType()->getTypePtr()->isUnionType()) {
2917	FI.getReturnInfo() = classifyRegCallStructType(
2918	Ty: FI.getReturnType(), NeededInt, NeededSSE, MaxVectorWidth);
2919	if (FreeIntRegs >= NeededInt && FreeSSERegs >= NeededSSE) {
2920	FreeIntRegs -= NeededInt;
2921	FreeSSERegs -= NeededSSE;
2922	} else {
2923	FI.getReturnInfo() = getIndirectReturnResult(Ty: FI.getReturnType());
2924	}
2925	} else if (IsRegCall && FI.getReturnType()->getAs<ComplexType>() &&
2926	getContext().getCanonicalType(FI.getReturnType()
2927	->getAs<ComplexType>()
2928	->getElementType()) ==
2929	getContext().LongDoubleTy)
2930	// Complex Long Double Type is passed in Memory when Regcall
2931	// calling convention is used.
2932	FI.getReturnInfo() = getIndirectReturnResult(Ty: FI.getReturnType());
2933	else
2934	FI.getReturnInfo() = classifyReturnType(RetTy: FI.getReturnType());
2935	}
2936
2937	// If the return value is indirect, then the hidden argument is consuming one
2938	// integer register.
2939	if (FI.getReturnInfo().isIndirect())
2940	--FreeIntRegs;
2941	else if (NeededSSE && MaxVectorWidth > 0)
2942	FI.setMaxVectorWidth(MaxVectorWidth);
2943
2944	// The chain argument effectively gives us another free register.
2945	if (FI.isChainCall())
2946	++FreeIntRegs;
2947
2948	unsigned NumRequiredArgs = FI.getNumRequiredArgs();
2949	// AMD64-ABI 3.2.3p3: Once arguments are classified, the registers
2950	// get assigned (in left-to-right order) for passing as follows...
2951	unsigned ArgNo = 0;
2952	for (CGFunctionInfo::arg_iterator it = FI.arg_begin(), ie = FI.arg_end();
2953	it != ie; ++it, ++ArgNo) {
2954	bool IsNamedArg = ArgNo < NumRequiredArgs;
2955
2956	if (IsRegCall && it->type->isStructureOrClassType())
2957	it->info = classifyRegCallStructType(Ty: it->type, NeededInt, NeededSSE,
2958	MaxVectorWidth);
2959	else
2960	it->info = classifyArgumentType(Ty: it->type, freeIntRegs: FreeIntRegs, neededInt&: NeededInt,
2961	neededSSE&: NeededSSE, isNamedArg: IsNamedArg);
2962
2963	// AMD64-ABI 3.2.3p3: If there are no registers available for any
2964	// eightbyte of an argument, the whole argument is passed on the
2965	// stack. If registers have already been assigned for some
2966	// eightbytes of such an argument, the assignments get reverted.
2967	if (FreeIntRegs >= NeededInt && FreeSSERegs >= NeededSSE) {
2968	FreeIntRegs -= NeededInt;
2969	FreeSSERegs -= NeededSSE;
2970	if (MaxVectorWidth > FI.getMaxVectorWidth())
2971	FI.setMaxVectorWidth(MaxVectorWidth);
2972	} else {
2973	it->info = getIndirectResult(Ty: it->type, freeIntRegs: FreeIntRegs);
2974	}
2975	}
2976	}
2977
2978	static Address EmitX86_64VAArgFromMemory(CodeGenFunction &CGF,
2979	Address VAListAddr, QualType Ty) {
2980	Address overflow_arg_area_p =
2981	CGF.Builder.CreateStructGEP(Addr: VAListAddr, Index: 2, Name: "overflow_arg_area_p");
2982	llvm::Value *overflow_arg_area =
2983	CGF.Builder.CreateLoad(Addr: overflow_arg_area_p, Name: "overflow_arg_area");
2984
2985	// AMD64-ABI 3.5.7p5: Step 7. Align l->overflow_arg_area upwards to a 16
2986	// byte boundary if alignment needed by type exceeds 8 byte boundary.
2987	// It isn't stated explicitly in the standard, but in practice we use
2988	// alignment greater than 16 where necessary.
2989	CharUnits Align = CGF.getContext().getTypeAlignInChars(T: Ty);
2990	if (Align > CharUnits::fromQuantity(Quantity: 8)) {
2991	overflow_arg_area = emitRoundPointerUpToAlignment(CGF, Ptr: overflow_arg_area,
2992	Align);
2993	}
2994
2995	// AMD64-ABI 3.5.7p5: Step 8. Fetch type from l->overflow_arg_area.
2996	llvm::Type *LTy = CGF.ConvertTypeForMem(T: Ty);
2997	llvm::Value *Res = overflow_arg_area;
2998
2999	// AMD64-ABI 3.5.7p5: Step 9. Set l->overflow_arg_area to:
3000	// l->overflow_arg_area + sizeof(type).
3001	// AMD64-ABI 3.5.7p5: Step 10. Align l->overflow_arg_area upwards to
3002	// an 8 byte boundary.
3003
3004	uint64_t SizeInBytes = (CGF.getContext().getTypeSize(T: Ty) + 7) / 8;
3005	llvm::Value *Offset =
3006	llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: (SizeInBytes + 7) & ~7);
3007	overflow_arg_area = CGF.Builder.CreateGEP(Ty: CGF.Int8Ty, Ptr: overflow_arg_area,
3008	IdxList: Offset, Name: "overflow_arg_area.next");
3009	CGF.Builder.CreateStore(Val: overflow_arg_area, Addr: overflow_arg_area_p);
3010
3011	// AMD64-ABI 3.5.7p5: Step 11. Return the fetched type.
3012	return Address(Res, LTy, Align);
3013	}
3014
3015	Address X86_64ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
3016	QualType Ty) const {
3017	// Assume that va_list type is correct; should be pointer to LLVM type:
3018	// struct {
3019	// i32 gp_offset;
3020	// i32 fp_offset;
3021	// i8* overflow_arg_area;
3022	// i8* reg_save_area;
3023	// };
3024	unsigned neededInt, neededSSE;
3025
3026	Ty = getContext().getCanonicalType(T: Ty);
3027	ABIArgInfo AI = classifyArgumentType(Ty, freeIntRegs: 0, neededInt, neededSSE,
3028	/*isNamedArg*/false);
3029
3030	// Empty records are ignored for parameter passing purposes.
3031	if (AI.isIgnore())
3032	return CGF.CreateMemTemp(T: Ty);
3033
3034	// AMD64-ABI 3.5.7p5: Step 1. Determine whether type may be passed
3035	// in the registers. If not go to step 7.
3036	if (!neededInt && !neededSSE)
3037	return EmitX86_64VAArgFromMemory(CGF, VAListAddr, Ty);
3038
3039	// AMD64-ABI 3.5.7p5: Step 2. Compute num_gp to hold the number of
3040	// general purpose registers needed to pass type and num_fp to hold
3041	// the number of floating point registers needed.
3042
3043	// AMD64-ABI 3.5.7p5: Step 3. Verify whether arguments fit into
3044	// registers. In the case: l->gp_offset > 48 - num_gp * 8 or
3045	// l->fp_offset > 304 - num_fp * 16 go to step 7.
3046	//
3047	// NOTE: 304 is a typo, there are (6 * 8 + 8 * 16) = 176 bytes of
3048	// register save space).
3049
3050	llvm::Value *InRegs = nullptr;
3051	Address gp_offset_p = Address::invalid(), fp_offset_p = Address::invalid();
3052	llvm::Value *gp_offset = nullptr, *fp_offset = nullptr;
3053	if (neededInt) {
3054	gp_offset_p = CGF.Builder.CreateStructGEP(Addr: VAListAddr, Index: 0, Name: "gp_offset_p");
3055	gp_offset = CGF.Builder.CreateLoad(Addr: gp_offset_p, Name: "gp_offset");
3056	InRegs = llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: 48 - neededInt * 8);
3057	InRegs = CGF.Builder.CreateICmpULE(LHS: gp_offset, RHS: InRegs, Name: "fits_in_gp");
3058	}
3059
3060	if (neededSSE) {
3061	fp_offset_p = CGF.Builder.CreateStructGEP(Addr: VAListAddr, Index: 1, Name: "fp_offset_p");
3062	fp_offset = CGF.Builder.CreateLoad(Addr: fp_offset_p, Name: "fp_offset");
3063	llvm::Value *FitsInFP =
3064	llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: 176 - neededSSE * 16);
3065	FitsInFP = CGF.Builder.CreateICmpULE(LHS: fp_offset, RHS: FitsInFP, Name: "fits_in_fp");
3066	InRegs = InRegs ? CGF.Builder.CreateAnd(LHS: InRegs, RHS: FitsInFP) : FitsInFP;
3067	}
3068
3069	llvm::BasicBlock *InRegBlock = CGF.createBasicBlock(name: "vaarg.in_reg");
3070	llvm::BasicBlock *InMemBlock = CGF.createBasicBlock(name: "vaarg.in_mem");
3071	llvm::BasicBlock *ContBlock = CGF.createBasicBlock(name: "vaarg.end");
3072	CGF.Builder.CreateCondBr(Cond: InRegs, True: InRegBlock, False: InMemBlock);
3073
3074	// Emit code to load the value if it was passed in registers.
3075
3076	CGF.EmitBlock(BB: InRegBlock);
3077
3078	// AMD64-ABI 3.5.7p5: Step 4. Fetch type from l->reg_save_area with
3079	// an offset of l->gp_offset and/or l->fp_offset. This may require
3080	// copying to a temporary location in case the parameter is passed
3081	// in different register classes or requires an alignment greater
3082	// than 8 for general purpose registers and 16 for XMM registers.
3083	//
3084	// FIXME: This really results in shameful code when we end up needing to
3085	// collect arguments from different places; often what should result in a
3086	// simple assembling of a structure from scattered addresses has many more
3087	// loads than necessary. Can we clean this up?
3088	llvm::Type *LTy = CGF.ConvertTypeForMem(T: Ty);
3089	llvm::Value *RegSaveArea = CGF.Builder.CreateLoad(
3090	Addr: CGF.Builder.CreateStructGEP(Addr: VAListAddr, Index: 3), Name: "reg_save_area");
3091
3092	Address RegAddr = Address::invalid();
3093	if (neededInt && neededSSE) {
3094	// FIXME: Cleanup.
3095	assert(AI.isDirect() && "Unexpected ABI info for mixed regs");
3096	llvm::StructType *ST = cast<llvm::StructType>(Val: AI.getCoerceToType());
3097	Address Tmp = CGF.CreateMemTemp(T: Ty);
3098	Tmp = Tmp.withElementType(ElemTy: ST);
3099	assert(ST->getNumElements() == 2 && "Unexpected ABI info for mixed regs");
3100	llvm::Type *TyLo = ST->getElementType(N: 0);
3101	llvm::Type *TyHi = ST->getElementType(N: 1);
3102	assert((TyLo->isFPOrFPVectorTy() ^ TyHi->isFPOrFPVectorTy()) &&
3103	"Unexpected ABI info for mixed regs");
3104	llvm::Value *GPAddr =
3105	CGF.Builder.CreateGEP(Ty: CGF.Int8Ty, Ptr: RegSaveArea, IdxList: gp_offset);
3106	llvm::Value *FPAddr =
3107	CGF.Builder.CreateGEP(Ty: CGF.Int8Ty, Ptr: RegSaveArea, IdxList: fp_offset);
3108	llvm::Value *RegLoAddr = TyLo->isFPOrFPVectorTy() ? FPAddr : GPAddr;
3109	llvm::Value *RegHiAddr = TyLo->isFPOrFPVectorTy() ? GPAddr : FPAddr;
3110
3111	// Copy the first element.
3112	// FIXME: Our choice of alignment here and below is probably pessimistic.
3113	llvm::Value *V = CGF.Builder.CreateAlignedLoad(
3114	Ty: TyLo, Addr: RegLoAddr,
3115	Align: CharUnits::fromQuantity(Quantity: getDataLayout().getABITypeAlign(Ty: TyLo)));
3116	CGF.Builder.CreateStore(Val: V, Addr: CGF.Builder.CreateStructGEP(Addr: Tmp, Index: 0));
3117
3118	// Copy the second element.
3119	V = CGF.Builder.CreateAlignedLoad(
3120	Ty: TyHi, Addr: RegHiAddr,
3121	Align: CharUnits::fromQuantity(Quantity: getDataLayout().getABITypeAlign(Ty: TyHi)));
3122	CGF.Builder.CreateStore(Val: V, Addr: CGF.Builder.CreateStructGEP(Addr: Tmp, Index: 1));
3123
3124	RegAddr = Tmp.withElementType(ElemTy: LTy);
3125	} else if (neededInt) {
3126	RegAddr = Address(CGF.Builder.CreateGEP(Ty: CGF.Int8Ty, Ptr: RegSaveArea, IdxList: gp_offset),
3127	LTy, CharUnits::fromQuantity(Quantity: 8));
3128
3129	// Copy to a temporary if necessary to ensure the appropriate alignment.
3130	auto TInfo = getContext().getTypeInfoInChars(T: Ty);
3131	uint64_t TySize = TInfo.Width.getQuantity();
3132	CharUnits TyAlign = TInfo.Align;
3133
3134	// Copy into a temporary if the type is more aligned than the
3135	// register save area.
3136	if (TyAlign.getQuantity() > 8) {
3137	Address Tmp = CGF.CreateMemTemp(T: Ty);
3138	CGF.Builder.CreateMemCpy(Dest: Tmp, Src: RegAddr, Size: TySize, IsVolatile: false);
3139	RegAddr = Tmp;
3140	}
3141
3142	} else if (neededSSE == 1) {
3143	RegAddr = Address(CGF.Builder.CreateGEP(Ty: CGF.Int8Ty, Ptr: RegSaveArea, IdxList: fp_offset),
3144	LTy, CharUnits::fromQuantity(Quantity: 16));
3145	} else {
3146	assert(neededSSE == 2 && "Invalid number of needed registers!");
3147	// SSE registers are spaced 16 bytes apart in the register save
3148	// area, we need to collect the two eightbytes together.
3149	// The ABI isn't explicit about this, but it seems reasonable
3150	// to assume that the slots are 16-byte aligned, since the stack is
3151	// naturally 16-byte aligned and the prologue is expected to store
3152	// all the SSE registers to the RSA.
3153	Address RegAddrLo = Address(CGF.Builder.CreateGEP(Ty: CGF.Int8Ty, Ptr: RegSaveArea,
3154	IdxList: fp_offset),
3155	CGF.Int8Ty, CharUnits::fromQuantity(Quantity: 16));
3156	Address RegAddrHi =
3157	CGF.Builder.CreateConstInBoundsByteGEP(Addr: RegAddrLo,
3158	Offset: CharUnits::fromQuantity(Quantity: 16));
3159	llvm::Type *ST = AI.canHaveCoerceToType()
3160	? AI.getCoerceToType()
3161	: llvm::StructType::get(elt1: CGF.DoubleTy, elts: CGF.DoubleTy);
3162	llvm::Value *V;
3163	Address Tmp = CGF.CreateMemTemp(T: Ty);
3164	Tmp = Tmp.withElementType(ElemTy: ST);
3165	V = CGF.Builder.CreateLoad(
3166	Addr: RegAddrLo.withElementType(ElemTy: ST->getStructElementType(N: 0)));
3167	CGF.Builder.CreateStore(Val: V, Addr: CGF.Builder.CreateStructGEP(Addr: Tmp, Index: 0));
3168	V = CGF.Builder.CreateLoad(
3169	Addr: RegAddrHi.withElementType(ElemTy: ST->getStructElementType(N: 1)));
3170	CGF.Builder.CreateStore(Val: V, Addr: CGF.Builder.CreateStructGEP(Addr: Tmp, Index: 1));
3171
3172	RegAddr = Tmp.withElementType(ElemTy: LTy);
3173	}
3174
3175	// AMD64-ABI 3.5.7p5: Step 5. Set:
3176	// l->gp_offset = l->gp_offset + num_gp * 8
3177	// l->fp_offset = l->fp_offset + num_fp * 16.
3178	if (neededInt) {
3179	llvm::Value *Offset = llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: neededInt * 8);
3180	CGF.Builder.CreateStore(Val: CGF.Builder.CreateAdd(LHS: gp_offset, RHS: Offset),
3181	Addr: gp_offset_p);
3182	}
3183	if (neededSSE) {
3184	llvm::Value *Offset = llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: neededSSE * 16);
3185	CGF.Builder.CreateStore(Val: CGF.Builder.CreateAdd(LHS: fp_offset, RHS: Offset),
3186	Addr: fp_offset_p);
3187	}
3188	CGF.EmitBranch(Block: ContBlock);
3189
3190	// Emit code to load the value if it was passed in memory.
3191
3192	CGF.EmitBlock(BB: InMemBlock);
3193	Address MemAddr = EmitX86_64VAArgFromMemory(CGF, VAListAddr, Ty);
3194
3195	// Return the appropriate result.
3196
3197	CGF.EmitBlock(BB: ContBlock);
3198	Address ResAddr = emitMergePHI(CGF, Addr1: RegAddr, Block1: InRegBlock, Addr2: MemAddr, Block2: InMemBlock,
3199	Name: "vaarg.addr");
3200	return ResAddr;
3201	}
3202
3203	Address X86_64ABIInfo::EmitMSVAArg(CodeGenFunction &CGF, Address VAListAddr,
3204	QualType Ty) const {
3205	// MS x64 ABI requirement: "Any argument that doesn't fit in 8 bytes, or is
3206	// not 1, 2, 4, or 8 bytes, must be passed by reference."
3207	uint64_t Width = getContext().getTypeSize(T: Ty);
3208	bool IsIndirect = Width > 64 || !llvm::isPowerOf2_64(Value: Width);
3209
3210	return emitVoidPtrVAArg(CGF, VAListAddr, ValueTy: Ty, IsIndirect,
3211	ValueInfo: CGF.getContext().getTypeInfoInChars(T: Ty),
3212	SlotSizeAndAlign: CharUnits::fromQuantity(Quantity: 8),
3213	/*allowHigherAlign*/ AllowHigherAlign: false);
3214	}
3215
3216	ABIArgInfo WinX86_64ABIInfo::reclassifyHvaArgForVectorCall(
3217	QualType Ty, unsigned &FreeSSERegs, const ABIArgInfo &current) const {
3218	const Type *Base = nullptr;
3219	uint64_t NumElts = 0;
3220
3221	if (!Ty->isBuiltinType() && !Ty->isVectorType() &&
3222	isHomogeneousAggregate(Ty, Base, Members&: NumElts) && FreeSSERegs >= NumElts) {
3223	FreeSSERegs -= NumElts;
3224	return getDirectX86Hva();
3225	}
3226	return current;
3227	}
3228
3229	ABIArgInfo WinX86_64ABIInfo::classify(QualType Ty, unsigned &FreeSSERegs,
3230	bool IsReturnType, bool IsVectorCall,
3231	bool IsRegCall) const {
3232
3233	if (Ty->isVoidType())
3234	return ABIArgInfo::getIgnore();
3235
3236	if (const EnumType *EnumTy = Ty->getAs<EnumType>())
3237	Ty = EnumTy->getDecl()->getIntegerType();
3238
3239	TypeInfo Info = getContext().getTypeInfo(T: Ty);
3240	uint64_t Width = Info.Width;
3241	CharUnits Align = getContext().toCharUnitsFromBits(BitSize: Info.Align);
3242
3243	const RecordType *RT = Ty->getAs<RecordType>();
3244	if (RT) {
3245	if (!IsReturnType) {
3246	if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(RT, CXXABI&: getCXXABI()))
3247	return getNaturalAlignIndirect(Ty, ByVal: RAA == CGCXXABI::RAA_DirectInMemory);
3248	}
3249
3250	if (RT->getDecl()->hasFlexibleArrayMember())
3251	return getNaturalAlignIndirect(Ty, /*ByVal=*/false);
3252
3253	}
3254
3255	const Type *Base = nullptr;
3256	uint64_t NumElts = 0;
3257	// vectorcall adds the concept of a homogenous vector aggregate, similar to
3258	// other targets.
3259	if ((IsVectorCall || IsRegCall) &&
3260	isHomogeneousAggregate(Ty, Base, Members&: NumElts)) {
3261	if (IsRegCall) {
3262	if (FreeSSERegs >= NumElts) {
3263	FreeSSERegs -= NumElts;
3264	if (IsReturnType || Ty->isBuiltinType() || Ty->isVectorType())
3265	return ABIArgInfo::getDirect();
3266	return ABIArgInfo::getExpand();
3267	}
3268	return ABIArgInfo::getIndirect(Alignment: Align, /*ByVal=*/false);
3269	} else if (IsVectorCall) {
3270	if (FreeSSERegs >= NumElts &&
3271	(IsReturnType || Ty->isBuiltinType() || Ty->isVectorType())) {
3272	FreeSSERegs -= NumElts;
3273	return ABIArgInfo::getDirect();
3274	} else if (IsReturnType) {
3275	return ABIArgInfo::getExpand();
3276	} else if (!Ty->isBuiltinType() && !Ty->isVectorType()) {
3277	// HVAs are delayed and reclassified in the 2nd step.
3278	return ABIArgInfo::getIndirect(Alignment: Align, /*ByVal=*/false);
3279	}
3280	}
3281	}
3282
3283	if (Ty->isMemberPointerType()) {
3284	// If the member pointer is represented by an LLVM int or ptr, pass it
3285	// directly.
3286	llvm::Type *LLTy = CGT.ConvertType(T: Ty);
3287	if (LLTy->isPointerTy() || LLTy->isIntegerTy())
3288	return ABIArgInfo::getDirect();
3289	}
3290
3291	if (RT || Ty->isAnyComplexType() || Ty->isMemberPointerType()) {
3292	// MS x64 ABI requirement: "Any argument that doesn't fit in 8 bytes, or is
3293	// not 1, 2, 4, or 8 bytes, must be passed by reference."
3294	if (Width > 64 || !llvm::isPowerOf2_64(Value: Width))
3295	return getNaturalAlignIndirect(Ty, /*ByVal=*/false);
3296
3297	// Otherwise, coerce it to a small integer.
3298	return ABIArgInfo::getDirect(T: llvm::IntegerType::get(C&: getVMContext(), NumBits: Width));
3299	}
3300
3301	if (const BuiltinType *BT = Ty->getAs<BuiltinType>()) {
3302	switch (BT->getKind()) {
3303	case BuiltinType::Bool:
3304	// Bool type is always extended to the ABI, other builtin types are not
3305	// extended.
3306	return ABIArgInfo::getExtend(Ty);
3307
3308	case BuiltinType::LongDouble:
3309	// Mingw64 GCC uses the old 80 bit extended precision floating point
3310	// unit. It passes them indirectly through memory.
3311	if (IsMingw64) {
3312	const llvm::fltSemantics *LDF = &getTarget().getLongDoubleFormat();
3313	if (LDF == &llvm::APFloat::x87DoubleExtended())
3314	return ABIArgInfo::getIndirect(Alignment: Align, /*ByVal=*/false);
3315	}
3316	break;
3317
3318	case BuiltinType::Int128:
3319	case BuiltinType::UInt128:
3320	// If it's a parameter type, the normal ABI rule is that arguments larger
3321	// than 8 bytes are passed indirectly. GCC follows it. We follow it too,
3322	// even though it isn't particularly efficient.
3323	if (!IsReturnType)
3324	return ABIArgInfo::getIndirect(Alignment: Align, /*ByVal=*/false);
3325
3326	// Mingw64 GCC returns i128 in XMM0. Coerce to v2i64 to handle that.
3327	// Clang matches them for compatibility.
3328	return ABIArgInfo::getDirect(T: llvm::FixedVectorType::get(
3329	ElementType: llvm::Type::getInt64Ty(C&: getVMContext()), NumElts: 2));
3330
3331	default:
3332	break;
3333	}
3334	}
3335
3336	if (Ty->isBitIntType()) {
3337	// MS x64 ABI requirement: "Any argument that doesn't fit in 8 bytes, or is
3338	// not 1, 2, 4, or 8 bytes, must be passed by reference."
3339	// However, non-power-of-two bit-precise integers will be passed as 1, 2, 4,
3340	// or 8 bytes anyway as long is it fits in them, so we don't have to check
3341	// the power of 2.
3342	if (Width <= 64)
3343	return ABIArgInfo::getDirect();
3344	return ABIArgInfo::getIndirect(Alignment: Align, /*ByVal=*/false);
3345	}
3346
3347	return ABIArgInfo::getDirect();
3348	}
3349
3350	void WinX86_64ABIInfo::computeInfo(CGFunctionInfo &FI) const {
3351	const unsigned CC = FI.getCallingConvention();
3352	bool IsVectorCall = CC == llvm::CallingConv::X86_VectorCall;
3353	bool IsRegCall = CC == llvm::CallingConv::X86_RegCall;
3354
3355	// If __attribute__((sysv_abi)) is in use, use the SysV argument
3356	// classification rules.
3357	if (CC == llvm::CallingConv::X86_64_SysV) {
3358	X86_64ABIInfo SysVABIInfo(CGT, AVXLevel);
3359	SysVABIInfo.computeInfo(FI);
3360	return;
3361	}
3362
3363	unsigned FreeSSERegs = 0;
3364	if (IsVectorCall) {
3365	// We can use up to 4 SSE return registers with vectorcall.
3366	FreeSSERegs = 4;
3367	} else if (IsRegCall) {
3368	// RegCall gives us 16 SSE registers.
3369	FreeSSERegs = 16;
3370	}
3371
3372	if (!getCXXABI().classifyReturnType(FI))
3373	FI.getReturnInfo() = classify(Ty: FI.getReturnType(), FreeSSERegs, IsReturnType: true,
3374	IsVectorCall, IsRegCall);
3375
3376	if (IsVectorCall) {
3377	// We can use up to 6 SSE register parameters with vectorcall.
3378	FreeSSERegs = 6;
3379	} else if (IsRegCall) {
3380	// RegCall gives us 16 SSE registers, we can reuse the return registers.
3381	FreeSSERegs = 16;
3382	}
3383
3384	unsigned ArgNum = 0;
3385	unsigned ZeroSSERegs = 0;
3386	for (auto &I : FI.arguments()) {
3387	// Vectorcall in x64 only permits the first 6 arguments to be passed as
3388	// XMM/YMM registers. After the sixth argument, pretend no vector
3389	// registers are left.
3390	unsigned *MaybeFreeSSERegs =
3391	(IsVectorCall && ArgNum >= 6) ? &ZeroSSERegs : &FreeSSERegs;
3392	I.info =
3393	classify(Ty: I.type, FreeSSERegs&: *MaybeFreeSSERegs, IsReturnType: false, IsVectorCall, IsRegCall);
3394	++ArgNum;
3395	}
3396
3397	if (IsVectorCall) {
3398	// For vectorcall, assign aggregate HVAs to any free vector registers in a
3399	// second pass.
3400	for (auto &I : FI.arguments())
3401	I.info = reclassifyHvaArgForVectorCall(Ty: I.type, FreeSSERegs, current: I.info);
3402	}
3403	}
3404
3405	Address WinX86_64ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
3406	QualType Ty) const {
3407	// MS x64 ABI requirement: "Any argument that doesn't fit in 8 bytes, or is
3408	// not 1, 2, 4, or 8 bytes, must be passed by reference."
3409	uint64_t Width = getContext().getTypeSize(T: Ty);
3410	bool IsIndirect = Width > 64 || !llvm::isPowerOf2_64(Value: Width);
3411
3412	return emitVoidPtrVAArg(CGF, VAListAddr, ValueTy: Ty, IsIndirect,
3413	ValueInfo: CGF.getContext().getTypeInfoInChars(T: Ty),
3414	SlotSizeAndAlign: CharUnits::fromQuantity(Quantity: 8),
3415	/*allowHigherAlign*/ AllowHigherAlign: false);
3416	}
3417
3418	std::unique_ptr<TargetCodeGenInfo> CodeGen::createX86_32TargetCodeGenInfo(
3419	CodeGenModule &CGM, bool DarwinVectorABI, bool Win32StructABI,
3420	unsigned NumRegisterParameters, bool SoftFloatABI) {
3421	bool RetSmallStructInRegABI = X86_32TargetCodeGenInfo::isStructReturnInRegABI(
3422	Triple: CGM.getTriple(), Opts: CGM.getCodeGenOpts());
3423	return std::make_unique<X86_32TargetCodeGenInfo>(
3424	args&: CGM.getTypes(), args&: DarwinVectorABI, args&: RetSmallStructInRegABI, args&: Win32StructABI,
3425	args&: NumRegisterParameters, args&: SoftFloatABI);
3426	}
3427
3428	std::unique_ptr<TargetCodeGenInfo> CodeGen::createWinX86_32TargetCodeGenInfo(
3429	CodeGenModule &CGM, bool DarwinVectorABI, bool Win32StructABI,
3430	unsigned NumRegisterParameters) {
3431	bool RetSmallStructInRegABI = X86_32TargetCodeGenInfo::isStructReturnInRegABI(
3432	Triple: CGM.getTriple(), Opts: CGM.getCodeGenOpts());
3433	return std::make_unique<WinX86_32TargetCodeGenInfo>(
3434	args&: CGM.getTypes(), args&: DarwinVectorABI, args&: RetSmallStructInRegABI, args&: Win32StructABI,
3435	args&: NumRegisterParameters);
3436	}
3437
3438	std::unique_ptr<TargetCodeGenInfo>
3439	CodeGen::createX86_64TargetCodeGenInfo(CodeGenModule &CGM,
3440	X86AVXABILevel AVXLevel) {
3441	return std::make_unique<X86_64TargetCodeGenInfo>(args&: CGM.getTypes(), args&: AVXLevel);
3442	}
3443
3444	std::unique_ptr<TargetCodeGenInfo>
3445	CodeGen::createWinX86_64TargetCodeGenInfo(CodeGenModule &CGM,
3446	X86AVXABILevel AVXLevel) {
3447	return std::make_unique<WinX86_64TargetCodeGenInfo>(args&: CGM.getTypes(), args&: AVXLevel);
3448	}
3449