1	//===--- CGCall.cpp - Encapsulate calling convention details --------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// These classes wrap the information about a call or function
10	// definition used to handle ABI compliancy.
11	//
12	//===----------------------------------------------------------------------===//
13
14	#include "CGCall.h"
15	#include "ABIInfo.h"
16	#include "ABIInfoImpl.h"
17	#include "CGBlocks.h"
18	#include "CGCXXABI.h"
19	#include "CGCleanup.h"
20	#include "CGRecordLayout.h"
21	#include "CodeGenFunction.h"
22	#include "CodeGenModule.h"
23	#include "TargetInfo.h"
24	#include "clang/AST/Attr.h"
25	#include "clang/AST/Decl.h"
26	#include "clang/AST/DeclCXX.h"
27	#include "clang/AST/DeclObjC.h"
28	#include "clang/Basic/CodeGenOptions.h"
29	#include "clang/Basic/TargetInfo.h"
30	#include "clang/CodeGen/CGFunctionInfo.h"
31	#include "clang/CodeGen/SwiftCallingConv.h"
32	#include "llvm/ADT/StringExtras.h"
33	#include "llvm/Analysis/ValueTracking.h"
34	#include "llvm/IR/Assumptions.h"
35	#include "llvm/IR/AttributeMask.h"
36	#include "llvm/IR/Attributes.h"
37	#include "llvm/IR/CallingConv.h"
38	#include "llvm/IR/DataLayout.h"
39	#include "llvm/IR/InlineAsm.h"
40	#include "llvm/IR/IntrinsicInst.h"
41	#include "llvm/IR/Intrinsics.h"
42	#include "llvm/IR/Type.h"
43	#include "llvm/Transforms/Utils/Local.h"
44	#include <optional>
45	using namespace clang;
46	using namespace CodeGen;
47
48	/***/
49
50	unsigned CodeGenTypes::ClangCallConvToLLVMCallConv(CallingConv CC) {
51	switch (CC) {
52	default: return llvm::CallingConv::C;
53	case CC_X86StdCall: return llvm::CallingConv::X86_StdCall;
54	case CC_X86FastCall: return llvm::CallingConv::X86_FastCall;
55	case CC_X86RegCall: return llvm::CallingConv::X86_RegCall;
56	case CC_X86ThisCall: return llvm::CallingConv::X86_ThisCall;
57	case CC_Win64: return llvm::CallingConv::Win64;
58	case CC_X86_64SysV: return llvm::CallingConv::X86_64_SysV;
59	case CC_AAPCS: return llvm::CallingConv::ARM_AAPCS;
60	case CC_AAPCS_VFP: return llvm::CallingConv::ARM_AAPCS_VFP;
61	case CC_IntelOclBicc: return llvm::CallingConv::Intel_OCL_BI;
62	// TODO: Add support for __pascal to LLVM.
63	case CC_X86Pascal: return llvm::CallingConv::C;
64	// TODO: Add support for __vectorcall to LLVM.
65	case CC_X86VectorCall: return llvm::CallingConv::X86_VectorCall;
66	case CC_AArch64VectorCall: return llvm::CallingConv::AArch64_VectorCall;
67	case CC_AArch64SVEPCS: return llvm::CallingConv::AArch64_SVE_VectorCall;
68	case CC_AMDGPUKernelCall: return llvm::CallingConv::AMDGPU_KERNEL;
69	case CC_SpirFunction: return llvm::CallingConv::SPIR_FUNC;
70	case CC_OpenCLKernel: return CGM.getTargetCodeGenInfo().getOpenCLKernelCallingConv();
71	case CC_PreserveMost: return llvm::CallingConv::PreserveMost;
72	case CC_PreserveAll: return llvm::CallingConv::PreserveAll;
73	case CC_Swift: return llvm::CallingConv::Swift;
74	case CC_SwiftAsync: return llvm::CallingConv::SwiftTail;
75	case CC_M68kRTD: return llvm::CallingConv::M68k_RTD;
76	case CC_PreserveNone: return llvm::CallingConv::PreserveNone;
77	}
78	}
79
80	/// Derives the 'this' type for codegen purposes, i.e. ignoring method CVR
81	/// qualification. Either or both of RD and MD may be null. A null RD indicates
82	/// that there is no meaningful 'this' type, and a null MD can occur when
83	/// calling a method pointer.
84	CanQualType CodeGenTypes::DeriveThisType(const CXXRecordDecl *RD,
85	const CXXMethodDecl *MD) {
86	QualType RecTy;
87	if (RD)
88	RecTy = Context.getTagDeclType(RD)->getCanonicalTypeInternal();
89	else
90	RecTy = Context.VoidTy;
91
92	if (MD)
93	RecTy = Context.getAddrSpaceQualType(T: RecTy, AddressSpace: MD->getMethodQualifiers().getAddressSpace());
94	return Context.getPointerType(T: CanQualType::CreateUnsafe(Other: RecTy));
95	}
96
97	/// Returns the canonical formal type of the given C++ method.
98	static CanQual<FunctionProtoType> GetFormalType(const CXXMethodDecl *MD) {
99	return MD->getType()->getCanonicalTypeUnqualified()
100	.getAs<FunctionProtoType>();
101	}
102
103	/// Returns the "extra-canonicalized" return type, which discards
104	/// qualifiers on the return type. Codegen doesn't care about them,
105	/// and it makes ABI code a little easier to be able to assume that
106	/// all parameter and return types are top-level unqualified.
107	static CanQualType GetReturnType(QualType RetTy) {
108	return RetTy->getCanonicalTypeUnqualified().getUnqualifiedType();
109	}
110
111	/// Arrange the argument and result information for a value of the given
112	/// unprototyped freestanding function type.
113	const CGFunctionInfo &
114	CodeGenTypes::arrangeFreeFunctionType(CanQual<FunctionNoProtoType> FTNP) {
115	// When translating an unprototyped function type, always use a
116	// variadic type.
117	return arrangeLLVMFunctionInfo(FTNP->getReturnType().getUnqualifiedType(),
118	FnInfoOpts::None, std::nullopt,
119	FTNP->getExtInfo(), {}, RequiredArgs(0));
120	}
121
122	static void addExtParameterInfosForCall(
123	llvm::SmallVectorImpl<FunctionProtoType::ExtParameterInfo> &paramInfos,
124	const FunctionProtoType *proto,
125	unsigned prefixArgs,
126	unsigned totalArgs) {
127	assert(proto->hasExtParameterInfos());
128	assert(paramInfos.size() <= prefixArgs);
129	assert(proto->getNumParams() + prefixArgs <= totalArgs);
130
131	paramInfos.reserve(N: totalArgs);
132
133	// Add default infos for any prefix args that don't already have infos.
134	paramInfos.resize(N: prefixArgs);
135
136	// Add infos for the prototype.
137	for (const auto &ParamInfo : proto->getExtParameterInfos()) {
138	paramInfos.push_back(Elt: ParamInfo);
139	// pass_object_size params have no parameter info.
140	if (ParamInfo.hasPassObjectSize())
141	paramInfos.emplace_back();
142	}
143
144	assert(paramInfos.size() <= totalArgs &&
145	"Did we forget to insert pass_object_size args?");
146	// Add default infos for the variadic and/or suffix arguments.
147	paramInfos.resize(N: totalArgs);
148	}
149
150	/// Adds the formal parameters in FPT to the given prefix. If any parameter in
151	/// FPT has pass_object_size attrs, then we'll add parameters for those, too.
152	static void appendParameterTypes(const CodeGenTypes &CGT,
153	SmallVectorImpl<CanQualType> &prefix,
154	SmallVectorImpl<FunctionProtoType::ExtParameterInfo> &paramInfos,
155	CanQual<FunctionProtoType> FPT) {
156	// Fast path: don't touch param info if we don't need to.
157	if (!FPT->hasExtParameterInfos()) {
158	assert(paramInfos.empty() &&
159	"We have paramInfos, but the prototype doesn't?");
160	prefix.append(FPT->param_type_begin(), FPT->param_type_end());
161	return;
162	}
163
164	unsigned PrefixSize = prefix.size();
165	// In the vast majority of cases, we'll have precisely FPT->getNumParams()
166	// parameters; the only thing that can change this is the presence of
167	// pass_object_size. So, we preallocate for the common case.
168	prefix.reserve(prefix.size() + FPT->getNumParams());
169
170	auto ExtInfos = FPT->getExtParameterInfos();
171	assert(ExtInfos.size() == FPT->getNumParams());
172	for (unsigned I = 0, E = FPT->getNumParams(); I != E; ++I) {
173	prefix.push_back(FPT->getParamType(I));
174	if (ExtInfos[I].hasPassObjectSize())
175	prefix.push_back(Elt: CGT.getContext().getSizeType());
176	}
177
178	addExtParameterInfosForCall(paramInfos, proto: FPT.getTypePtr(), prefixArgs: PrefixSize,
179	totalArgs: prefix.size());
180	}
181
182	/// Arrange the LLVM function layout for a value of the given function
183	/// type, on top of any implicit parameters already stored.
184	static const CGFunctionInfo &
185	arrangeLLVMFunctionInfo(CodeGenTypes &CGT, bool instanceMethod,
186	SmallVectorImpl<CanQualType> &prefix,
187	CanQual<FunctionProtoType> FTP) {
188	SmallVector<FunctionProtoType::ExtParameterInfo, 16> paramInfos;
189	RequiredArgs Required = RequiredArgs::forPrototypePlus(prototype: FTP, additional: prefix.size());
190	// FIXME: Kill copy.
191	appendParameterTypes(CGT, prefix, paramInfos, FPT: FTP);
192	CanQualType resultType = FTP->getReturnType().getUnqualifiedType();
193
194	FnInfoOpts opts =
195	instanceMethod ? FnInfoOpts::IsInstanceMethod : FnInfoOpts::None;
196	return CGT.arrangeLLVMFunctionInfo(resultType, opts, prefix,
197	FTP->getExtInfo(), paramInfos, Required);
198	}
199
200	/// Arrange the argument and result information for a value of the
201	/// given freestanding function type.
202	const CGFunctionInfo &
203	CodeGenTypes::arrangeFreeFunctionType(CanQual<FunctionProtoType> FTP) {
204	SmallVector<CanQualType, 16> argTypes;
205	return ::arrangeLLVMFunctionInfo(CGT&: *this, /*instanceMethod=*/false, prefix&: argTypes,
206	FTP);
207	}
208
209	static CallingConv getCallingConventionForDecl(const ObjCMethodDecl *D,
210	bool IsWindows) {
211	// Set the appropriate calling convention for the Function.
212	if (D->hasAttr<StdCallAttr>())
213	return CC_X86StdCall;
214
215	if (D->hasAttr<FastCallAttr>())
216	return CC_X86FastCall;
217
218	if (D->hasAttr<RegCallAttr>())
219	return CC_X86RegCall;
220
221	if (D->hasAttr<ThisCallAttr>())
222	return CC_X86ThisCall;
223
224	if (D->hasAttr<VectorCallAttr>())
225	return CC_X86VectorCall;
226
227	if (D->hasAttr<PascalAttr>())
228	return CC_X86Pascal;
229
230	if (PcsAttr *PCS = D->getAttr<PcsAttr>())
231	return (PCS->getPCS() == PcsAttr::AAPCS ? CC_AAPCS : CC_AAPCS_VFP);
232
233	if (D->hasAttr<AArch64VectorPcsAttr>())
234	return CC_AArch64VectorCall;
235
236	if (D->hasAttr<AArch64SVEPcsAttr>())
237	return CC_AArch64SVEPCS;
238
239	if (D->hasAttr<AMDGPUKernelCallAttr>())
240	return CC_AMDGPUKernelCall;
241
242	if (D->hasAttr<IntelOclBiccAttr>())
243	return CC_IntelOclBicc;
244
245	if (D->hasAttr<MSABIAttr>())
246	return IsWindows ? CC_C : CC_Win64;
247
248	if (D->hasAttr<SysVABIAttr>())
249	return IsWindows ? CC_X86_64SysV : CC_C;
250
251	if (D->hasAttr<PreserveMostAttr>())
252	return CC_PreserveMost;
253
254	if (D->hasAttr<PreserveAllAttr>())
255	return CC_PreserveAll;
256
257	if (D->hasAttr<M68kRTDAttr>())
258	return CC_M68kRTD;
259
260	if (D->hasAttr<PreserveNoneAttr>())
261	return CC_PreserveNone;
262
263	return CC_C;
264	}
265
266	/// Arrange the argument and result information for a call to an
267	/// unknown C++ non-static member function of the given abstract type.
268	/// (A null RD means we don't have any meaningful "this" argument type,
269	/// so fall back to a generic pointer type).
270	/// The member function must be an ordinary function, i.e. not a
271	/// constructor or destructor.
272	const CGFunctionInfo &
273	CodeGenTypes::arrangeCXXMethodType(const CXXRecordDecl *RD,
274	const FunctionProtoType *FTP,
275	const CXXMethodDecl *MD) {
276	SmallVector<CanQualType, 16> argTypes;
277
278	// Add the 'this' pointer.
279	argTypes.push_back(Elt: DeriveThisType(RD, MD));
280
281	return ::arrangeLLVMFunctionInfo(
282	CGT&: *this, /*instanceMethod=*/true, prefix&: argTypes,
283	FTP: FTP->getCanonicalTypeUnqualified().getAs<FunctionProtoType>());
284	}
285
286	/// Set calling convention for CUDA/HIP kernel.
287	static void setCUDAKernelCallingConvention(CanQualType &FTy, CodeGenModule &CGM,
288	const FunctionDecl *FD) {
289	if (FD->hasAttr<CUDAGlobalAttr>()) {
290	const FunctionType *FT = FTy->getAs<FunctionType>();
291	CGM.getTargetCodeGenInfo().setCUDAKernelCallingConvention(FT);
292	FTy = FT->getCanonicalTypeUnqualified();
293	}
294	}
295
296	/// Arrange the argument and result information for a declaration or
297	/// definition of the given C++ non-static member function. The
298	/// member function must be an ordinary function, i.e. not a
299	/// constructor or destructor.
300	const CGFunctionInfo &
301	CodeGenTypes::arrangeCXXMethodDeclaration(const CXXMethodDecl *MD) {
302	assert(!isa<CXXConstructorDecl>(MD) && "wrong method for constructors!");
303	assert(!isa<CXXDestructorDecl>(MD) && "wrong method for destructors!");
304
305	CanQualType FT = GetFormalType(MD).getAs<Type>();
306	setCUDAKernelCallingConvention(FT, CGM, MD);
307	auto prototype = FT.getAs<FunctionProtoType>();
308
309	if (MD->isImplicitObjectMemberFunction()) {
310	// The abstract case is perfectly fine.
311	const CXXRecordDecl *ThisType = TheCXXABI.getThisArgumentTypeForMethod(MD);
312	return arrangeCXXMethodType(RD: ThisType, FTP: prototype.getTypePtr(), MD);
313	}
314
315	return arrangeFreeFunctionType(FTP: prototype);
316	}
317
318	bool CodeGenTypes::inheritingCtorHasParams(
319	const InheritedConstructor &Inherited, CXXCtorType Type) {
320	// Parameters are unnecessary if we're constructing a base class subobject
321	// and the inherited constructor lives in a virtual base.
322	return Type == Ctor_Complete ||
323	!Inherited.getShadowDecl()->constructsVirtualBase() ||
324	!Target.getCXXABI().hasConstructorVariants();
325	}
326
327	const CGFunctionInfo &
328	CodeGenTypes::arrangeCXXStructorDeclaration(GlobalDecl GD) {
329	auto *MD = cast<CXXMethodDecl>(Val: GD.getDecl());
330
331	SmallVector<CanQualType, 16> argTypes;
332	SmallVector<FunctionProtoType::ExtParameterInfo, 16> paramInfos;
333
334	const CXXRecordDecl *ThisType = TheCXXABI.getThisArgumentTypeForMethod(GD);
335	argTypes.push_back(Elt: DeriveThisType(RD: ThisType, MD));
336
337	bool PassParams = true;
338
339	if (auto *CD = dyn_cast<CXXConstructorDecl>(Val: MD)) {
340	// A base class inheriting constructor doesn't get forwarded arguments
341	// needed to construct a virtual base (or base class thereof).
342	if (auto Inherited = CD->getInheritedConstructor())
343	PassParams = inheritingCtorHasParams(Inherited, Type: GD.getCtorType());
344	}
345
346	CanQual<FunctionProtoType> FTP = GetFormalType(MD);
347
348	// Add the formal parameters.
349	if (PassParams)
350	appendParameterTypes(CGT: *this, prefix&: argTypes, paramInfos, FPT: FTP);
351
352	CGCXXABI::AddedStructorArgCounts AddedArgs =
353	TheCXXABI.buildStructorSignature(GD, ArgTys&: argTypes);
354	if (!paramInfos.empty()) {
355	// Note: prefix implies after the first param.
356	if (AddedArgs.Prefix)
357	paramInfos.insert(I: paramInfos.begin() + 1, NumToInsert: AddedArgs.Prefix,
358	Elt: FunctionProtoType::ExtParameterInfo{});
359	if (AddedArgs.Suffix)
360	paramInfos.append(NumInputs: AddedArgs.Suffix,
361	Elt: FunctionProtoType::ExtParameterInfo{});
362	}
363
364	RequiredArgs required =
365	(PassParams && MD->isVariadic() ? RequiredArgs(argTypes.size())
366	: RequiredArgs::All);
367
368	FunctionType::ExtInfo extInfo = FTP->getExtInfo();
369	CanQualType resultType = TheCXXABI.HasThisReturn(GD)
370	? argTypes.front()
371	: TheCXXABI.hasMostDerivedReturn(GD)
372	? CGM.getContext().VoidPtrTy
373	: Context.VoidTy;
374	return arrangeLLVMFunctionInfo(returnType: resultType, opts: FnInfoOpts::IsInstanceMethod,
375	argTypes, info: extInfo, paramInfos, args: required);
376	}
377
378	static SmallVector<CanQualType, 16>
379	getArgTypesForCall(ASTContext &ctx, const CallArgList &args) {
380	SmallVector<CanQualType, 16> argTypes;
381	for (auto &arg : args)
382	argTypes.push_back(Elt: ctx.getCanonicalParamType(T: arg.Ty));
383	return argTypes;
384	}
385
386	static SmallVector<CanQualType, 16>
387	getArgTypesForDeclaration(ASTContext &ctx, const FunctionArgList &args) {
388	SmallVector<CanQualType, 16> argTypes;
389	for (auto &arg : args)
390	argTypes.push_back(Elt: ctx.getCanonicalParamType(T: arg->getType()));
391	return argTypes;
392	}
393
394	static llvm::SmallVector<FunctionProtoType::ExtParameterInfo, 16>
395	getExtParameterInfosForCall(const FunctionProtoType *proto,
396	unsigned prefixArgs, unsigned totalArgs) {
397	llvm::SmallVector<FunctionProtoType::ExtParameterInfo, 16> result;
398	if (proto->hasExtParameterInfos()) {
399	addExtParameterInfosForCall(paramInfos&: result, proto, prefixArgs, totalArgs);
400	}
401	return result;
402	}
403
404	/// Arrange a call to a C++ method, passing the given arguments.
405	///
406	/// ExtraPrefixArgs is the number of ABI-specific args passed after the `this`
407	/// parameter.
408	/// ExtraSuffixArgs is the number of ABI-specific args passed at the end of
409	/// args.
410	/// PassProtoArgs indicates whether `args` has args for the parameters in the
411	/// given CXXConstructorDecl.
412	const CGFunctionInfo &
413	CodeGenTypes::arrangeCXXConstructorCall(const CallArgList &args,
414	const CXXConstructorDecl *D,
415	CXXCtorType CtorKind,
416	unsigned ExtraPrefixArgs,
417	unsigned ExtraSuffixArgs,
418	bool PassProtoArgs) {
419	// FIXME: Kill copy.
420	SmallVector<CanQualType, 16> ArgTypes;
421	for (const auto &Arg : args)
422	ArgTypes.push_back(Elt: Context.getCanonicalParamType(T: Arg.Ty));
423
424	// +1 for implicit this, which should always be args[0].
425	unsigned TotalPrefixArgs = 1 + ExtraPrefixArgs;
426
427	CanQual<FunctionProtoType> FPT = GetFormalType(D);
428	RequiredArgs Required = PassProtoArgs
429	? RequiredArgs::forPrototypePlus(
430	prototype: FPT, additional: TotalPrefixArgs + ExtraSuffixArgs)
431	: RequiredArgs::All;
432
433	GlobalDecl GD(D, CtorKind);
434	CanQualType ResultType = TheCXXABI.HasThisReturn(GD)
435	? ArgTypes.front()
436	: TheCXXABI.hasMostDerivedReturn(GD)
437	? CGM.getContext().VoidPtrTy
438	: Context.VoidTy;
439
440	FunctionType::ExtInfo Info = FPT->getExtInfo();
441	llvm::SmallVector<FunctionProtoType::ExtParameterInfo, 16> ParamInfos;
442	// If the prototype args are elided, we should only have ABI-specific args,
443	// which never have param info.
444	if (PassProtoArgs && FPT->hasExtParameterInfos()) {
445	// ABI-specific suffix arguments are treated the same as variadic arguments.
446	addExtParameterInfosForCall(paramInfos&: ParamInfos, proto: FPT.getTypePtr(), prefixArgs: TotalPrefixArgs,
447	totalArgs: ArgTypes.size());
448	}
449
450	return arrangeLLVMFunctionInfo(returnType: ResultType, opts: FnInfoOpts::IsInstanceMethod,
451	argTypes: ArgTypes, info: Info, paramInfos: ParamInfos, args: Required);
452	}
453
454	/// Arrange the argument and result information for the declaration or
455	/// definition of the given function.
456	const CGFunctionInfo &
457	CodeGenTypes::arrangeFunctionDeclaration(const FunctionDecl *FD) {
458	if (const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(Val: FD))
459	if (MD->isImplicitObjectMemberFunction())
460	return arrangeCXXMethodDeclaration(MD);
461
462	CanQualType FTy = FD->getType()->getCanonicalTypeUnqualified();
463
464	assert(isa<FunctionType>(FTy));
465	setCUDAKernelCallingConvention(FTy, CGM, FD);
466
467	// When declaring a function without a prototype, always use a
468	// non-variadic type.
469	if (CanQual<FunctionNoProtoType> noProto = FTy.getAs<FunctionNoProtoType>()) {
470	return arrangeLLVMFunctionInfo(noProto->getReturnType(), FnInfoOpts::None,
471	std::nullopt, noProto->getExtInfo(), {},
472	RequiredArgs::All);
473	}
474
475	return arrangeFreeFunctionType(FTP: FTy.castAs<FunctionProtoType>());
476	}
477
478	/// Arrange the argument and result information for the declaration or
479	/// definition of an Objective-C method.
480	const CGFunctionInfo &
481	CodeGenTypes::arrangeObjCMethodDeclaration(const ObjCMethodDecl *MD) {
482	// It happens that this is the same as a call with no optional
483	// arguments, except also using the formal 'self' type.
484	return arrangeObjCMessageSendSignature(MD, receiverType: MD->getSelfDecl()->getType());
485	}
486
487	/// Arrange the argument and result information for the function type
488	/// through which to perform a send to the given Objective-C method,
489	/// using the given receiver type. The receiver type is not always
490	/// the 'self' type of the method or even an Objective-C pointer type.
491	/// This is *not* the right method for actually performing such a
492	/// message send, due to the possibility of optional arguments.
493	const CGFunctionInfo &
494	CodeGenTypes::arrangeObjCMessageSendSignature(const ObjCMethodDecl *MD,
495	QualType receiverType) {
496	SmallVector<CanQualType, 16> argTys;
497	SmallVector<FunctionProtoType::ExtParameterInfo, 4> extParamInfos(
498	MD->isDirectMethod() ? 1 : 2);
499	argTys.push_back(Elt: Context.getCanonicalParamType(T: receiverType));
500	if (!MD->isDirectMethod())
501	argTys.push_back(Elt: Context.getCanonicalParamType(T: Context.getObjCSelType()));
502	// FIXME: Kill copy?
503	for (const auto *I : MD->parameters()) {
504	argTys.push_back(Elt: Context.getCanonicalParamType(T: I->getType()));
505	auto extParamInfo = FunctionProtoType::ExtParameterInfo().withIsNoEscape(
506	I->hasAttr<NoEscapeAttr>());
507	extParamInfos.push_back(Elt: extParamInfo);
508	}
509
510	FunctionType::ExtInfo einfo;
511	bool IsWindows = getContext().getTargetInfo().getTriple().isOSWindows();
512	einfo = einfo.withCallingConv(cc: getCallingConventionForDecl(D: MD, IsWindows));
513
514	if (getContext().getLangOpts().ObjCAutoRefCount &&
515	MD->hasAttr<NSReturnsRetainedAttr>())
516	einfo = einfo.withProducesResult(producesResult: true);
517
518	RequiredArgs required =
519	(MD->isVariadic() ? RequiredArgs(argTys.size()) : RequiredArgs::All);
520
521	return arrangeLLVMFunctionInfo(returnType: GetReturnType(RetTy: MD->getReturnType()),
522	opts: FnInfoOpts::None, argTypes: argTys, info: einfo, paramInfos: extParamInfos,
523	args: required);
524	}
525
526	const CGFunctionInfo &
527	CodeGenTypes::arrangeUnprototypedObjCMessageSend(QualType returnType,
528	const CallArgList &args) {
529	auto argTypes = getArgTypesForCall(ctx&: Context, args);
530	FunctionType::ExtInfo einfo;
531
532	return arrangeLLVMFunctionInfo(returnType: GetReturnType(RetTy: returnType), opts: FnInfoOpts::None,
533	argTypes, info: einfo, paramInfos: {}, args: RequiredArgs::All);
534	}
535
536	const CGFunctionInfo &
537	CodeGenTypes::arrangeGlobalDeclaration(GlobalDecl GD) {
538	// FIXME: Do we need to handle ObjCMethodDecl?
539	const FunctionDecl *FD = cast<FunctionDecl>(Val: GD.getDecl());
540
541	if (isa<CXXConstructorDecl>(Val: GD.getDecl()) ||
542	isa<CXXDestructorDecl>(Val: GD.getDecl()))
543	return arrangeCXXStructorDeclaration(GD);
544
545	return arrangeFunctionDeclaration(FD);
546	}
547
548	/// Arrange a thunk that takes 'this' as the first parameter followed by
549	/// varargs. Return a void pointer, regardless of the actual return type.
550	/// The body of the thunk will end in a musttail call to a function of the
551	/// correct type, and the caller will bitcast the function to the correct
552	/// prototype.
553	const CGFunctionInfo &
554	CodeGenTypes::arrangeUnprototypedMustTailThunk(const CXXMethodDecl *MD) {
555	assert(MD->isVirtual() && "only methods have thunks");
556	CanQual<FunctionProtoType> FTP = GetFormalType(MD);
557	CanQualType ArgTys[] = {DeriveThisType(RD: MD->getParent(), MD)};
558	return arrangeLLVMFunctionInfo(Context.VoidTy, FnInfoOpts::None, ArgTys,
559	FTP->getExtInfo(), {}, RequiredArgs(1));
560	}
561
562	const CGFunctionInfo &
563	CodeGenTypes::arrangeMSCtorClosure(const CXXConstructorDecl *CD,
564	CXXCtorType CT) {
565	assert(CT == Ctor_CopyingClosure || CT == Ctor_DefaultClosure);
566
567	CanQual<FunctionProtoType> FTP = GetFormalType(CD);
568	SmallVector<CanQualType, 2> ArgTys;
569	const CXXRecordDecl *RD = CD->getParent();
570	ArgTys.push_back(Elt: DeriveThisType(RD, CD));
571	if (CT == Ctor_CopyingClosure)
572	ArgTys.push_back(*FTP->param_type_begin());
573	if (RD->getNumVBases() > 0)
574	ArgTys.push_back(Elt: Context.IntTy);
575	CallingConv CC = Context.getDefaultCallingConvention(
576	/*IsVariadic=*/false, /*IsCXXMethod=*/true);
577	return arrangeLLVMFunctionInfo(returnType: Context.VoidTy, opts: FnInfoOpts::IsInstanceMethod,
578	argTypes: ArgTys, info: FunctionType::ExtInfo(CC), paramInfos: {},
579	args: RequiredArgs::All);
580	}
581
582	/// Arrange a call as unto a free function, except possibly with an
583	/// additional number of formal parameters considered required.
584	static const CGFunctionInfo &
585	arrangeFreeFunctionLikeCall(CodeGenTypes &CGT,
586	CodeGenModule &CGM,
587	const CallArgList &args,
588	const FunctionType *fnType,
589	unsigned numExtraRequiredArgs,
590	bool chainCall) {
591	assert(args.size() >= numExtraRequiredArgs);
592
593	llvm::SmallVector<FunctionProtoType::ExtParameterInfo, 16> paramInfos;
594
595	// In most cases, there are no optional arguments.
596	RequiredArgs required = RequiredArgs::All;
597
598	// If we have a variadic prototype, the required arguments are the
599	// extra prefix plus the arguments in the prototype.
600	if (const FunctionProtoType *proto = dyn_cast<FunctionProtoType>(Val: fnType)) {
601	if (proto->isVariadic())
602	required = RequiredArgs::forPrototypePlus(prototype: proto, additional: numExtraRequiredArgs);
603
604	if (proto->hasExtParameterInfos())
605	addExtParameterInfosForCall(paramInfos, proto, prefixArgs: numExtraRequiredArgs,
606	totalArgs: args.size());
607
608	// If we don't have a prototype at all, but we're supposed to
609	// explicitly use the variadic convention for unprototyped calls,
610	// treat all of the arguments as required but preserve the nominal
611	// possibility of variadics.
612	} else if (CGM.getTargetCodeGenInfo()
613	.isNoProtoCallVariadic(args,
614	fnType: cast<FunctionNoProtoType>(Val: fnType))) {
615	required = RequiredArgs(args.size());
616	}
617
618	// FIXME: Kill copy.
619	SmallVector<CanQualType, 16> argTypes;
620	for (const auto &arg : args)
621	argTypes.push_back(Elt: CGT.getContext().getCanonicalParamType(T: arg.Ty));
622	FnInfoOpts opts = chainCall ? FnInfoOpts::IsChainCall : FnInfoOpts::None;
623	return CGT.arrangeLLVMFunctionInfo(returnType: GetReturnType(RetTy: fnType->getReturnType()),
624	opts, argTypes, info: fnType->getExtInfo(),
625	paramInfos, args: required);
626	}
627
628	/// Figure out the rules for calling a function with the given formal
629	/// type using the given arguments. The arguments are necessary
630	/// because the function might be unprototyped, in which case it's
631	/// target-dependent in crazy ways.
632	const CGFunctionInfo &
633	CodeGenTypes::arrangeFreeFunctionCall(const CallArgList &args,
634	const FunctionType *fnType,
635	bool chainCall) {
636	return arrangeFreeFunctionLikeCall(CGT&: *this, CGM, args, fnType,
637	numExtraRequiredArgs: chainCall ? 1 : 0, chainCall);
638	}
639
640	/// A block function is essentially a free function with an
641	/// extra implicit argument.
642	const CGFunctionInfo &
643	CodeGenTypes::arrangeBlockFunctionCall(const CallArgList &args,
644	const FunctionType *fnType) {
645	return arrangeFreeFunctionLikeCall(CGT&: *this, CGM, args, fnType, numExtraRequiredArgs: 1,
646	/*chainCall=*/false);
647	}
648
649	const CGFunctionInfo &
650	CodeGenTypes::arrangeBlockFunctionDeclaration(const FunctionProtoType *proto,
651	const FunctionArgList &params) {
652	auto paramInfos = getExtParameterInfosForCall(proto, prefixArgs: 1, totalArgs: params.size());
653	auto argTypes = getArgTypesForDeclaration(ctx&: Context, args: params);
654
655	return arrangeLLVMFunctionInfo(returnType: GetReturnType(proto->getReturnType()),
656	opts: FnInfoOpts::None, argTypes,
657	info: proto->getExtInfo(), paramInfos,
658	args: RequiredArgs::forPrototypePlus(prototype: proto, additional: 1));
659	}
660
661	const CGFunctionInfo &
662	CodeGenTypes::arrangeBuiltinFunctionCall(QualType resultType,
663	const CallArgList &args) {
664	// FIXME: Kill copy.
665	SmallVector<CanQualType, 16> argTypes;
666	for (const auto &Arg : args)
667	argTypes.push_back(Elt: Context.getCanonicalParamType(T: Arg.Ty));
668	return arrangeLLVMFunctionInfo(returnType: GetReturnType(RetTy: resultType), opts: FnInfoOpts::None,
669	argTypes, info: FunctionType::ExtInfo(),
670	/*paramInfos=*/{}, args: RequiredArgs::All);
671	}
672
673	const CGFunctionInfo &
674	CodeGenTypes::arrangeBuiltinFunctionDeclaration(QualType resultType,
675	const FunctionArgList &args) {
676	auto argTypes = getArgTypesForDeclaration(ctx&: Context, args);
677
678	return arrangeLLVMFunctionInfo(returnType: GetReturnType(RetTy: resultType), opts: FnInfoOpts::None,
679	argTypes, info: FunctionType::ExtInfo(), paramInfos: {},
680	args: RequiredArgs::All);
681	}
682
683	const CGFunctionInfo &
684	CodeGenTypes::arrangeBuiltinFunctionDeclaration(CanQualType resultType,
685	ArrayRef<CanQualType> argTypes) {
686	return arrangeLLVMFunctionInfo(returnType: resultType, opts: FnInfoOpts::None, argTypes,
687	info: FunctionType::ExtInfo(), paramInfos: {},
688	args: RequiredArgs::All);
689	}
690
691	/// Arrange a call to a C++ method, passing the given arguments.
692	///
693	/// numPrefixArgs is the number of ABI-specific prefix arguments we have. It
694	/// does not count `this`.
695	const CGFunctionInfo &
696	CodeGenTypes::arrangeCXXMethodCall(const CallArgList &args,
697	const FunctionProtoType *proto,
698	RequiredArgs required,
699	unsigned numPrefixArgs) {
700	assert(numPrefixArgs + 1 <= args.size() &&
701	"Emitting a call with less args than the required prefix?");
702	// Add one to account for `this`. It's a bit awkward here, but we don't count
703	// `this` in similar places elsewhere.
704	auto paramInfos =
705	getExtParameterInfosForCall(proto, prefixArgs: numPrefixArgs + 1, totalArgs: args.size());
706
707	// FIXME: Kill copy.
708	auto argTypes = getArgTypesForCall(ctx&: Context, args);
709
710	FunctionType::ExtInfo info = proto->getExtInfo();
711	return arrangeLLVMFunctionInfo(returnType: GetReturnType(proto->getReturnType()),
712	opts: FnInfoOpts::IsInstanceMethod, argTypes, info,
713	paramInfos, args: required);
714	}
715
716	const CGFunctionInfo &CodeGenTypes::arrangeNullaryFunction() {
717	return arrangeLLVMFunctionInfo(returnType: getContext().VoidTy, opts: FnInfoOpts::None,
718	argTypes: std::nullopt, info: FunctionType::ExtInfo(), paramInfos: {},
719	args: RequiredArgs::All);
720	}
721
722	const CGFunctionInfo &
723	CodeGenTypes::arrangeCall(const CGFunctionInfo &signature,
724	const CallArgList &args) {
725	assert(signature.arg_size() <= args.size());
726	if (signature.arg_size() == args.size())
727	return signature;
728
729	SmallVector<FunctionProtoType::ExtParameterInfo, 16> paramInfos;
730	auto sigParamInfos = signature.getExtParameterInfos();
731	if (!sigParamInfos.empty()) {
732	paramInfos.append(in_start: sigParamInfos.begin(), in_end: sigParamInfos.end());
733	paramInfos.resize(N: args.size());
734	}
735
736	auto argTypes = getArgTypesForCall(ctx&: Context, args);
737
738	assert(signature.getRequiredArgs().allowsOptionalArgs());
739	FnInfoOpts opts = FnInfoOpts::None;
740	if (signature.isInstanceMethod())
741	opts |= FnInfoOpts::IsInstanceMethod;
742	if (signature.isChainCall())
743	opts |= FnInfoOpts::IsChainCall;
744	if (signature.isDelegateCall())
745	opts |= FnInfoOpts::IsDelegateCall;
746	return arrangeLLVMFunctionInfo(returnType: signature.getReturnType(), opts, argTypes,
747	info: signature.getExtInfo(), paramInfos,
748	args: signature.getRequiredArgs());
749	}
750
751	namespace clang {
752	namespace CodeGen {
753	void computeSPIRKernelABIInfo(CodeGenModule &CGM, CGFunctionInfo &FI);
754	}
755	}
756
757	/// Arrange the argument and result information for an abstract value
758	/// of a given function type. This is the method which all of the
759	/// above functions ultimately defer to.
760	const CGFunctionInfo &CodeGenTypes::arrangeLLVMFunctionInfo(
761	CanQualType resultType, FnInfoOpts opts, ArrayRef<CanQualType> argTypes,
762	FunctionType::ExtInfo info,
763	ArrayRef<FunctionProtoType::ExtParameterInfo> paramInfos,
764	RequiredArgs required) {
765	assert(llvm::all_of(argTypes,
766	[](CanQualType T) { return T.isCanonicalAsParam(); }));
767
768	// Lookup or create unique function info.
769	llvm::FoldingSetNodeID ID;
770	bool isInstanceMethod =
771	(opts & FnInfoOpts::IsInstanceMethod) == FnInfoOpts::IsInstanceMethod;
772	bool isChainCall =
773	(opts & FnInfoOpts::IsChainCall) == FnInfoOpts::IsChainCall;
774	bool isDelegateCall =
775	(opts & FnInfoOpts::IsDelegateCall) == FnInfoOpts::IsDelegateCall;
776	CGFunctionInfo::Profile(ID, InstanceMethod: isInstanceMethod, ChainCall: isChainCall, IsDelegateCall: isDelegateCall,
777	info, paramInfos, required, resultType, argTypes);
778
779	void *insertPos = nullptr;
780	CGFunctionInfo *FI = FunctionInfos.FindNodeOrInsertPos(ID, InsertPos&: insertPos);
781	if (FI)
782	return *FI;
783
784	unsigned CC = ClangCallConvToLLVMCallConv(CC: info.getCC());
785
786	// Construct the function info. We co-allocate the ArgInfos.
787	FI = CGFunctionInfo::create(llvmCC: CC, instanceMethod: isInstanceMethod, chainCall: isChainCall, delegateCall: isDelegateCall,
788	extInfo: info, paramInfos, resultType, argTypes, required);
789	FunctionInfos.InsertNode(N: FI, InsertPos: insertPos);
790
791	bool inserted = FunctionsBeingProcessed.insert(Ptr: FI).second;
792	(void)inserted;
793	assert(inserted && "Recursively being processed?");
794
795	// Compute ABI information.
796	if (CC == llvm::CallingConv::SPIR_KERNEL) {
797	// Force target independent argument handling for the host visible
798	// kernel functions.
799	computeSPIRKernelABIInfo(CGM, FI&: *FI);
800	} else if (info.getCC() == CC_Swift || info.getCC() == CC_SwiftAsync) {
801	swiftcall::computeABIInfo(CGM, FI&: *FI);
802	} else {
803	getABIInfo().computeInfo(FI&: *FI);
804	}
805
806	// Loop over all of the computed argument and return value info. If any of
807	// them are direct or extend without a specified coerce type, specify the
808	// default now.
809	ABIArgInfo &retInfo = FI->getReturnInfo();
810	if (retInfo.canHaveCoerceToType() && retInfo.getCoerceToType() == nullptr)
811	retInfo.setCoerceToType(ConvertType(T: FI->getReturnType()));
812
813	for (auto &I : FI->arguments())
814	if (I.info.canHaveCoerceToType() && I.info.getCoerceToType() == nullptr)
815	I.info.setCoerceToType(ConvertType(T: I.type));
816
817	bool erased = FunctionsBeingProcessed.erase(Ptr: FI); (void)erased;
818	assert(erased && "Not in set?");
819
820	return *FI;
821	}
822
823	CGFunctionInfo *CGFunctionInfo::create(unsigned llvmCC, bool instanceMethod,
824	bool chainCall, bool delegateCall,
825	const FunctionType::ExtInfo &info,
826	ArrayRef<ExtParameterInfo> paramInfos,
827	CanQualType resultType,
828	ArrayRef<CanQualType> argTypes,
829	RequiredArgs required) {
830	assert(paramInfos.empty() || paramInfos.size() == argTypes.size());
831	assert(!required.allowsOptionalArgs() ||
832	required.getNumRequiredArgs() <= argTypes.size());
833
834	void *buffer =
835	operator new(totalSizeToAlloc<ArgInfo, ExtParameterInfo>(
836	Counts: argTypes.size() + 1, Counts: paramInfos.size()));
837
838	CGFunctionInfo *FI = new(buffer) CGFunctionInfo();
839	FI->CallingConvention = llvmCC;
840	FI->EffectiveCallingConvention = llvmCC;
841	FI->ASTCallingConvention = info.getCC();
842	FI->InstanceMethod = instanceMethod;
843	FI->ChainCall = chainCall;
844	FI->DelegateCall = delegateCall;
845	FI->CmseNSCall = info.getCmseNSCall();
846	FI->NoReturn = info.getNoReturn();
847	FI->ReturnsRetained = info.getProducesResult();
848	FI->NoCallerSavedRegs = info.getNoCallerSavedRegs();
849	FI->NoCfCheck = info.getNoCfCheck();
850	FI->Required = required;
851	FI->HasRegParm = info.getHasRegParm();
852	FI->RegParm = info.getRegParm();
853	FI->ArgStruct = nullptr;
854	FI->ArgStructAlign = 0;
855	FI->NumArgs = argTypes.size();
856	FI->HasExtParameterInfos = !paramInfos.empty();
857	FI->getArgsBuffer()[0].type = resultType;
858	FI->MaxVectorWidth = 0;
859	for (unsigned i = 0, e = argTypes.size(); i != e; ++i)
860	FI->getArgsBuffer()[i + 1].type = argTypes[i];
861	for (unsigned i = 0, e = paramInfos.size(); i != e; ++i)
862	FI->getExtParameterInfosBuffer()[i] = paramInfos[i];
863	return FI;
864	}
865
866	/***/
867
868	namespace {
869	// ABIArgInfo::Expand implementation.
870
871	// Specifies the way QualType passed as ABIArgInfo::Expand is expanded.
872	struct TypeExpansion {
873	enum TypeExpansionKind {
874	// Elements of constant arrays are expanded recursively.
875	TEK_ConstantArray,
876	// Record fields are expanded recursively (but if record is a union, only
877	// the field with the largest size is expanded).
878	TEK_Record,
879	// For complex types, real and imaginary parts are expanded recursively.
880	TEK_Complex,
881	// All other types are not expandable.
882	TEK_None
883	};
884
885	const TypeExpansionKind Kind;
886
887	TypeExpansion(TypeExpansionKind K) : Kind(K) {}
888	virtual ~TypeExpansion() {}
889	};
890
891	struct ConstantArrayExpansion : TypeExpansion {
892	QualType EltTy;
893	uint64_t NumElts;
894
895	ConstantArrayExpansion(QualType EltTy, uint64_t NumElts)
896	: TypeExpansion(TEK_ConstantArray), EltTy(EltTy), NumElts(NumElts) {}
897	static bool classof(const TypeExpansion *TE) {
898	return TE->Kind == TEK_ConstantArray;
899	}
900	};
901
902	struct RecordExpansion : TypeExpansion {
903	SmallVector<const CXXBaseSpecifier *, 1> Bases;
904
905	SmallVector<const FieldDecl *, 1> Fields;
906
907	RecordExpansion(SmallVector<const CXXBaseSpecifier *, 1> &&Bases,
908	SmallVector<const FieldDecl *, 1> &&Fields)
909	: TypeExpansion(TEK_Record), Bases(std::move(Bases)),
910	Fields(std::move(Fields)) {}
911	static bool classof(const TypeExpansion *TE) {
912	return TE->Kind == TEK_Record;
913	}
914	};
915
916	struct ComplexExpansion : TypeExpansion {
917	QualType EltTy;
918
919	ComplexExpansion(QualType EltTy) : TypeExpansion(TEK_Complex), EltTy(EltTy) {}
920	static bool classof(const TypeExpansion *TE) {
921	return TE->Kind == TEK_Complex;
922	}
923	};
924
925	struct NoExpansion : TypeExpansion {
926	NoExpansion() : TypeExpansion(TEK_None) {}
927	static bool classof(const TypeExpansion *TE) {
928	return TE->Kind == TEK_None;
929	}
930	};
931	} // namespace
932
933	static std::unique_ptr<TypeExpansion>
934	getTypeExpansion(QualType Ty, const ASTContext &Context) {
935	if (const ConstantArrayType *AT = Context.getAsConstantArrayType(T: Ty)) {
936	return std::make_unique<ConstantArrayExpansion>(
937	AT->getElementType(), AT->getSize().getZExtValue());
938	}
939	if (const RecordType *RT = Ty->getAs<RecordType>()) {
940	SmallVector<const CXXBaseSpecifier *, 1> Bases;
941	SmallVector<const FieldDecl *, 1> Fields;
942	const RecordDecl *RD = RT->getDecl();
943	assert(!RD->hasFlexibleArrayMember() &&
944	"Cannot expand structure with flexible array.");
945	if (RD->isUnion()) {
946	// Unions can be here only in degenerative cases - all the fields are same
947	// after flattening. Thus we have to use the "largest" field.
948	const FieldDecl *LargestFD = nullptr;
949	CharUnits UnionSize = CharUnits::Zero();
950
951	for (const auto *FD : RD->fields()) {
952	if (FD->isZeroLengthBitField(Ctx: Context))
953	continue;
954	assert(!FD->isBitField() &&
955	"Cannot expand structure with bit-field members.");
956	CharUnits FieldSize = Context.getTypeSizeInChars(FD->getType());
957	if (UnionSize < FieldSize) {
958	UnionSize = FieldSize;
959	LargestFD = FD;
960	}
961	}
962	if (LargestFD)
963	Fields.push_back(Elt: LargestFD);
964	} else {
965	if (const auto *CXXRD = dyn_cast<CXXRecordDecl>(Val: RD)) {
966	assert(!CXXRD->isDynamicClass() &&
967	"cannot expand vtable pointers in dynamic classes");
968	llvm::append_range(C&: Bases, R: llvm::make_pointer_range(Range: CXXRD->bases()));
969	}
970
971	for (const auto *FD : RD->fields()) {
972	if (FD->isZeroLengthBitField(Ctx: Context))
973	continue;
974	assert(!FD->isBitField() &&
975	"Cannot expand structure with bit-field members.");
976	Fields.push_back(Elt: FD);
977	}
978	}
979	return std::make_unique<RecordExpansion>(args: std::move(Bases),
980	args: std::move(Fields));
981	}
982	if (const ComplexType *CT = Ty->getAs<ComplexType>()) {
983	return std::make_unique<ComplexExpansion>(args: CT->getElementType());
984	}
985	return std::make_unique<NoExpansion>();
986	}
987
988	static int getExpansionSize(QualType Ty, const ASTContext &Context) {
989	auto Exp = getTypeExpansion(Ty, Context);
990	if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Val: Exp.get())) {
991	return CAExp->NumElts * getExpansionSize(CAExp->EltTy, Context);
992	}
993	if (auto RExp = dyn_cast<RecordExpansion>(Val: Exp.get())) {
994	int Res = 0;
995	for (auto BS : RExp->Bases)
996	Res += getExpansionSize(Ty: BS->getType(), Context);
997	for (auto FD : RExp->Fields)
998	Res += getExpansionSize(FD->getType(), Context);
999	return Res;
1000	}
1001	if (isa<ComplexExpansion>(Val: Exp.get()))
1002	return 2;
1003	assert(isa<NoExpansion>(Exp.get()));
1004	return 1;
1005	}
1006
1007	void
1008	CodeGenTypes::getExpandedTypes(QualType Ty,
1009	SmallVectorImpl<llvm::Type *>::iterator &TI) {
1010	auto Exp = getTypeExpansion(Ty, Context);
1011	if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Val: Exp.get())) {
1012	for (int i = 0, n = CAExp->NumElts; i < n; i++) {
1013	getExpandedTypes(Ty: CAExp->EltTy, TI);
1014	}
1015	} else if (auto RExp = dyn_cast<RecordExpansion>(Val: Exp.get())) {
1016	for (auto BS : RExp->Bases)
1017	getExpandedTypes(Ty: BS->getType(), TI);
1018	for (auto FD : RExp->Fields)
1019	getExpandedTypes(Ty: FD->getType(), TI);
1020	} else if (auto CExp = dyn_cast<ComplexExpansion>(Val: Exp.get())) {
1021	llvm::Type *EltTy = ConvertType(T: CExp->EltTy);
1022	*TI++ = EltTy;
1023	*TI++ = EltTy;
1024	} else {
1025	assert(isa<NoExpansion>(Exp.get()));
1026	*TI++ = ConvertType(T: Ty);
1027	}
1028	}
1029
1030	static void forConstantArrayExpansion(CodeGenFunction &CGF,
1031	ConstantArrayExpansion *CAE,
1032	Address BaseAddr,
1033	llvm::function_ref<void(Address)> Fn) {
1034	CharUnits EltSize = CGF.getContext().getTypeSizeInChars(CAE->EltTy);
1035	CharUnits EltAlign =
1036	BaseAddr.getAlignment().alignmentOfArrayElement(elementSize: EltSize);
1037	llvm::Type *EltTy = CGF.ConvertTypeForMem(T: CAE->EltTy);
1038
1039	for (int i = 0, n = CAE->NumElts; i < n; i++) {
1040	llvm::Value *EltAddr = CGF.Builder.CreateConstGEP2_32(
1041	Ty: BaseAddr.getElementType(), Ptr: BaseAddr.getPointer(), Idx0: 0, Idx1: i);
1042	Fn(Address(EltAddr, EltTy, EltAlign));
1043	}
1044	}
1045
1046	void CodeGenFunction::ExpandTypeFromArgs(QualType Ty, LValue LV,
1047	llvm::Function::arg_iterator &AI) {
1048	assert(LV.isSimple() &&
1049	"Unexpected non-simple lvalue during struct expansion.");
1050
1051	auto Exp = getTypeExpansion(Ty, Context: getContext());
1052	if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Val: Exp.get())) {
1053	forConstantArrayExpansion(
1054	CGF&: *this, CAE: CAExp, BaseAddr: LV.getAddress(CGF&: *this), Fn: [&](Address EltAddr) {
1055	LValue LV = MakeAddrLValue(EltAddr, CAExp->EltTy);
1056	ExpandTypeFromArgs(Ty: CAExp->EltTy, LV, AI);
1057	});
1058	} else if (auto RExp = dyn_cast<RecordExpansion>(Val: Exp.get())) {
1059	Address This = LV.getAddress(CGF&: *this);
1060	for (const CXXBaseSpecifier *BS : RExp->Bases) {
1061	// Perform a single step derived-to-base conversion.
1062	Address Base =
1063	GetAddressOfBaseClass(Value: This, Derived: Ty->getAsCXXRecordDecl(), PathBegin: &BS, PathEnd: &BS + 1,
1064	/*NullCheckValue=*/false, Loc: SourceLocation());
1065	LValue SubLV = MakeAddrLValue(Addr: Base, T: BS->getType());
1066
1067	// Recurse onto bases.
1068	ExpandTypeFromArgs(Ty: BS->getType(), LV: SubLV, AI);
1069	}
1070	for (auto FD : RExp->Fields) {
1071	// FIXME: What are the right qualifiers here?
1072	LValue SubLV = EmitLValueForFieldInitialization(Base: LV, Field: FD);
1073	ExpandTypeFromArgs(Ty: FD->getType(), LV: SubLV, AI);
1074	}
1075	} else if (isa<ComplexExpansion>(Val: Exp.get())) {
1076	auto realValue = &*AI++;
1077	auto imagValue = &*AI++;
1078	EmitStoreOfComplex(V: ComplexPairTy(realValue, imagValue), dest: LV, /*init*/ isInit: true);
1079	} else {
1080	// Call EmitStoreOfScalar except when the lvalue is a bitfield to emit a
1081	// primitive store.
1082	assert(isa<NoExpansion>(Exp.get()));
1083	llvm::Value *Arg = &*AI++;
1084	if (LV.isBitField()) {
1085	EmitStoreThroughLValue(Src: RValue::get(V: Arg), Dst: LV);
1086	} else {
1087	// TODO: currently there are some places are inconsistent in what LLVM
1088	// pointer type they use (see D118744). Once clang uses opaque pointers
1089	// all LLVM pointer types will be the same and we can remove this check.
1090	if (Arg->getType()->isPointerTy()) {
1091	Address Addr = LV.getAddress(CGF&: *this);
1092	Arg = Builder.CreateBitCast(V: Arg, DestTy: Addr.getElementType());
1093	}
1094	EmitStoreOfScalar(value: Arg, lvalue: LV);
1095	}
1096	}
1097	}
1098
1099	void CodeGenFunction::ExpandTypeToArgs(
1100	QualType Ty, CallArg Arg, llvm::FunctionType *IRFuncTy,
1101	SmallVectorImpl<llvm::Value *> &IRCallArgs, unsigned &IRCallArgPos) {
1102	auto Exp = getTypeExpansion(Ty, Context: getContext());
1103	if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Val: Exp.get())) {
1104	Address Addr = Arg.hasLValue() ? Arg.getKnownLValue().getAddress(CGF&: *this)
1105	: Arg.getKnownRValue().getAggregateAddress();
1106	forConstantArrayExpansion(
1107	CGF&: *this, CAE: CAExp, BaseAddr: Addr, Fn: [&](Address EltAddr) {
1108	CallArg EltArg = CallArg(
1109	convertTempToRValue(addr: EltAddr, type: CAExp->EltTy, Loc: SourceLocation()),
1110	CAExp->EltTy);
1111	ExpandTypeToArgs(Ty: CAExp->EltTy, Arg: EltArg, IRFuncTy, IRCallArgs,
1112	IRCallArgPos);
1113	});
1114	} else if (auto RExp = dyn_cast<RecordExpansion>(Val: Exp.get())) {
1115	Address This = Arg.hasLValue() ? Arg.getKnownLValue().getAddress(CGF&: *this)
1116	: Arg.getKnownRValue().getAggregateAddress();
1117	for (const CXXBaseSpecifier *BS : RExp->Bases) {
1118	// Perform a single step derived-to-base conversion.
1119	Address Base =
1120	GetAddressOfBaseClass(Value: This, Derived: Ty->getAsCXXRecordDecl(), PathBegin: &BS, PathEnd: &BS + 1,
1121	/*NullCheckValue=*/false, Loc: SourceLocation());
1122	CallArg BaseArg = CallArg(RValue::getAggregate(addr: Base), BS->getType());
1123
1124	// Recurse onto bases.
1125	ExpandTypeToArgs(Ty: BS->getType(), Arg: BaseArg, IRFuncTy, IRCallArgs,
1126	IRCallArgPos);
1127	}
1128
1129	LValue LV = MakeAddrLValue(Addr: This, T: Ty);
1130	for (auto FD : RExp->Fields) {
1131	CallArg FldArg =
1132	CallArg(EmitRValueForField(LV, FD, Loc: SourceLocation()), FD->getType());
1133	ExpandTypeToArgs(Ty: FD->getType(), Arg: FldArg, IRFuncTy, IRCallArgs,
1134	IRCallArgPos);
1135	}
1136	} else if (isa<ComplexExpansion>(Val: Exp.get())) {
1137	ComplexPairTy CV = Arg.getKnownRValue().getComplexVal();
1138	IRCallArgs[IRCallArgPos++] = CV.first;
1139	IRCallArgs[IRCallArgPos++] = CV.second;
1140	} else {
1141	assert(isa<NoExpansion>(Exp.get()));
1142	auto RV = Arg.getKnownRValue();
1143	assert(RV.isScalar() &&
1144	"Unexpected non-scalar rvalue during struct expansion.");
1145
1146	// Insert a bitcast as needed.
1147	llvm::Value *V = RV.getScalarVal();
1148	if (IRCallArgPos < IRFuncTy->getNumParams() &&
1149	V->getType() != IRFuncTy->getParamType(i: IRCallArgPos))
1150	V = Builder.CreateBitCast(V, DestTy: IRFuncTy->getParamType(i: IRCallArgPos));
1151
1152	IRCallArgs[IRCallArgPos++] = V;
1153	}
1154	}
1155
1156	/// Create a temporary allocation for the purposes of coercion.
1157	static Address CreateTempAllocaForCoercion(CodeGenFunction &CGF, llvm::Type *Ty,
1158	CharUnits MinAlign,
1159	const Twine &Name = "tmp") {
1160	// Don't use an alignment that's worse than what LLVM would prefer.
1161	auto PrefAlign = CGF.CGM.getDataLayout().getPrefTypeAlign(Ty);
1162	CharUnits Align = std::max(a: MinAlign, b: CharUnits::fromQuantity(Quantity: PrefAlign));
1163
1164	return CGF.CreateTempAlloca(Ty, align: Align, Name: Name + ".coerce");
1165	}
1166
1167	/// EnterStructPointerForCoercedAccess - Given a struct pointer that we are
1168	/// accessing some number of bytes out of it, try to gep into the struct to get
1169	/// at its inner goodness. Dive as deep as possible without entering an element
1170	/// with an in-memory size smaller than DstSize.
1171	static Address
1172	EnterStructPointerForCoercedAccess(Address SrcPtr,
1173	llvm::StructType *SrcSTy,
1174	uint64_t DstSize, CodeGenFunction &CGF) {
1175	// We can't dive into a zero-element struct.
1176	if (SrcSTy->getNumElements() == 0) return SrcPtr;
1177
1178	llvm::Type *FirstElt = SrcSTy->getElementType(N: 0);
1179
1180	// If the first elt is at least as large as what we're looking for, or if the
1181	// first element is the same size as the whole struct, we can enter it. The
1182	// comparison must be made on the store size and not the alloca size. Using
1183	// the alloca size may overstate the size of the load.
1184	uint64_t FirstEltSize =
1185	CGF.CGM.getDataLayout().getTypeStoreSize(Ty: FirstElt);
1186	if (FirstEltSize < DstSize &&
1187	FirstEltSize < CGF.CGM.getDataLayout().getTypeStoreSize(Ty: SrcSTy))
1188	return SrcPtr;
1189
1190	// GEP into the first element.
1191	SrcPtr = CGF.Builder.CreateStructGEP(Addr: SrcPtr, Index: 0, Name: "coerce.dive");
1192
1193	// If the first element is a struct, recurse.
1194	llvm::Type *SrcTy = SrcPtr.getElementType();
1195	if (llvm::StructType *SrcSTy = dyn_cast<llvm::StructType>(Val: SrcTy))
1196	return EnterStructPointerForCoercedAccess(SrcPtr, SrcSTy, DstSize, CGF);
1197
1198	return SrcPtr;
1199	}
1200
1201	/// CoerceIntOrPtrToIntOrPtr - Convert a value Val to the specific Ty where both
1202	/// are either integers or pointers. This does a truncation of the value if it
1203	/// is too large or a zero extension if it is too small.
1204	///
1205	/// This behaves as if the value were coerced through memory, so on big-endian
1206	/// targets the high bits are preserved in a truncation, while little-endian
1207	/// targets preserve the low bits.
1208	static llvm::Value *CoerceIntOrPtrToIntOrPtr(llvm::Value *Val,
1209	llvm::Type *Ty,
1210	CodeGenFunction &CGF) {
1211	if (Val->getType() == Ty)
1212	return Val;
1213
1214	if (isa<llvm::PointerType>(Val: Val->getType())) {
1215	// If this is Pointer->Pointer avoid conversion to and from int.
1216	if (isa<llvm::PointerType>(Val: Ty))
1217	return CGF.Builder.CreateBitCast(V: Val, DestTy: Ty, Name: "coerce.val");
1218
1219	// Convert the pointer to an integer so we can play with its width.
1220	Val = CGF.Builder.CreatePtrToInt(V: Val, DestTy: CGF.IntPtrTy, Name: "coerce.val.pi");
1221	}
1222
1223	llvm::Type *DestIntTy = Ty;
1224	if (isa<llvm::PointerType>(Val: DestIntTy))
1225	DestIntTy = CGF.IntPtrTy;
1226
1227	if (Val->getType() != DestIntTy) {
1228	const llvm::DataLayout &DL = CGF.CGM.getDataLayout();
1229	if (DL.isBigEndian()) {
1230	// Preserve the high bits on big-endian targets.
1231	// That is what memory coercion does.
1232	uint64_t SrcSize = DL.getTypeSizeInBits(Ty: Val->getType());
1233	uint64_t DstSize = DL.getTypeSizeInBits(Ty: DestIntTy);
1234
1235	if (SrcSize > DstSize) {
1236	Val = CGF.Builder.CreateLShr(LHS: Val, RHS: SrcSize - DstSize, Name: "coerce.highbits");
1237	Val = CGF.Builder.CreateTrunc(V: Val, DestTy: DestIntTy, Name: "coerce.val.ii");
1238	} else {
1239	Val = CGF.Builder.CreateZExt(V: Val, DestTy: DestIntTy, Name: "coerce.val.ii");
1240	Val = CGF.Builder.CreateShl(LHS: Val, RHS: DstSize - SrcSize, Name: "coerce.highbits");
1241	}
1242	} else {
1243	// Little-endian targets preserve the low bits. No shifts required.
1244	Val = CGF.Builder.CreateIntCast(V: Val, DestTy: DestIntTy, isSigned: false, Name: "coerce.val.ii");
1245	}
1246	}
1247
1248	if (isa<llvm::PointerType>(Val: Ty))
1249	Val = CGF.Builder.CreateIntToPtr(V: Val, DestTy: Ty, Name: "coerce.val.ip");
1250	return Val;
1251	}
1252
1253
1254
1255	/// CreateCoercedLoad - Create a load from \arg SrcPtr interpreted as
1256	/// a pointer to an object of type \arg Ty, known to be aligned to
1257	/// \arg SrcAlign bytes.
1258	///
1259	/// This safely handles the case when the src type is smaller than the
1260	/// destination type; in this situation the values of bits which not
1261	/// present in the src are undefined.
1262	static llvm::Value *CreateCoercedLoad(Address Src, llvm::Type *Ty,
1263	CodeGenFunction &CGF) {
1264	llvm::Type *SrcTy = Src.getElementType();
1265
1266	// If SrcTy and Ty are the same, just do a load.
1267	if (SrcTy == Ty)
1268	return CGF.Builder.CreateLoad(Addr: Src);
1269
1270	llvm::TypeSize DstSize = CGF.CGM.getDataLayout().getTypeAllocSize(Ty);
1271
1272	if (llvm::StructType *SrcSTy = dyn_cast<llvm::StructType>(Val: SrcTy)) {
1273	Src = EnterStructPointerForCoercedAccess(SrcPtr: Src, SrcSTy,
1274	DstSize: DstSize.getFixedValue(), CGF);
1275	SrcTy = Src.getElementType();
1276	}
1277
1278	llvm::TypeSize SrcSize = CGF.CGM.getDataLayout().getTypeAllocSize(Ty: SrcTy);
1279
1280	// If the source and destination are integer or pointer types, just do an
1281	// extension or truncation to the desired type.
1282	if ((isa<llvm::IntegerType>(Val: Ty) || isa<llvm::PointerType>(Val: Ty)) &&
1283	(isa<llvm::IntegerType>(Val: SrcTy) || isa<llvm::PointerType>(Val: SrcTy))) {
1284	llvm::Value *Load = CGF.Builder.CreateLoad(Addr: Src);
1285	return CoerceIntOrPtrToIntOrPtr(Val: Load, Ty, CGF);
1286	}
1287
1288	// If load is legal, just bitcast the src pointer.
1289	if (!SrcSize.isScalable() && !DstSize.isScalable() &&
1290	SrcSize.getFixedValue() >= DstSize.getFixedValue()) {
1291	// Generally SrcSize is never greater than DstSize, since this means we are
1292	// losing bits. However, this can happen in cases where the structure has
1293	// additional padding, for example due to a user specified alignment.
1294	//
1295	// FIXME: Assert that we aren't truncating non-padding bits when have access
1296	// to that information.
1297	Src = Src.withElementType(ElemTy: Ty);
1298	return CGF.Builder.CreateLoad(Addr: Src);
1299	}
1300
1301	// If coercing a fixed vector to a scalable vector for ABI compatibility, and
1302	// the types match, use the llvm.vector.insert intrinsic to perform the
1303	// conversion.
1304	if (auto *ScalableDstTy = dyn_cast<llvm::ScalableVectorType>(Val: Ty)) {
1305	if (auto *FixedSrcTy = dyn_cast<llvm::FixedVectorType>(Val: SrcTy)) {
1306	// If we are casting a fixed i8 vector to a scalable i1 predicate
1307	// vector, use a vector insert and bitcast the result.
1308	if (ScalableDstTy->getElementType()->isIntegerTy(Bitwidth: 1) &&
1309	ScalableDstTy->getElementCount().isKnownMultipleOf(RHS: 8) &&
1310	FixedSrcTy->getElementType()->isIntegerTy(Bitwidth: 8)) {
1311	ScalableDstTy = llvm::ScalableVectorType::get(
1312	ElementType: FixedSrcTy->getElementType(),
1313	MinNumElts: ScalableDstTy->getElementCount().getKnownMinValue() / 8);
1314	}
1315	if (ScalableDstTy->getElementType() == FixedSrcTy->getElementType()) {
1316	auto *Load = CGF.Builder.CreateLoad(Addr: Src);
1317	auto *UndefVec = llvm::UndefValue::get(T: ScalableDstTy);
1318	auto *Zero = llvm::Constant::getNullValue(Ty: CGF.CGM.Int64Ty);
1319	llvm::Value *Result = CGF.Builder.CreateInsertVector(
1320	DstType: ScalableDstTy, SrcVec: UndefVec, SubVec: Load, Idx: Zero, Name: "cast.scalable");
1321	if (ScalableDstTy != Ty)
1322	Result = CGF.Builder.CreateBitCast(V: Result, DestTy: Ty);
1323	return Result;
1324	}
1325	}
1326	}
1327
1328	// Otherwise do coercion through memory. This is stupid, but simple.
1329	Address Tmp =
1330	CreateTempAllocaForCoercion(CGF, Ty, MinAlign: Src.getAlignment(), Name: Src.getName());
1331	CGF.Builder.CreateMemCpy(
1332	Dst: Tmp.getPointer(), DstAlign: Tmp.getAlignment().getAsAlign(), Src: Src.getPointer(),
1333	SrcAlign: Src.getAlignment().getAsAlign(),
1334	Size: llvm::ConstantInt::get(Ty: CGF.IntPtrTy, V: SrcSize.getKnownMinValue()));
1335	return CGF.Builder.CreateLoad(Addr: Tmp);
1336	}
1337
1338	// Function to store a first-class aggregate into memory. We prefer to
1339	// store the elements rather than the aggregate to be more friendly to
1340	// fast-isel.
1341	// FIXME: Do we need to recurse here?
1342	void CodeGenFunction::EmitAggregateStore(llvm::Value *Val, Address Dest,
1343	bool DestIsVolatile) {
1344	// Prefer scalar stores to first-class aggregate stores.
1345	if (llvm::StructType *STy = dyn_cast<llvm::StructType>(Val: Val->getType())) {
1346	for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
1347	Address EltPtr = Builder.CreateStructGEP(Addr: Dest, Index: i);
1348	llvm::Value *Elt = Builder.CreateExtractValue(Agg: Val, Idxs: i);
1349	Builder.CreateStore(Val: Elt, Addr: EltPtr, IsVolatile: DestIsVolatile);
1350	}
1351	} else {
1352	Builder.CreateStore(Val, Addr: Dest, IsVolatile: DestIsVolatile);
1353	}
1354	}
1355
1356	/// CreateCoercedStore - Create a store to \arg DstPtr from \arg Src,
1357	/// where the source and destination may have different types. The
1358	/// destination is known to be aligned to \arg DstAlign bytes.
1359	///
1360	/// This safely handles the case when the src type is larger than the
1361	/// destination type; the upper bits of the src will be lost.
1362	static void CreateCoercedStore(llvm::Value *Src,
1363	Address Dst,
1364	bool DstIsVolatile,
1365	CodeGenFunction &CGF) {
1366	llvm::Type *SrcTy = Src->getType();
1367	llvm::Type *DstTy = Dst.getElementType();
1368	if (SrcTy == DstTy) {
1369	CGF.Builder.CreateStore(Val: Src, Addr: Dst, IsVolatile: DstIsVolatile);
1370	return;
1371	}
1372
1373	llvm::TypeSize SrcSize = CGF.CGM.getDataLayout().getTypeAllocSize(Ty: SrcTy);
1374
1375	if (llvm::StructType *DstSTy = dyn_cast<llvm::StructType>(Val: DstTy)) {
1376	Dst = EnterStructPointerForCoercedAccess(SrcPtr: Dst, SrcSTy: DstSTy,
1377	DstSize: SrcSize.getFixedValue(), CGF);
1378	DstTy = Dst.getElementType();
1379	}
1380
1381	llvm::PointerType *SrcPtrTy = llvm::dyn_cast<llvm::PointerType>(Val: SrcTy);
1382	llvm::PointerType *DstPtrTy = llvm::dyn_cast<llvm::PointerType>(Val: DstTy);
1383	if (SrcPtrTy && DstPtrTy &&
1384	SrcPtrTy->getAddressSpace() != DstPtrTy->getAddressSpace()) {
1385	Src = CGF.Builder.CreateAddrSpaceCast(V: Src, DestTy: DstTy);
1386	CGF.Builder.CreateStore(Val: Src, Addr: Dst, IsVolatile: DstIsVolatile);
1387	return;
1388	}
1389
1390	// If the source and destination are integer or pointer types, just do an
1391	// extension or truncation to the desired type.
1392	if ((isa<llvm::IntegerType>(Val: SrcTy) || isa<llvm::PointerType>(Val: SrcTy)) &&
1393	(isa<llvm::IntegerType>(Val: DstTy) || isa<llvm::PointerType>(Val: DstTy))) {
1394	Src = CoerceIntOrPtrToIntOrPtr(Val: Src, Ty: DstTy, CGF);
1395	CGF.Builder.CreateStore(Val: Src, Addr: Dst, IsVolatile: DstIsVolatile);
1396	return;
1397	}
1398
1399	llvm::TypeSize DstSize = CGF.CGM.getDataLayout().getTypeAllocSize(Ty: DstTy);
1400
1401	// If store is legal, just bitcast the src pointer.
1402	if (isa<llvm::ScalableVectorType>(Val: SrcTy) ||
1403	isa<llvm::ScalableVectorType>(Val: DstTy) ||
1404	SrcSize.getFixedValue() <= DstSize.getFixedValue()) {
1405	Dst = Dst.withElementType(ElemTy: SrcTy);
1406	CGF.EmitAggregateStore(Val: Src, Dest: Dst, DestIsVolatile: DstIsVolatile);
1407	} else {
1408	// Otherwise do coercion through memory. This is stupid, but
1409	// simple.
1410
1411	// Generally SrcSize is never greater than DstSize, since this means we are
1412	// losing bits. However, this can happen in cases where the structure has
1413	// additional padding, for example due to a user specified alignment.
1414	//
1415	// FIXME: Assert that we aren't truncating non-padding bits when have access
1416	// to that information.
1417	Address Tmp = CreateTempAllocaForCoercion(CGF, Ty: SrcTy, MinAlign: Dst.getAlignment());
1418	CGF.Builder.CreateStore(Val: Src, Addr: Tmp);
1419	CGF.Builder.CreateMemCpy(
1420	Dst: Dst.getPointer(), DstAlign: Dst.getAlignment().getAsAlign(), Src: Tmp.getPointer(),
1421	SrcAlign: Tmp.getAlignment().getAsAlign(),
1422	Size: llvm::ConstantInt::get(Ty: CGF.IntPtrTy, V: DstSize.getFixedValue()));
1423	}
1424	}
1425
1426	static Address emitAddressAtOffset(CodeGenFunction &CGF, Address addr,
1427	const ABIArgInfo &info) {
1428	if (unsigned offset = info.getDirectOffset()) {
1429	addr = addr.withElementType(ElemTy: CGF.Int8Ty);
1430	addr = CGF.Builder.CreateConstInBoundsByteGEP(Addr: addr,
1431	Offset: CharUnits::fromQuantity(Quantity: offset));
1432	addr = addr.withElementType(ElemTy: info.getCoerceToType());
1433	}
1434	return addr;
1435	}
1436
1437	namespace {
1438
1439	/// Encapsulates information about the way function arguments from
1440	/// CGFunctionInfo should be passed to actual LLVM IR function.
1441	class ClangToLLVMArgMapping {
1442	static const unsigned InvalidIndex = ~0U;
1443	unsigned InallocaArgNo;
1444	unsigned SRetArgNo;
1445	unsigned TotalIRArgs;
1446
1447	/// Arguments of LLVM IR function corresponding to single Clang argument.
1448	struct IRArgs {
1449	unsigned PaddingArgIndex;
1450	// Argument is expanded to IR arguments at positions
1451	// [FirstArgIndex, FirstArgIndex + NumberOfArgs).
1452	unsigned FirstArgIndex;
1453	unsigned NumberOfArgs;
1454
1455	IRArgs()
1456	: PaddingArgIndex(InvalidIndex), FirstArgIndex(InvalidIndex),
1457	NumberOfArgs(0) {}
1458	};
1459
1460	SmallVector<IRArgs, 8> ArgInfo;
1461
1462	public:
1463	ClangToLLVMArgMapping(const ASTContext &Context, const CGFunctionInfo &FI,
1464	bool OnlyRequiredArgs = false)
1465	: InallocaArgNo(InvalidIndex), SRetArgNo(InvalidIndex), TotalIRArgs(0),
1466	ArgInfo(OnlyRequiredArgs ? FI.getNumRequiredArgs() : FI.arg_size()) {
1467	construct(Context, FI, OnlyRequiredArgs);
1468	}
1469
1470	bool hasInallocaArg() const { return InallocaArgNo != InvalidIndex; }
1471	unsigned getInallocaArgNo() const {
1472	assert(hasInallocaArg());
1473	return InallocaArgNo;
1474	}
1475
1476	bool hasSRetArg() const { return SRetArgNo != InvalidIndex; }
1477	unsigned getSRetArgNo() const {
1478	assert(hasSRetArg());
1479	return SRetArgNo;
1480	}
1481
1482	unsigned totalIRArgs() const { return TotalIRArgs; }
1483
1484	bool hasPaddingArg(unsigned ArgNo) const {
1485	assert(ArgNo < ArgInfo.size());
1486	return ArgInfo[ArgNo].PaddingArgIndex != InvalidIndex;
1487	}
1488	unsigned getPaddingArgNo(unsigned ArgNo) const {
1489	assert(hasPaddingArg(ArgNo));
1490	return ArgInfo[ArgNo].PaddingArgIndex;
1491	}
1492
1493	/// Returns index of first IR argument corresponding to ArgNo, and their
1494	/// quantity.
1495	std::pair<unsigned, unsigned> getIRArgs(unsigned ArgNo) const {
1496	assert(ArgNo < ArgInfo.size());
1497	return std::make_pair(x: ArgInfo[ArgNo].FirstArgIndex,
1498	y: ArgInfo[ArgNo].NumberOfArgs);
1499	}
1500
1501	private:
1502	void construct(const ASTContext &Context, const CGFunctionInfo &FI,
1503	bool OnlyRequiredArgs);
1504	};
1505
1506	void ClangToLLVMArgMapping::construct(const ASTContext &Context,
1507	const CGFunctionInfo &FI,
1508	bool OnlyRequiredArgs) {
1509	unsigned IRArgNo = 0;
1510	bool SwapThisWithSRet = false;
1511	const ABIArgInfo &RetAI = FI.getReturnInfo();
1512
1513	if (RetAI.getKind() == ABIArgInfo::Indirect) {
1514	SwapThisWithSRet = RetAI.isSRetAfterThis();
1515	SRetArgNo = SwapThisWithSRet ? 1 : IRArgNo++;
1516	}
1517
1518	unsigned ArgNo = 0;
1519	unsigned NumArgs = OnlyRequiredArgs ? FI.getNumRequiredArgs() : FI.arg_size();
1520	for (CGFunctionInfo::const_arg_iterator I = FI.arg_begin(); ArgNo < NumArgs;
1521	++I, ++ArgNo) {
1522	assert(I != FI.arg_end());
1523	QualType ArgType = I->type;
1524	const ABIArgInfo &AI = I->info;
1525	// Collect data about IR arguments corresponding to Clang argument ArgNo.
1526	auto &IRArgs = ArgInfo[ArgNo];
1527
1528	if (AI.getPaddingType())
1529	IRArgs.PaddingArgIndex = IRArgNo++;
1530
1531	switch (AI.getKind()) {
1532	case ABIArgInfo::Extend:
1533	case ABIArgInfo::Direct: {
1534	// FIXME: handle sseregparm someday...
1535	llvm::StructType *STy = dyn_cast<llvm::StructType>(Val: AI.getCoerceToType());
1536	if (AI.isDirect() && AI.getCanBeFlattened() && STy) {
1537	IRArgs.NumberOfArgs = STy->getNumElements();
1538	} else {
1539	IRArgs.NumberOfArgs = 1;
1540	}
1541	break;
1542	}
1543	case ABIArgInfo::Indirect:
1544	case ABIArgInfo::IndirectAliased:
1545	IRArgs.NumberOfArgs = 1;
1546	break;
1547	case ABIArgInfo::Ignore:
1548	case ABIArgInfo::InAlloca:
1549	// ignore and inalloca doesn't have matching LLVM parameters.
1550	IRArgs.NumberOfArgs = 0;
1551	break;
1552	case ABIArgInfo::CoerceAndExpand:
1553	IRArgs.NumberOfArgs = AI.getCoerceAndExpandTypeSequence().size();
1554	break;
1555	case ABIArgInfo::Expand:
1556	IRArgs.NumberOfArgs = getExpansionSize(Ty: ArgType, Context);
1557	break;
1558	}
1559
1560	if (IRArgs.NumberOfArgs > 0) {
1561	IRArgs.FirstArgIndex = IRArgNo;
1562	IRArgNo += IRArgs.NumberOfArgs;
1563	}
1564
1565	// Skip over the sret parameter when it comes second. We already handled it
1566	// above.
1567	if (IRArgNo == 1 && SwapThisWithSRet)
1568	IRArgNo++;
1569	}
1570	assert(ArgNo == ArgInfo.size());
1571
1572	if (FI.usesInAlloca())
1573	InallocaArgNo = IRArgNo++;
1574
1575	TotalIRArgs = IRArgNo;
1576	}
1577	} // namespace
1578
1579	/***/
1580
1581	bool CodeGenModule::ReturnTypeUsesSRet(const CGFunctionInfo &FI) {
1582	const auto &RI = FI.getReturnInfo();
1583	return RI.isIndirect() || (RI.isInAlloca() && RI.getInAllocaSRet());
1584	}
1585
1586	bool CodeGenModule::ReturnSlotInterferesWithArgs(const CGFunctionInfo &FI) {
1587	return ReturnTypeUsesSRet(FI) &&
1588	getTargetCodeGenInfo().doesReturnSlotInterfereWithArgs();
1589	}
1590
1591	bool CodeGenModule::ReturnTypeUsesFPRet(QualType ResultType) {
1592	if (const BuiltinType *BT = ResultType->getAs<BuiltinType>()) {
1593	switch (BT->getKind()) {
1594	default:
1595	return false;
1596	case BuiltinType::Float:
1597	return getTarget().useObjCFPRetForRealType(T: FloatModeKind::Float);
1598	case BuiltinType::Double:
1599	return getTarget().useObjCFPRetForRealType(T: FloatModeKind::Double);
1600	case BuiltinType::LongDouble:
1601	return getTarget().useObjCFPRetForRealType(T: FloatModeKind::LongDouble);
1602	}
1603	}
1604
1605	return false;
1606	}
1607
1608	bool CodeGenModule::ReturnTypeUsesFP2Ret(QualType ResultType) {
1609	if (const ComplexType *CT = ResultType->getAs<ComplexType>()) {
1610	if (const BuiltinType *BT = CT->getElementType()->getAs<BuiltinType>()) {
1611	if (BT->getKind() == BuiltinType::LongDouble)
1612	return getTarget().useObjCFP2RetForComplexLongDouble();
1613	}
1614	}
1615
1616	return false;
1617	}
1618
1619	llvm::FunctionType *CodeGenTypes::GetFunctionType(GlobalDecl GD) {
1620	const CGFunctionInfo &FI = arrangeGlobalDeclaration(GD);
1621	return GetFunctionType(Info: FI);
1622	}
1623
1624	llvm::FunctionType *
1625	CodeGenTypes::GetFunctionType(const CGFunctionInfo &FI) {
1626
1627	bool Inserted = FunctionsBeingProcessed.insert(Ptr: &FI).second;
1628	(void)Inserted;
1629	assert(Inserted && "Recursively being processed?");
1630
1631	llvm::Type *resultType = nullptr;
1632	const ABIArgInfo &retAI = FI.getReturnInfo();
1633	switch (retAI.getKind()) {
1634	case ABIArgInfo::Expand:
1635	case ABIArgInfo::IndirectAliased:
1636	llvm_unreachable("Invalid ABI kind for return argument");
1637
1638	case ABIArgInfo::Extend:
1639	case ABIArgInfo::Direct:
1640	resultType = retAI.getCoerceToType();
1641	break;
1642
1643	case ABIArgInfo::InAlloca:
1644	if (retAI.getInAllocaSRet()) {
1645	// sret things on win32 aren't void, they return the sret pointer.
1646	QualType ret = FI.getReturnType();
1647	unsigned addressSpace = CGM.getTypes().getTargetAddressSpace(T: ret);
1648	resultType = llvm::PointerType::get(C&: getLLVMContext(), AddressSpace: addressSpace);
1649	} else {
1650	resultType = llvm::Type::getVoidTy(C&: getLLVMContext());
1651	}
1652	break;
1653
1654	case ABIArgInfo::Indirect:
1655	case ABIArgInfo::Ignore:
1656	resultType = llvm::Type::getVoidTy(C&: getLLVMContext());
1657	break;
1658
1659	case ABIArgInfo::CoerceAndExpand:
1660	resultType = retAI.getUnpaddedCoerceAndExpandType();
1661	break;
1662	}
1663
1664	ClangToLLVMArgMapping IRFunctionArgs(getContext(), FI, true);
1665	SmallVector<llvm::Type*, 8> ArgTypes(IRFunctionArgs.totalIRArgs());
1666
1667	// Add type for sret argument.
1668	if (IRFunctionArgs.hasSRetArg()) {
1669	QualType Ret = FI.getReturnType();
1670	unsigned AddressSpace = CGM.getTypes().getTargetAddressSpace(T: Ret);
1671	ArgTypes[IRFunctionArgs.getSRetArgNo()] =
1672	llvm::PointerType::get(C&: getLLVMContext(), AddressSpace);
1673	}
1674
1675	// Add type for inalloca argument.
1676	if (IRFunctionArgs.hasInallocaArg())
1677	ArgTypes[IRFunctionArgs.getInallocaArgNo()] =
1678	llvm::PointerType::getUnqual(C&: getLLVMContext());
1679
1680	// Add in all of the required arguments.
1681	unsigned ArgNo = 0;
1682	CGFunctionInfo::const_arg_iterator it = FI.arg_begin(),
1683	ie = it + FI.getNumRequiredArgs();
1684	for (; it != ie; ++it, ++ArgNo) {
1685	const ABIArgInfo &ArgInfo = it->info;
1686
1687	// Insert a padding type to ensure proper alignment.
1688	if (IRFunctionArgs.hasPaddingArg(ArgNo))
1689	ArgTypes[IRFunctionArgs.getPaddingArgNo(ArgNo)] =
1690	ArgInfo.getPaddingType();
1691
1692	unsigned FirstIRArg, NumIRArgs;
1693	std::tie(args&: FirstIRArg, args&: NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
1694
1695	switch (ArgInfo.getKind()) {
1696	case ABIArgInfo::Ignore:
1697	case ABIArgInfo::InAlloca:
1698	assert(NumIRArgs == 0);
1699	break;
1700
1701	case ABIArgInfo::Indirect:
1702	assert(NumIRArgs == 1);
1703	// indirect arguments are always on the stack, which is alloca addr space.
1704	ArgTypes[FirstIRArg] = llvm::PointerType::get(
1705	C&: getLLVMContext(), AddressSpace: CGM.getDataLayout().getAllocaAddrSpace());
1706	break;
1707	case ABIArgInfo::IndirectAliased:
1708	assert(NumIRArgs == 1);
1709	ArgTypes[FirstIRArg] = llvm::PointerType::get(
1710	C&: getLLVMContext(), AddressSpace: ArgInfo.getIndirectAddrSpace());
1711	break;
1712	case ABIArgInfo::Extend:
1713	case ABIArgInfo::Direct: {
1714	// Fast-isel and the optimizer generally like scalar values better than
1715	// FCAs, so we flatten them if this is safe to do for this argument.
1716	llvm::Type *argType = ArgInfo.getCoerceToType();
1717	llvm::StructType *st = dyn_cast<llvm::StructType>(Val: argType);
1718	if (st && ArgInfo.isDirect() && ArgInfo.getCanBeFlattened()) {
1719	assert(NumIRArgs == st->getNumElements());
1720	for (unsigned i = 0, e = st->getNumElements(); i != e; ++i)
1721	ArgTypes[FirstIRArg + i] = st->getElementType(N: i);
1722	} else {
1723	assert(NumIRArgs == 1);
1724	ArgTypes[FirstIRArg] = argType;
1725	}
1726	break;
1727	}
1728
1729	case ABIArgInfo::CoerceAndExpand: {
1730	auto ArgTypesIter = ArgTypes.begin() + FirstIRArg;
1731	for (auto *EltTy : ArgInfo.getCoerceAndExpandTypeSequence()) {
1732	*ArgTypesIter++ = EltTy;
1733	}
1734	assert(ArgTypesIter == ArgTypes.begin() + FirstIRArg + NumIRArgs);
1735	break;
1736	}
1737
1738	case ABIArgInfo::Expand:
1739	auto ArgTypesIter = ArgTypes.begin() + FirstIRArg;
1740	getExpandedTypes(Ty: it->type, TI&: ArgTypesIter);
1741	assert(ArgTypesIter == ArgTypes.begin() + FirstIRArg + NumIRArgs);
1742	break;
1743	}
1744	}
1745
1746	bool Erased = FunctionsBeingProcessed.erase(Ptr: &FI); (void)Erased;
1747	assert(Erased && "Not in set?");
1748
1749	return llvm::FunctionType::get(Result: resultType, Params: ArgTypes, isVarArg: FI.isVariadic());
1750	}
1751
1752	llvm::Type *CodeGenTypes::GetFunctionTypeForVTable(GlobalDecl GD) {
1753	const CXXMethodDecl *MD = cast<CXXMethodDecl>(Val: GD.getDecl());
1754	const FunctionProtoType *FPT = MD->getType()->castAs<FunctionProtoType>();
1755
1756	if (!isFuncTypeConvertible(FPT))
1757	return llvm::StructType::get(Context&: getLLVMContext());
1758
1759	return GetFunctionType(GD);
1760	}
1761
1762	static void AddAttributesFromFunctionProtoType(ASTContext &Ctx,
1763	llvm::AttrBuilder &FuncAttrs,
1764	const FunctionProtoType *FPT) {
1765	if (!FPT)
1766	return;
1767
1768	if (!isUnresolvedExceptionSpec(FPT->getExceptionSpecType()) &&
1769	FPT->isNothrow())
1770	FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
1771
1772	unsigned SMEBits = FPT->getAArch64SMEAttributes();
1773	if (SMEBits & FunctionType::SME_PStateSMEnabledMask)
1774	FuncAttrs.addAttribute(A: "aarch64_pstate_sm_enabled");
1775	if (SMEBits & FunctionType::SME_PStateSMCompatibleMask)
1776	FuncAttrs.addAttribute(A: "aarch64_pstate_sm_compatible");
1777
1778	// ZA
1779	if (FunctionType::getArmZAState(AttrBits: SMEBits) == FunctionType::ARM_Preserves)
1780	FuncAttrs.addAttribute(A: "aarch64_preserves_za");
1781	if (FunctionType::getArmZAState(AttrBits: SMEBits) == FunctionType::ARM_In)
1782	FuncAttrs.addAttribute(A: "aarch64_in_za");
1783	if (FunctionType::getArmZAState(AttrBits: SMEBits) == FunctionType::ARM_Out)
1784	FuncAttrs.addAttribute(A: "aarch64_out_za");
1785	if (FunctionType::getArmZAState(AttrBits: SMEBits) == FunctionType::ARM_InOut)
1786	FuncAttrs.addAttribute(A: "aarch64_inout_za");
1787
1788	// ZT0
1789	if (FunctionType::getArmZT0State(AttrBits: SMEBits) == FunctionType::ARM_Preserves)
1790	FuncAttrs.addAttribute(A: "aarch64_preserves_zt0");
1791	if (FunctionType::getArmZT0State(AttrBits: SMEBits) == FunctionType::ARM_In)
1792	FuncAttrs.addAttribute(A: "aarch64_in_zt0");
1793	if (FunctionType::getArmZT0State(AttrBits: SMEBits) == FunctionType::ARM_Out)
1794	FuncAttrs.addAttribute(A: "aarch64_out_zt0");
1795	if (FunctionType::getArmZT0State(AttrBits: SMEBits) == FunctionType::ARM_InOut)
1796	FuncAttrs.addAttribute(A: "aarch64_inout_zt0");
1797	}
1798
1799	static void AddAttributesFromAssumes(llvm::AttrBuilder &FuncAttrs,
1800	const Decl *Callee) {
1801	if (!Callee)
1802	return;
1803
1804	SmallVector<StringRef, 4> Attrs;
1805
1806	for (const AssumptionAttr *AA : Callee->specific_attrs<AssumptionAttr>())
1807	AA->getAssumption().split(Attrs, ",");
1808
1809	if (!Attrs.empty())
1810	FuncAttrs.addAttribute(A: llvm::AssumptionAttrKey,
1811	V: llvm::join(Begin: Attrs.begin(), End: Attrs.end(), Separator: ","));
1812	}
1813
1814	bool CodeGenModule::MayDropFunctionReturn(const ASTContext &Context,
1815	QualType ReturnType) const {
1816	// We can't just discard the return value for a record type with a
1817	// complex destructor or a non-trivially copyable type.
1818	if (const RecordType *RT =
1819	ReturnType.getCanonicalType()->getAs<RecordType>()) {
1820	if (const auto *ClassDecl = dyn_cast<CXXRecordDecl>(Val: RT->getDecl()))
1821	return ClassDecl->hasTrivialDestructor();
1822	}
1823	return ReturnType.isTriviallyCopyableType(Context);
1824	}
1825
1826	static bool HasStrictReturn(const CodeGenModule &Module, QualType RetTy,
1827	const Decl *TargetDecl) {
1828	// As-is msan can not tolerate noundef mismatch between caller and
1829	// implementation. Mismatch is possible for e.g. indirect calls from C-caller
1830	// into C++. Such mismatches lead to confusing false reports. To avoid
1831	// expensive workaround on msan we enforce initialization event in uncommon
1832	// cases where it's allowed.
1833	if (Module.getLangOpts().Sanitize.has(K: SanitizerKind::Memory))
1834	return true;
1835	// C++ explicitly makes returning undefined values UB. C's rule only applies
1836	// to used values, so we never mark them noundef for now.
1837	if (!Module.getLangOpts().CPlusPlus)
1838	return false;
1839	if (TargetDecl) {
1840	if (const FunctionDecl *FDecl = dyn_cast<FunctionDecl>(Val: TargetDecl)) {
1841	if (FDecl->isExternC())
1842	return false;
1843	} else if (const VarDecl *VDecl = dyn_cast<VarDecl>(Val: TargetDecl)) {
1844	// Function pointer.
1845	if (VDecl->isExternC())
1846	return false;
1847	}
1848	}
1849
1850	// We don't want to be too aggressive with the return checking, unless
1851	// it's explicit in the code opts or we're using an appropriate sanitizer.
1852	// Try to respect what the programmer intended.
1853	return Module.getCodeGenOpts().StrictReturn ||
1854	!Module.MayDropFunctionReturn(Context: Module.getContext(), ReturnType: RetTy) ||
1855	Module.getLangOpts().Sanitize.has(K: SanitizerKind::Return);
1856	}
1857
1858	/// Add denormal-fp-math and denormal-fp-math-f32 as appropriate for the
1859	/// requested denormal behavior, accounting for the overriding behavior of the
1860	/// -f32 case.
1861	static void addDenormalModeAttrs(llvm::DenormalMode FPDenormalMode,
1862	llvm::DenormalMode FP32DenormalMode,
1863	llvm::AttrBuilder &FuncAttrs) {
1864	if (FPDenormalMode != llvm::DenormalMode::getDefault())
1865	FuncAttrs.addAttribute(A: "denormal-fp-math", V: FPDenormalMode.str());
1866
1867	if (FP32DenormalMode != FPDenormalMode && FP32DenormalMode.isValid())
1868	FuncAttrs.addAttribute(A: "denormal-fp-math-f32", V: FP32DenormalMode.str());
1869	}
1870
1871	/// Add default attributes to a function, which have merge semantics under
1872	/// -mlink-builtin-bitcode and should not simply overwrite any existing
1873	/// attributes in the linked library.
1874	static void
1875	addMergableDefaultFunctionAttributes(const CodeGenOptions &CodeGenOpts,
1876	llvm::AttrBuilder &FuncAttrs) {
1877	addDenormalModeAttrs(FPDenormalMode: CodeGenOpts.FPDenormalMode, FP32DenormalMode: CodeGenOpts.FP32DenormalMode,
1878	FuncAttrs);
1879	}
1880
1881	static void getTrivialDefaultFunctionAttributes(
1882	StringRef Name, bool HasOptnone, const CodeGenOptions &CodeGenOpts,
1883	const LangOptions &LangOpts, bool AttrOnCallSite,
1884	llvm::AttrBuilder &FuncAttrs) {
1885	// OptimizeNoneAttr takes precedence over -Os or -Oz. No warning needed.
1886	if (!HasOptnone) {
1887	if (CodeGenOpts.OptimizeSize)
1888	FuncAttrs.addAttribute(llvm::Attribute::OptimizeForSize);
1889	if (CodeGenOpts.OptimizeSize == 2)
1890	FuncAttrs.addAttribute(llvm::Attribute::MinSize);
1891	}
1892
1893	if (CodeGenOpts.DisableRedZone)
1894	FuncAttrs.addAttribute(llvm::Attribute::NoRedZone);
1895	if (CodeGenOpts.IndirectTlsSegRefs)
1896	FuncAttrs.addAttribute(A: "indirect-tls-seg-refs");
1897	if (CodeGenOpts.NoImplicitFloat)
1898	FuncAttrs.addAttribute(llvm::Attribute::NoImplicitFloat);
1899
1900	if (AttrOnCallSite) {
1901	// Attributes that should go on the call site only.
1902	// FIXME: Look for 'BuiltinAttr' on the function rather than re-checking
1903	// the -fno-builtin-foo list.
1904	if (!CodeGenOpts.SimplifyLibCalls || LangOpts.isNoBuiltinFunc(Name))
1905	FuncAttrs.addAttribute(llvm::Attribute::NoBuiltin);
1906	if (!CodeGenOpts.TrapFuncName.empty())
1907	FuncAttrs.addAttribute(A: "trap-func-name", V: CodeGenOpts.TrapFuncName);
1908	} else {
1909	switch (CodeGenOpts.getFramePointer()) {
1910	case CodeGenOptions::FramePointerKind::None:
1911	// This is the default behavior.
1912	break;
1913	case CodeGenOptions::FramePointerKind::NonLeaf:
1914	case CodeGenOptions::FramePointerKind::All:
1915	FuncAttrs.addAttribute(A: "frame-pointer",
1916	V: CodeGenOptions::getFramePointerKindName(
1917	Kind: CodeGenOpts.getFramePointer()));
1918	}
1919
1920	if (CodeGenOpts.LessPreciseFPMAD)
1921	FuncAttrs.addAttribute(A: "less-precise-fpmad", V: "true");
1922
1923	if (CodeGenOpts.NullPointerIsValid)
1924	FuncAttrs.addAttribute(llvm::Attribute::NullPointerIsValid);
1925
1926	if (LangOpts.getDefaultExceptionMode() == LangOptions::FPE_Ignore)
1927	FuncAttrs.addAttribute(A: "no-trapping-math", V: "true");
1928
1929	// TODO: Are these all needed?
1930	// unsafe/inf/nan/nsz are handled by instruction-level FastMathFlags.
1931	if (LangOpts.NoHonorInfs)
1932	FuncAttrs.addAttribute(A: "no-infs-fp-math", V: "true");
1933	if (LangOpts.NoHonorNaNs)
1934	FuncAttrs.addAttribute(A: "no-nans-fp-math", V: "true");
1935	if (LangOpts.ApproxFunc)
1936	FuncAttrs.addAttribute(A: "approx-func-fp-math", V: "true");
1937	if (LangOpts.AllowFPReassoc && LangOpts.AllowRecip &&
1938	LangOpts.NoSignedZero && LangOpts.ApproxFunc &&
1939	(LangOpts.getDefaultFPContractMode() ==
1940	LangOptions::FPModeKind::FPM_Fast ||
1941	LangOpts.getDefaultFPContractMode() ==
1942	LangOptions::FPModeKind::FPM_FastHonorPragmas))
1943	FuncAttrs.addAttribute(A: "unsafe-fp-math", V: "true");
1944	if (CodeGenOpts.SoftFloat)
1945	FuncAttrs.addAttribute(A: "use-soft-float", V: "true");
1946	FuncAttrs.addAttribute(A: "stack-protector-buffer-size",
1947	V: llvm::utostr(X: CodeGenOpts.SSPBufferSize));
1948	if (LangOpts.NoSignedZero)
1949	FuncAttrs.addAttribute(A: "no-signed-zeros-fp-math", V: "true");
1950
1951	// TODO: Reciprocal estimate codegen options should apply to instructions?
1952	const std::vector<std::string> &Recips = CodeGenOpts.Reciprocals;
1953	if (!Recips.empty())
1954	FuncAttrs.addAttribute(A: "reciprocal-estimates",
1955	V: llvm::join(R: Recips, Separator: ","));
1956
1957	if (!CodeGenOpts.PreferVectorWidth.empty() &&
1958	CodeGenOpts.PreferVectorWidth != "none")
1959	FuncAttrs.addAttribute(A: "prefer-vector-width",
1960	V: CodeGenOpts.PreferVectorWidth);
1961
1962	if (CodeGenOpts.StackRealignment)
1963	FuncAttrs.addAttribute(A: "stackrealign");
1964	if (CodeGenOpts.Backchain)
1965	FuncAttrs.addAttribute(A: "backchain");
1966	if (CodeGenOpts.EnableSegmentedStacks)
1967	FuncAttrs.addAttribute(A: "split-stack");
1968
1969	if (CodeGenOpts.SpeculativeLoadHardening)
1970	FuncAttrs.addAttribute(llvm::Attribute::SpeculativeLoadHardening);
1971
1972	// Add zero-call-used-regs attribute.
1973	switch (CodeGenOpts.getZeroCallUsedRegs()) {
1974	case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::Skip:
1975	FuncAttrs.removeAttribute(A: "zero-call-used-regs");
1976	break;
1977	case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::UsedGPRArg:
1978	FuncAttrs.addAttribute(A: "zero-call-used-regs", V: "used-gpr-arg");
1979	break;
1980	case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::UsedGPR:
1981	FuncAttrs.addAttribute(A: "zero-call-used-regs", V: "used-gpr");
1982	break;
1983	case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::UsedArg:
1984	FuncAttrs.addAttribute(A: "zero-call-used-regs", V: "used-arg");
1985	break;
1986	case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::Used:
1987	FuncAttrs.addAttribute(A: "zero-call-used-regs", V: "used");
1988	break;
1989	case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::AllGPRArg:
1990	FuncAttrs.addAttribute(A: "zero-call-used-regs", V: "all-gpr-arg");
1991	break;
1992	case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::AllGPR:
1993	FuncAttrs.addAttribute(A: "zero-call-used-regs", V: "all-gpr");
1994	break;
1995	case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::AllArg:
1996	FuncAttrs.addAttribute(A: "zero-call-used-regs", V: "all-arg");
1997	break;
1998	case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::All:
1999	FuncAttrs.addAttribute(A: "zero-call-used-regs", V: "all");
2000	break;
2001	}
2002	}
2003
2004	if (LangOpts.assumeFunctionsAreConvergent()) {
2005	// Conservatively, mark all functions and calls in CUDA and OpenCL as
2006	// convergent (meaning, they may call an intrinsically convergent op, such
2007	// as __syncthreads() / barrier(), and so can't have certain optimizations
2008	// applied around them). LLVM will remove this attribute where it safely
2009	// can.
2010	FuncAttrs.addAttribute(llvm::Attribute::Convergent);
2011	}
2012
2013	// TODO: NoUnwind attribute should be added for other GPU modes HIP,
2014	// OpenMP offload. AFAIK, neither of them support exceptions in device code.
2015	if ((LangOpts.CUDA && LangOpts.CUDAIsDevice) || LangOpts.OpenCL ||
2016	LangOpts.SYCLIsDevice) {
2017	FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
2018	}
2019
2020	for (StringRef Attr : CodeGenOpts.DefaultFunctionAttrs) {
2021	StringRef Var, Value;
2022	std::tie(args&: Var, args&: Value) = Attr.split(Separator: '=');
2023	FuncAttrs.addAttribute(A: Var, V: Value);
2024	}
2025	}
2026
2027	/// Merges `target-features` from \TargetOpts and \F, and sets the result in
2028	/// \FuncAttr
2029	/// * features from \F are always kept
2030	/// * a feature from \TargetOpts is kept if itself and its opposite are absent
2031	/// from \F
2032	static void
2033	overrideFunctionFeaturesWithTargetFeatures(llvm::AttrBuilder &FuncAttr,
2034	const llvm::Function &F,
2035	const TargetOptions &TargetOpts) {
2036	auto FFeatures = F.getFnAttribute(Kind: "target-features");
2037
2038	llvm::StringSet<> MergedNames;
2039	SmallVector<StringRef> MergedFeatures;
2040	MergedFeatures.reserve(N: TargetOpts.Features.size());
2041
2042	auto AddUnmergedFeatures = [&](auto &&FeatureRange) {
2043	for (StringRef Feature : FeatureRange) {
2044	if (Feature.empty())
2045	continue;
2046	assert(Feature[0] == '+' || Feature[0] == '-');
2047	StringRef Name = Feature.drop_front(N: 1);
2048	bool Merged = !MergedNames.insert(key: Name).second;
2049	if (!Merged)
2050	MergedFeatures.push_back(Elt: Feature);
2051	}
2052	};
2053
2054	if (FFeatures.isValid())
2055	AddUnmergedFeatures(llvm::split(Str: FFeatures.getValueAsString(), Separator: ','));
2056	AddUnmergedFeatures(TargetOpts.Features);
2057
2058	if (!MergedFeatures.empty()) {
2059	llvm::sort(C&: MergedFeatures);
2060	FuncAttr.addAttribute(A: "target-features", V: llvm::join(R&: MergedFeatures, Separator: ","));
2061	}
2062	}
2063
2064	void CodeGen::mergeDefaultFunctionDefinitionAttributes(
2065	llvm::Function &F, const CodeGenOptions &CodeGenOpts,
2066	const LangOptions &LangOpts, const TargetOptions &TargetOpts,
2067	bool WillInternalize) {
2068
2069	llvm::AttrBuilder FuncAttrs(F.getContext());
2070	// Here we only extract the options that are relevant compared to the version
2071	// from GetCPUAndFeaturesAttributes.
2072	if (!TargetOpts.CPU.empty())
2073	FuncAttrs.addAttribute(A: "target-cpu", V: TargetOpts.CPU);
2074	if (!TargetOpts.TuneCPU.empty())
2075	FuncAttrs.addAttribute(A: "tune-cpu", V: TargetOpts.TuneCPU);
2076
2077	::getTrivialDefaultFunctionAttributes(Name: F.getName(), HasOptnone: F.hasOptNone(),
2078	CodeGenOpts, LangOpts,
2079	/*AttrOnCallSite=*/false, FuncAttrs);
2080
2081	if (!WillInternalize && F.isInterposable()) {
2082	// Do not promote "dynamic" denormal-fp-math to this translation unit's
2083	// setting for weak functions that won't be internalized. The user has no
2084	// real control for how builtin bitcode is linked, so we shouldn't assume
2085	// later copies will use a consistent mode.
2086	F.addFnAttrs(Attrs: FuncAttrs);
2087	return;
2088	}
2089
2090	llvm::AttributeMask AttrsToRemove;
2091
2092	llvm::DenormalMode DenormModeToMerge = F.getDenormalModeRaw();
2093	llvm::DenormalMode DenormModeToMergeF32 = F.getDenormalModeF32Raw();
2094	llvm::DenormalMode Merged =
2095	CodeGenOpts.FPDenormalMode.mergeCalleeMode(Callee: DenormModeToMerge);
2096	llvm::DenormalMode MergedF32 = CodeGenOpts.FP32DenormalMode;
2097
2098	if (DenormModeToMergeF32.isValid()) {
2099	MergedF32 =
2100	CodeGenOpts.FP32DenormalMode.mergeCalleeMode(Callee: DenormModeToMergeF32);
2101	}
2102
2103	if (Merged == llvm::DenormalMode::getDefault()) {
2104	AttrsToRemove.addAttribute(A: "denormal-fp-math");
2105	} else if (Merged != DenormModeToMerge) {
2106	// Overwrite existing attribute
2107	FuncAttrs.addAttribute(A: "denormal-fp-math",
2108	V: CodeGenOpts.FPDenormalMode.str());
2109	}
2110
2111	if (MergedF32 == llvm::DenormalMode::getDefault()) {
2112	AttrsToRemove.addAttribute(A: "denormal-fp-math-f32");
2113	} else if (MergedF32 != DenormModeToMergeF32) {
2114	// Overwrite existing attribute
2115	FuncAttrs.addAttribute(A: "denormal-fp-math-f32",
2116	V: CodeGenOpts.FP32DenormalMode.str());
2117	}
2118
2119	F.removeFnAttrs(Attrs: AttrsToRemove);
2120	addDenormalModeAttrs(FPDenormalMode: Merged, FP32DenormalMode: MergedF32, FuncAttrs);
2121
2122	overrideFunctionFeaturesWithTargetFeatures(FuncAttr&: FuncAttrs, F, TargetOpts);
2123
2124	F.addFnAttrs(Attrs: FuncAttrs);
2125	}
2126
2127	void CodeGenModule::getTrivialDefaultFunctionAttributes(
2128	StringRef Name, bool HasOptnone, bool AttrOnCallSite,
2129	llvm::AttrBuilder &FuncAttrs) {
2130	::getTrivialDefaultFunctionAttributes(Name, HasOptnone, CodeGenOpts: getCodeGenOpts(),
2131	LangOpts: getLangOpts(), AttrOnCallSite,
2132	FuncAttrs);
2133	}
2134
2135	void CodeGenModule::getDefaultFunctionAttributes(StringRef Name,
2136	bool HasOptnone,
2137	bool AttrOnCallSite,
2138	llvm::AttrBuilder &FuncAttrs) {
2139	getTrivialDefaultFunctionAttributes(Name, HasOptnone, AttrOnCallSite,
2140	FuncAttrs);
2141	// If we're just getting the default, get the default values for mergeable
2142	// attributes.
2143	if (!AttrOnCallSite)
2144	addMergableDefaultFunctionAttributes(CodeGenOpts, FuncAttrs);
2145	}
2146
2147	void CodeGenModule::addDefaultFunctionDefinitionAttributes(
2148	llvm::AttrBuilder &attrs) {
2149	getDefaultFunctionAttributes(/*function name*/ Name: "", /*optnone*/ HasOptnone: false,
2150	/*for call*/ AttrOnCallSite: false, FuncAttrs&: attrs);
2151	GetCPUAndFeaturesAttributes(GD: GlobalDecl(), AttrBuilder&: attrs);
2152	}
2153
2154	static void addNoBuiltinAttributes(llvm::AttrBuilder &FuncAttrs,
2155	const LangOptions &LangOpts,
2156	const NoBuiltinAttr *NBA = nullptr) {
2157	auto AddNoBuiltinAttr = [&FuncAttrs](StringRef BuiltinName) {
2158	SmallString<32> AttributeName;
2159	AttributeName += "no-builtin-";
2160	AttributeName += BuiltinName;
2161	FuncAttrs.addAttribute(A: AttributeName);
2162	};
2163
2164	// First, handle the language options passed through -fno-builtin.
2165	if (LangOpts.NoBuiltin) {
2166	// -fno-builtin disables them all.
2167	FuncAttrs.addAttribute(A: "no-builtins");
2168	return;
2169	}
2170
2171	// Then, add attributes for builtins specified through -fno-builtin-<name>.
2172	llvm::for_each(Range: LangOpts.NoBuiltinFuncs, F: AddNoBuiltinAttr);
2173
2174	// Now, let's check the __attribute__((no_builtin("...")) attribute added to
2175	// the source.
2176	if (!NBA)
2177	return;
2178
2179	// If there is a wildcard in the builtin names specified through the
2180	// attribute, disable them all.
2181	if (llvm::is_contained(NBA->builtinNames(), "*")) {
2182	FuncAttrs.addAttribute(A: "no-builtins");
2183	return;
2184	}
2185
2186	// And last, add the rest of the builtin names.
2187	llvm::for_each(NBA->builtinNames(), AddNoBuiltinAttr);
2188	}
2189
2190	static bool DetermineNoUndef(QualType QTy, CodeGenTypes &Types,
2191	const llvm::DataLayout &DL, const ABIArgInfo &AI,
2192	bool CheckCoerce = true) {
2193	llvm::Type *Ty = Types.ConvertTypeForMem(T: QTy);
2194	if (AI.getKind() == ABIArgInfo::Indirect ||
2195	AI.getKind() == ABIArgInfo::IndirectAliased)
2196	return true;
2197	if (AI.getKind() == ABIArgInfo::Extend)
2198	return true;
2199	if (!DL.typeSizeEqualsStoreSize(Ty))
2200	// TODO: This will result in a modest amount of values not marked noundef
2201	// when they could be. We care about values that *invisibly* contain undef
2202	// bits from the perspective of LLVM IR.
2203	return false;
2204	if (CheckCoerce && AI.canHaveCoerceToType()) {
2205	llvm::Type *CoerceTy = AI.getCoerceToType();
2206	if (llvm::TypeSize::isKnownGT(LHS: DL.getTypeSizeInBits(Ty: CoerceTy),
2207	RHS: DL.getTypeSizeInBits(Ty)))
2208	// If we're coercing to a type with a greater size than the canonical one,
2209	// we're introducing new undef bits.
2210	// Coercing to a type of smaller or equal size is ok, as we know that
2211	// there's no internal padding (typeSizeEqualsStoreSize).
2212	return false;
2213	}
2214	if (QTy->isBitIntType())
2215	return true;
2216	if (QTy->isReferenceType())
2217	return true;
2218	if (QTy->isNullPtrType())
2219	return false;
2220	if (QTy->isMemberPointerType())
2221	// TODO: Some member pointers are `noundef`, but it depends on the ABI. For
2222	// now, never mark them.
2223	return false;
2224	if (QTy->isScalarType()) {
2225	if (const ComplexType *Complex = dyn_cast<ComplexType>(Val&: QTy))
2226	return DetermineNoUndef(QTy: Complex->getElementType(), Types, DL, AI, CheckCoerce: false);
2227	return true;
2228	}
2229	if (const VectorType *Vector = dyn_cast<VectorType>(Val&: QTy))
2230	return DetermineNoUndef(QTy: Vector->getElementType(), Types, DL, AI, CheckCoerce: false);
2231	if (const MatrixType *Matrix = dyn_cast<MatrixType>(Val&: QTy))
2232	return DetermineNoUndef(QTy: Matrix->getElementType(), Types, DL, AI, CheckCoerce: false);
2233	if (const ArrayType *Array = dyn_cast<ArrayType>(Val&: QTy))
2234	return DetermineNoUndef(QTy: Array->getElementType(), Types, DL, AI, CheckCoerce: false);
2235
2236	// TODO: Some structs may be `noundef`, in specific situations.
2237	return false;
2238	}
2239
2240	/// Check if the argument of a function has maybe_undef attribute.
2241	static bool IsArgumentMaybeUndef(const Decl *TargetDecl,
2242	unsigned NumRequiredArgs, unsigned ArgNo) {
2243	const auto *FD = dyn_cast_or_null<FunctionDecl>(Val: TargetDecl);
2244	if (!FD)
2245	return false;
2246
2247	// Assume variadic arguments do not have maybe_undef attribute.
2248	if (ArgNo >= NumRequiredArgs)
2249	return false;
2250
2251	// Check if argument has maybe_undef attribute.
2252	if (ArgNo < FD->getNumParams()) {
2253	const ParmVarDecl *Param = FD->getParamDecl(i: ArgNo);
2254	if (Param && Param->hasAttr<MaybeUndefAttr>())
2255	return true;
2256	}
2257
2258	return false;
2259	}
2260
2261	/// Test if it's legal to apply nofpclass for the given parameter type and it's
2262	/// lowered IR type.
2263	static bool canApplyNoFPClass(const ABIArgInfo &AI, QualType ParamType,
2264	bool IsReturn) {
2265	// Should only apply to FP types in the source, not ABI promoted.
2266	if (!ParamType->hasFloatingRepresentation())
2267	return false;
2268
2269	// The promoted-to IR type also needs to support nofpclass.
2270	llvm::Type *IRTy = AI.getCoerceToType();
2271	if (llvm::AttributeFuncs::isNoFPClassCompatibleType(Ty: IRTy))
2272	return true;
2273
2274	if (llvm::StructType *ST = dyn_cast<llvm::StructType>(Val: IRTy)) {
2275	return !IsReturn && AI.getCanBeFlattened() &&
2276	llvm::all_of(Range: ST->elements(), P: [](llvm::Type *Ty) {
2277	return llvm::AttributeFuncs::isNoFPClassCompatibleType(Ty);
2278	});
2279	}
2280
2281	return false;
2282	}
2283
2284	/// Return the nofpclass mask that can be applied to floating-point parameters.
2285	static llvm::FPClassTest getNoFPClassTestMask(const LangOptions &LangOpts) {
2286	llvm::FPClassTest Mask = llvm::fcNone;
2287	if (LangOpts.NoHonorInfs)
2288	Mask |= llvm::fcInf;
2289	if (LangOpts.NoHonorNaNs)
2290	Mask |= llvm::fcNan;
2291	return Mask;
2292	}
2293
2294	void CodeGenModule::AdjustMemoryAttribute(StringRef Name,
2295	CGCalleeInfo CalleeInfo,
2296	llvm::AttributeList &Attrs) {
2297	if (Attrs.getMemoryEffects().getModRef() == llvm::ModRefInfo::NoModRef) {
2298	Attrs = Attrs.removeFnAttribute(getLLVMContext(), llvm::Attribute::Memory);
2299	llvm::Attribute MemoryAttr = llvm::Attribute::getWithMemoryEffects(
2300	Context&: getLLVMContext(), ME: llvm::MemoryEffects::writeOnly());
2301	Attrs = Attrs.addFnAttribute(C&: getLLVMContext(), Attr: MemoryAttr);
2302	}
2303	}
2304
2305	/// Construct the IR attribute list of a function or call.
2306	///
2307	/// When adding an attribute, please consider where it should be handled:
2308	///
2309	/// - getDefaultFunctionAttributes is for attributes that are essentially
2310	/// part of the global target configuration (but perhaps can be
2311	/// overridden on a per-function basis). Adding attributes there
2312	/// will cause them to also be set in frontends that build on Clang's
2313	/// target-configuration logic, as well as for code defined in library
2314	/// modules such as CUDA's libdevice.
2315	///
2316	/// - ConstructAttributeList builds on top of getDefaultFunctionAttributes
2317	/// and adds declaration-specific, convention-specific, and
2318	/// frontend-specific logic. The last is of particular importance:
2319	/// attributes that restrict how the frontend generates code must be
2320	/// added here rather than getDefaultFunctionAttributes.
2321	///
2322	void CodeGenModule::ConstructAttributeList(StringRef Name,
2323	const CGFunctionInfo &FI,
2324	CGCalleeInfo CalleeInfo,
2325	llvm::AttributeList &AttrList,
2326	unsigned &CallingConv,
2327	bool AttrOnCallSite, bool IsThunk) {
2328	llvm::AttrBuilder FuncAttrs(getLLVMContext());
2329	llvm::AttrBuilder RetAttrs(getLLVMContext());
2330
2331	// Collect function IR attributes from the CC lowering.
2332	// We'll collect the paramete and result attributes later.
2333	CallingConv = FI.getEffectiveCallingConvention();
2334	if (FI.isNoReturn())
2335	FuncAttrs.addAttribute(llvm::Attribute::NoReturn);
2336	if (FI.isCmseNSCall())
2337	FuncAttrs.addAttribute(A: "cmse_nonsecure_call");
2338
2339	// Collect function IR attributes from the callee prototype if we have one.
2340	AddAttributesFromFunctionProtoType(Ctx&: getContext(), FuncAttrs,
2341	FPT: CalleeInfo.getCalleeFunctionProtoType());
2342
2343	const Decl *TargetDecl = CalleeInfo.getCalleeDecl().getDecl();
2344
2345	// Attach assumption attributes to the declaration. If this is a call
2346	// site, attach assumptions from the caller to the call as well.
2347	AddAttributesFromAssumes(FuncAttrs, Callee: TargetDecl);
2348
2349	bool HasOptnone = false;
2350	// The NoBuiltinAttr attached to the target FunctionDecl.
2351	const NoBuiltinAttr *NBA = nullptr;
2352
2353	// Some ABIs may result in additional accesses to arguments that may
2354	// otherwise not be present.
2355	auto AddPotentialArgAccess = [&]() {
2356	llvm::Attribute A = FuncAttrs.getAttribute(llvm::Attribute::Memory);
2357	if (A.isValid())
2358	FuncAttrs.addMemoryAttr(ME: A.getMemoryEffects() |
2359	llvm::MemoryEffects::argMemOnly());
2360	};
2361
2362	// Collect function IR attributes based on declaration-specific
2363	// information.
2364	// FIXME: handle sseregparm someday...
2365	if (TargetDecl) {
2366	if (TargetDecl->hasAttr<ReturnsTwiceAttr>())
2367	FuncAttrs.addAttribute(llvm::Attribute::ReturnsTwice);
2368	if (TargetDecl->hasAttr<NoThrowAttr>())
2369	FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
2370	if (TargetDecl->hasAttr<NoReturnAttr>())
2371	FuncAttrs.addAttribute(llvm::Attribute::NoReturn);
2372	if (TargetDecl->hasAttr<ColdAttr>())
2373	FuncAttrs.addAttribute(llvm::Attribute::Cold);
2374	if (TargetDecl->hasAttr<HotAttr>())
2375	FuncAttrs.addAttribute(llvm::Attribute::Hot);
2376	if (TargetDecl->hasAttr<NoDuplicateAttr>())
2377	FuncAttrs.addAttribute(llvm::Attribute::NoDuplicate);
2378	if (TargetDecl->hasAttr<ConvergentAttr>())
2379	FuncAttrs.addAttribute(llvm::Attribute::Convergent);
2380
2381	if (const FunctionDecl *Fn = dyn_cast<FunctionDecl>(Val: TargetDecl)) {
2382	AddAttributesFromFunctionProtoType(
2383	getContext(), FuncAttrs, Fn->getType()->getAs<FunctionProtoType>());
2384	if (AttrOnCallSite && Fn->isReplaceableGlobalAllocationFunction()) {
2385	// A sane operator new returns a non-aliasing pointer.
2386	auto Kind = Fn->getDeclName().getCXXOverloadedOperator();
2387	if (getCodeGenOpts().AssumeSaneOperatorNew &&
2388	(Kind == OO_New || Kind == OO_Array_New))
2389	RetAttrs.addAttribute(llvm::Attribute::NoAlias);
2390	}
2391	const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(Val: Fn);
2392	const bool IsVirtualCall = MD && MD->isVirtual();
2393	// Don't use [[noreturn]], _Noreturn or [[no_builtin]] for a call to a
2394	// virtual function. These attributes are not inherited by overloads.
2395	if (!(AttrOnCallSite && IsVirtualCall)) {
2396	if (Fn->isNoReturn())
2397	FuncAttrs.addAttribute(llvm::Attribute::NoReturn);
2398	NBA = Fn->getAttr<NoBuiltinAttr>();
2399	}
2400	}
2401
2402	if (isa<FunctionDecl>(Val: TargetDecl) || isa<VarDecl>(Val: TargetDecl)) {
2403	// Only place nomerge attribute on call sites, never functions. This
2404	// allows it to work on indirect virtual function calls.
2405	if (AttrOnCallSite && TargetDecl->hasAttr<NoMergeAttr>())
2406	FuncAttrs.addAttribute(llvm::Attribute::NoMerge);
2407	}
2408
2409	// 'const', 'pure' and 'noalias' attributed functions are also nounwind.
2410	if (TargetDecl->hasAttr<ConstAttr>()) {
2411	FuncAttrs.addMemoryAttr(ME: llvm::MemoryEffects::none());
2412	FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
2413	// gcc specifies that 'const' functions have greater restrictions than
2414	// 'pure' functions, so they also cannot have infinite loops.
2415	FuncAttrs.addAttribute(llvm::Attribute::WillReturn);
2416	} else if (TargetDecl->hasAttr<PureAttr>()) {
2417	FuncAttrs.addMemoryAttr(ME: llvm::MemoryEffects::readOnly());
2418	FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
2419	// gcc specifies that 'pure' functions cannot have infinite loops.
2420	FuncAttrs.addAttribute(llvm::Attribute::WillReturn);
2421	} else if (TargetDecl->hasAttr<NoAliasAttr>()) {
2422	FuncAttrs.addMemoryAttr(ME: llvm::MemoryEffects::inaccessibleOrArgMemOnly());
2423	FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
2424	}
2425	if (TargetDecl->hasAttr<RestrictAttr>())
2426	RetAttrs.addAttribute(llvm::Attribute::NoAlias);
2427	if (TargetDecl->hasAttr<ReturnsNonNullAttr>() &&
2428	!CodeGenOpts.NullPointerIsValid)
2429	RetAttrs.addAttribute(llvm::Attribute::NonNull);
2430	if (TargetDecl->hasAttr<AnyX86NoCallerSavedRegistersAttr>())
2431	FuncAttrs.addAttribute(A: "no_caller_saved_registers");
2432	if (TargetDecl->hasAttr<AnyX86NoCfCheckAttr>())
2433	FuncAttrs.addAttribute(llvm::Attribute::NoCfCheck);
2434	if (TargetDecl->hasAttr<LeafAttr>())
2435	FuncAttrs.addAttribute(llvm::Attribute::NoCallback);
2436
2437	HasOptnone = TargetDecl->hasAttr<OptimizeNoneAttr>();
2438	if (auto *AllocSize = TargetDecl->getAttr<AllocSizeAttr>()) {
2439	std::optional<unsigned> NumElemsParam;
2440	if (AllocSize->getNumElemsParam().isValid())
2441	NumElemsParam = AllocSize->getNumElemsParam().getLLVMIndex();
2442	FuncAttrs.addAllocSizeAttr(ElemSizeArg: AllocSize->getElemSizeParam().getLLVMIndex(),
2443	NumElemsArg: NumElemsParam);
2444	}
2445
2446	if (TargetDecl->hasAttr<OpenCLKernelAttr>()) {
2447	if (getLangOpts().OpenCLVersion <= 120) {
2448	// OpenCL v1.2 Work groups are always uniform
2449	FuncAttrs.addAttribute(A: "uniform-work-group-size", V: "true");
2450	} else {
2451	// OpenCL v2.0 Work groups may be whether uniform or not.
2452	// '-cl-uniform-work-group-size' compile option gets a hint
2453	// to the compiler that the global work-size be a multiple of
2454	// the work-group size specified to clEnqueueNDRangeKernel
2455	// (i.e. work groups are uniform).
2456	FuncAttrs.addAttribute(
2457	A: "uniform-work-group-size",
2458	V: llvm::toStringRef(B: getLangOpts().OffloadUniformBlock));
2459	}
2460	}
2461
2462	if (TargetDecl->hasAttr<CUDAGlobalAttr>() &&
2463	getLangOpts().OffloadUniformBlock)
2464	FuncAttrs.addAttribute(A: "uniform-work-group-size", V: "true");
2465
2466	if (TargetDecl->hasAttr<ArmLocallyStreamingAttr>())
2467	FuncAttrs.addAttribute(A: "aarch64_pstate_sm_body");
2468	}
2469
2470	// Attach "no-builtins" attributes to:
2471	// * call sites: both `nobuiltin` and "no-builtins" or "no-builtin-<name>".
2472	// * definitions: "no-builtins" or "no-builtin-<name>" only.
2473	// The attributes can come from:
2474	// * LangOpts: -ffreestanding, -fno-builtin, -fno-builtin-<name>
2475	// * FunctionDecl attributes: __attribute__((no_builtin(...)))
2476	addNoBuiltinAttributes(FuncAttrs, getLangOpts(), NBA);
2477
2478	// Collect function IR attributes based on global settiings.
2479	getDefaultFunctionAttributes(Name, HasOptnone, AttrOnCallSite, FuncAttrs);
2480
2481	// Override some default IR attributes based on declaration-specific
2482	// information.
2483	if (TargetDecl) {
2484	if (TargetDecl->hasAttr<NoSpeculativeLoadHardeningAttr>())
2485	FuncAttrs.removeAttribute(llvm::Attribute::SpeculativeLoadHardening);
2486	if (TargetDecl->hasAttr<SpeculativeLoadHardeningAttr>())
2487	FuncAttrs.addAttribute(llvm::Attribute::SpeculativeLoadHardening);
2488	if (TargetDecl->hasAttr<NoSplitStackAttr>())
2489	FuncAttrs.removeAttribute(A: "split-stack");
2490	if (TargetDecl->hasAttr<ZeroCallUsedRegsAttr>()) {
2491	// A function "__attribute__((...))" overrides the command-line flag.
2492	auto Kind =
2493	TargetDecl->getAttr<ZeroCallUsedRegsAttr>()->getZeroCallUsedRegs();
2494	FuncAttrs.removeAttribute(A: "zero-call-used-regs");
2495	FuncAttrs.addAttribute(
2496	"zero-call-used-regs",
2497	ZeroCallUsedRegsAttr::ConvertZeroCallUsedRegsKindToStr(Kind));
2498	}
2499
2500	// Add NonLazyBind attribute to function declarations when -fno-plt
2501	// is used.
2502	// FIXME: what if we just haven't processed the function definition
2503	// yet, or if it's an external definition like C99 inline?
2504	if (CodeGenOpts.NoPLT) {
2505	if (auto *Fn = dyn_cast<FunctionDecl>(Val: TargetDecl)) {
2506	if (!Fn->isDefined() && !AttrOnCallSite) {
2507	FuncAttrs.addAttribute(llvm::Attribute::NonLazyBind);
2508	}
2509	}
2510	}
2511	}
2512
2513	// Add "sample-profile-suffix-elision-policy" attribute for internal linkage
2514	// functions with -funique-internal-linkage-names.
2515	if (TargetDecl && CodeGenOpts.UniqueInternalLinkageNames) {
2516	if (const auto *FD = dyn_cast_or_null<FunctionDecl>(Val: TargetDecl)) {
2517	if (!FD->isExternallyVisible())
2518	FuncAttrs.addAttribute(A: "sample-profile-suffix-elision-policy",
2519	V: "selected");
2520	}
2521	}
2522
2523	// Collect non-call-site function IR attributes from declaration-specific
2524	// information.
2525	if (!AttrOnCallSite) {
2526	if (TargetDecl && TargetDecl->hasAttr<CmseNSEntryAttr>())
2527	FuncAttrs.addAttribute(A: "cmse_nonsecure_entry");
2528
2529	// Whether tail calls are enabled.
2530	auto shouldDisableTailCalls = [&] {
2531	// Should this be honored in getDefaultFunctionAttributes?
2532	if (CodeGenOpts.DisableTailCalls)
2533	return true;
2534
2535	if (!TargetDecl)
2536	return false;
2537
2538	if (TargetDecl->hasAttr<DisableTailCallsAttr>() ||
2539	TargetDecl->hasAttr<AnyX86InterruptAttr>())
2540	return true;
2541
2542	if (CodeGenOpts.NoEscapingBlockTailCalls) {
2543	if (const auto *BD = dyn_cast<BlockDecl>(Val: TargetDecl))
2544	if (!BD->doesNotEscape())
2545	return true;
2546	}
2547
2548	return false;
2549	};
2550	if (shouldDisableTailCalls())
2551	FuncAttrs.addAttribute(A: "disable-tail-calls", V: "true");
2552
2553	// CPU/feature overrides. addDefaultFunctionDefinitionAttributes
2554	// handles these separately to set them based on the global defaults.
2555	GetCPUAndFeaturesAttributes(GD: CalleeInfo.getCalleeDecl(), AttrBuilder&: FuncAttrs);
2556	}
2557
2558	// Collect attributes from arguments and return values.
2559	ClangToLLVMArgMapping IRFunctionArgs(getContext(), FI);
2560
2561	QualType RetTy = FI.getReturnType();
2562	const ABIArgInfo &RetAI = FI.getReturnInfo();
2563	const llvm::DataLayout &DL = getDataLayout();
2564
2565	// Determine if the return type could be partially undef
2566	if (CodeGenOpts.EnableNoundefAttrs &&
2567	HasStrictReturn(Module: *this, RetTy, TargetDecl)) {
2568	if (!RetTy->isVoidType() && RetAI.getKind() != ABIArgInfo::Indirect &&
2569	DetermineNoUndef(RetTy, getTypes(), DL, RetAI))
2570	RetAttrs.addAttribute(llvm::Attribute::NoUndef);
2571	}
2572
2573	switch (RetAI.getKind()) {
2574	case ABIArgInfo::Extend:
2575	if (RetAI.isSignExt())
2576	RetAttrs.addAttribute(llvm::Attribute::SExt);
2577	else
2578	RetAttrs.addAttribute(llvm::Attribute::ZExt);
2579	[[fallthrough]];
2580	case ABIArgInfo::Direct:
2581	if (RetAI.getInReg())
2582	RetAttrs.addAttribute(llvm::Attribute::InReg);
2583
2584	if (canApplyNoFPClass(AI: RetAI, ParamType: RetTy, IsReturn: true))
2585	RetAttrs.addNoFPClassAttr(NoFPClassMask: getNoFPClassTestMask(LangOpts: getLangOpts()));
2586
2587	break;
2588	case ABIArgInfo::Ignore:
2589	break;
2590
2591	case ABIArgInfo::InAlloca:
2592	case ABIArgInfo::Indirect: {
2593	// inalloca and sret disable readnone and readonly
2594	AddPotentialArgAccess();
2595	break;
2596	}
2597
2598	case ABIArgInfo::CoerceAndExpand:
2599	break;
2600
2601	case ABIArgInfo::Expand:
2602	case ABIArgInfo::IndirectAliased:
2603	llvm_unreachable("Invalid ABI kind for return argument");
2604	}
2605
2606	if (!IsThunk) {
2607	// FIXME: fix this properly, https://reviews.llvm.org/D100388
2608	if (const auto *RefTy = RetTy->getAs<ReferenceType>()) {
2609	QualType PTy = RefTy->getPointeeType();
2610	if (!PTy->isIncompleteType() && PTy->isConstantSizeType())
2611	RetAttrs.addDereferenceableAttr(
2612	Bytes: getMinimumObjectSize(Ty: PTy).getQuantity());
2613	if (getTypes().getTargetAddressSpace(PTy) == 0 &&
2614	!CodeGenOpts.NullPointerIsValid)
2615	RetAttrs.addAttribute(llvm::Attribute::NonNull);
2616	if (PTy->isObjectType()) {
2617	llvm::Align Alignment =
2618	getNaturalPointeeTypeAlignment(T: RetTy).getAsAlign();
2619	RetAttrs.addAlignmentAttr(Align: Alignment);
2620	}
2621	}
2622	}
2623
2624	bool hasUsedSRet = false;
2625	SmallVector<llvm::AttributeSet, 4> ArgAttrs(IRFunctionArgs.totalIRArgs());
2626
2627	// Attach attributes to sret.
2628	if (IRFunctionArgs.hasSRetArg()) {
2629	llvm::AttrBuilder SRETAttrs(getLLVMContext());
2630	SRETAttrs.addStructRetAttr(Ty: getTypes().ConvertTypeForMem(T: RetTy));
2631	SRETAttrs.addAttribute(llvm::Attribute::Writable);
2632	SRETAttrs.addAttribute(llvm::Attribute::DeadOnUnwind);
2633	hasUsedSRet = true;
2634	if (RetAI.getInReg())
2635	SRETAttrs.addAttribute(llvm::Attribute::InReg);
2636	SRETAttrs.addAlignmentAttr(Align: RetAI.getIndirectAlign().getQuantity());
2637	ArgAttrs[IRFunctionArgs.getSRetArgNo()] =
2638	llvm::AttributeSet::get(C&: getLLVMContext(), B: SRETAttrs);
2639	}
2640
2641	// Attach attributes to inalloca argument.
2642	if (IRFunctionArgs.hasInallocaArg()) {
2643	llvm::AttrBuilder Attrs(getLLVMContext());
2644	Attrs.addInAllocaAttr(Ty: FI.getArgStruct());
2645	ArgAttrs[IRFunctionArgs.getInallocaArgNo()] =
2646	llvm::AttributeSet::get(C&: getLLVMContext(), B: Attrs);
2647	}
2648
2649	// Apply `nonnull`, `dereferencable(N)` and `align N` to the `this` argument,
2650	// unless this is a thunk function.
2651	// FIXME: fix this properly, https://reviews.llvm.org/D100388
2652	if (FI.isInstanceMethod() && !IRFunctionArgs.hasInallocaArg() &&
2653	!FI.arg_begin()->type->isVoidPointerType() && !IsThunk) {
2654	auto IRArgs = IRFunctionArgs.getIRArgs(ArgNo: 0);
2655
2656	assert(IRArgs.second == 1 && "Expected only a single `this` pointer.");
2657
2658	llvm::AttrBuilder Attrs(getLLVMContext());
2659
2660	QualType ThisTy =
2661	FI.arg_begin()->type.getTypePtr()->getPointeeType();
2662
2663	if (!CodeGenOpts.NullPointerIsValid &&
2664	getTypes().getTargetAddressSpace(T: FI.arg_begin()->type) == 0) {
2665	Attrs.addAttribute(llvm::Attribute::NonNull);
2666	Attrs.addDereferenceableAttr(Bytes: getMinimumObjectSize(Ty: ThisTy).getQuantity());
2667	} else {
2668	// FIXME dereferenceable should be correct here, regardless of
2669	// NullPointerIsValid. However, dereferenceable currently does not always
2670	// respect NullPointerIsValid and may imply nonnull and break the program.
2671	// See https://reviews.llvm.org/D66618 for discussions.
2672	Attrs.addDereferenceableOrNullAttr(
2673	Bytes: getMinimumObjectSize(
2674	Ty: FI.arg_begin()->type.castAs<PointerType>()->getPointeeType())
2675	.getQuantity());
2676	}
2677
2678	llvm::Align Alignment =
2679	getNaturalTypeAlignment(T: ThisTy, /*BaseInfo=*/nullptr,
2680	/*TBAAInfo=*/nullptr, /*forPointeeType=*/true)
2681	.getAsAlign();
2682	Attrs.addAlignmentAttr(Align: Alignment);
2683
2684	ArgAttrs[IRArgs.first] = llvm::AttributeSet::get(C&: getLLVMContext(), B: Attrs);
2685	}
2686
2687	unsigned ArgNo = 0;
2688	for (CGFunctionInfo::const_arg_iterator I = FI.arg_begin(),
2689	E = FI.arg_end();
2690	I != E; ++I, ++ArgNo) {
2691	QualType ParamType = I->type;
2692	const ABIArgInfo &AI = I->info;
2693	llvm::AttrBuilder Attrs(getLLVMContext());
2694
2695	// Add attribute for padding argument, if necessary.
2696	if (IRFunctionArgs.hasPaddingArg(ArgNo)) {
2697	if (AI.getPaddingInReg()) {
2698	ArgAttrs[IRFunctionArgs.getPaddingArgNo(ArgNo)] =
2699	llvm::AttributeSet::get(
2700	getLLVMContext(),
2701	llvm::AttrBuilder(getLLVMContext()).addAttribute(llvm::Attribute::InReg));
2702	}
2703	}
2704
2705	// Decide whether the argument we're handling could be partially undef
2706	if (CodeGenOpts.EnableNoundefAttrs &&
2707	DetermineNoUndef(QTy: ParamType, Types&: getTypes(), DL, AI)) {
2708	Attrs.addAttribute(llvm::Attribute::NoUndef);
2709	}
2710
2711	// 'restrict' -> 'noalias' is done in EmitFunctionProlog when we
2712	// have the corresponding parameter variable. It doesn't make
2713	// sense to do it here because parameters are so messed up.
2714	switch (AI.getKind()) {
2715	case ABIArgInfo::Extend:
2716	if (AI.isSignExt())
2717	Attrs.addAttribute(llvm::Attribute::SExt);
2718	else
2719	Attrs.addAttribute(llvm::Attribute::ZExt);
2720	[[fallthrough]];
2721	case ABIArgInfo::Direct:
2722	if (ArgNo == 0 && FI.isChainCall())
2723	Attrs.addAttribute(llvm::Attribute::Nest);
2724	else if (AI.getInReg())
2725	Attrs.addAttribute(llvm::Attribute::InReg);
2726	Attrs.addStackAlignmentAttr(Align: llvm::MaybeAlign(AI.getDirectAlign()));
2727
2728	if (canApplyNoFPClass(AI, ParamType, IsReturn: false))
2729	Attrs.addNoFPClassAttr(NoFPClassMask: getNoFPClassTestMask(LangOpts: getLangOpts()));
2730	break;
2731	case ABIArgInfo::Indirect: {
2732	if (AI.getInReg())
2733	Attrs.addAttribute(llvm::Attribute::InReg);
2734
2735	if (AI.getIndirectByVal())
2736	Attrs.addByValAttr(Ty: getTypes().ConvertTypeForMem(T: ParamType));
2737
2738	auto *Decl = ParamType->getAsRecordDecl();
2739	if (CodeGenOpts.PassByValueIsNoAlias && Decl &&
2740	Decl->getArgPassingRestrictions() ==
2741	RecordArgPassingKind::CanPassInRegs)
2742	// When calling the function, the pointer passed in will be the only
2743	// reference to the underlying object. Mark it accordingly.
2744	Attrs.addAttribute(llvm::Attribute::NoAlias);
2745
2746	// TODO: We could add the byref attribute if not byval, but it would
2747	// require updating many testcases.
2748
2749	CharUnits Align = AI.getIndirectAlign();
2750
2751	// In a byval argument, it is important that the required
2752	// alignment of the type is honored, as LLVM might be creating a
2753	// *new* stack object, and needs to know what alignment to give
2754	// it. (Sometimes it can deduce a sensible alignment on its own,
2755	// but not if clang decides it must emit a packed struct, or the
2756	// user specifies increased alignment requirements.)
2757	//
2758	// This is different from indirect *not* byval, where the object
2759	// exists already, and the align attribute is purely
2760	// informative.
2761	assert(!Align.isZero());
2762
2763	// For now, only add this when we have a byval argument.
2764	// TODO: be less lazy about updating test cases.
2765	if (AI.getIndirectByVal())
2766	Attrs.addAlignmentAttr(Align: Align.getQuantity());
2767
2768	// byval disables readnone and readonly.
2769	AddPotentialArgAccess();
2770	break;
2771	}
2772	case ABIArgInfo::IndirectAliased: {
2773	CharUnits Align = AI.getIndirectAlign();
2774	Attrs.addByRefAttr(Ty: getTypes().ConvertTypeForMem(T: ParamType));
2775	Attrs.addAlignmentAttr(Align: Align.getQuantity());
2776	break;
2777	}
2778	case ABIArgInfo::Ignore:
2779	case ABIArgInfo::Expand:
2780	case ABIArgInfo::CoerceAndExpand:
2781	break;
2782
2783	case ABIArgInfo::InAlloca:
2784	// inalloca disables readnone and readonly.
2785	AddPotentialArgAccess();
2786	continue;
2787	}
2788
2789	if (const auto *RefTy = ParamType->getAs<ReferenceType>()) {
2790	QualType PTy = RefTy->getPointeeType();
2791	if (!PTy->isIncompleteType() && PTy->isConstantSizeType())
2792	Attrs.addDereferenceableAttr(
2793	Bytes: getMinimumObjectSize(Ty: PTy).getQuantity());
2794	if (getTypes().getTargetAddressSpace(PTy) == 0 &&
2795	!CodeGenOpts.NullPointerIsValid)
2796	Attrs.addAttribute(llvm::Attribute::NonNull);
2797	if (PTy->isObjectType()) {
2798	llvm::Align Alignment =
2799	getNaturalPointeeTypeAlignment(T: ParamType).getAsAlign();
2800	Attrs.addAlignmentAttr(Align: Alignment);
2801	}
2802	}
2803
2804	// From OpenCL spec v3.0.10 section 6.3.5 Alignment of Types:
2805	// > For arguments to a __kernel function declared to be a pointer to a
2806	// > data type, the OpenCL compiler can assume that the pointee is always
2807	// > appropriately aligned as required by the data type.
2808	if (TargetDecl && TargetDecl->hasAttr<OpenCLKernelAttr>() &&
2809	ParamType->isPointerType()) {
2810	QualType PTy = ParamType->getPointeeType();
2811	if (!PTy->isIncompleteType() && PTy->isConstantSizeType()) {
2812	llvm::Align Alignment =
2813	getNaturalPointeeTypeAlignment(T: ParamType).getAsAlign();
2814	Attrs.addAlignmentAttr(Align: Alignment);
2815	}
2816	}
2817
2818	switch (FI.getExtParameterInfo(argIndex: ArgNo).getABI()) {
2819	case ParameterABI::Ordinary:
2820	break;
2821
2822	case ParameterABI::SwiftIndirectResult: {
2823	// Add 'sret' if we haven't already used it for something, but
2824	// only if the result is void.
2825	if (!hasUsedSRet && RetTy->isVoidType()) {
2826	Attrs.addStructRetAttr(Ty: getTypes().ConvertTypeForMem(T: ParamType));
2827	hasUsedSRet = true;
2828	}
2829
2830	// Add 'noalias' in either case.
2831	Attrs.addAttribute(llvm::Attribute::NoAlias);
2832
2833	// Add 'dereferenceable' and 'alignment'.
2834	auto PTy = ParamType->getPointeeType();
2835	if (!PTy->isIncompleteType() && PTy->isConstantSizeType()) {
2836	auto info = getContext().getTypeInfoInChars(PTy);
2837	Attrs.addDereferenceableAttr(Bytes: info.Width.getQuantity());
2838	Attrs.addAlignmentAttr(info.Align.getAsAlign());
2839	}
2840	break;
2841	}
2842
2843	case ParameterABI::SwiftErrorResult:
2844	Attrs.addAttribute(llvm::Attribute::SwiftError);
2845	break;
2846
2847	case ParameterABI::SwiftContext:
2848	Attrs.addAttribute(llvm::Attribute::SwiftSelf);
2849	break;
2850
2851	case ParameterABI::SwiftAsyncContext:
2852	Attrs.addAttribute(llvm::Attribute::SwiftAsync);
2853	break;
2854	}
2855
2856	if (FI.getExtParameterInfo(ArgNo).isNoEscape())
2857	Attrs.addAttribute(llvm::Attribute::NoCapture);
2858
2859	if (Attrs.hasAttributes()) {
2860	unsigned FirstIRArg, NumIRArgs;
2861	std::tie(args&: FirstIRArg, args&: NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
2862	for (unsigned i = 0; i < NumIRArgs; i++)
2863	ArgAttrs[FirstIRArg + i] = ArgAttrs[FirstIRArg + i].addAttributes(
2864	C&: getLLVMContext(), AS: llvm::AttributeSet::get(C&: getLLVMContext(), B: Attrs));
2865	}
2866	}
2867	assert(ArgNo == FI.arg_size());
2868
2869	AttrList = llvm::AttributeList::get(
2870	C&: getLLVMContext(), FnAttrs: llvm::AttributeSet::get(C&: getLLVMContext(), B: FuncAttrs),
2871	RetAttrs: llvm::AttributeSet::get(C&: getLLVMContext(), B: RetAttrs), ArgAttrs);
2872	}
2873
2874	/// An argument came in as a promoted argument; demote it back to its
2875	/// declared type.
2876	static llvm::Value *emitArgumentDemotion(CodeGenFunction &CGF,
2877	const VarDecl *var,
2878	llvm::Value *value) {
2879	llvm::Type *varType = CGF.ConvertType(var->getType());
2880
2881	// This can happen with promotions that actually don't change the
2882	// underlying type, like the enum promotions.
2883	if (value->getType() == varType) return value;
2884
2885	assert((varType->isIntegerTy() || varType->isFloatingPointTy())
2886	&& "unexpected promotion type");
2887
2888	if (isa<llvm::IntegerType>(Val: varType))
2889	return CGF.Builder.CreateTrunc(V: value, DestTy: varType, Name: "arg.unpromote");
2890
2891	return CGF.Builder.CreateFPCast(V: value, DestTy: varType, Name: "arg.unpromote");
2892	}
2893
2894	/// Returns the attribute (either parameter attribute, or function
2895	/// attribute), which declares argument ArgNo to be non-null.
2896	static const NonNullAttr *getNonNullAttr(const Decl *FD, const ParmVarDecl *PVD,
2897	QualType ArgType, unsigned ArgNo) {
2898	// FIXME: __attribute__((nonnull)) can also be applied to:
2899	// - references to pointers, where the pointee is known to be
2900	// nonnull (apparently a Clang extension)
2901	// - transparent unions containing pointers
2902	// In the former case, LLVM IR cannot represent the constraint. In
2903	// the latter case, we have no guarantee that the transparent union
2904	// is in fact passed as a pointer.
2905	if (!ArgType->isAnyPointerType() && !ArgType->isBlockPointerType())
2906	return nullptr;
2907	// First, check attribute on parameter itself.
2908	if (PVD) {
2909	if (auto ParmNNAttr = PVD->getAttr<NonNullAttr>())
2910	return ParmNNAttr;
2911	}
2912	// Check function attributes.
2913	if (!FD)
2914	return nullptr;
2915	for (const auto *NNAttr : FD->specific_attrs<NonNullAttr>()) {
2916	if (NNAttr->isNonNull(ArgNo))
2917	return NNAttr;
2918	}
2919	return nullptr;
2920	}
2921
2922	namespace {
2923	struct CopyBackSwiftError final : EHScopeStack::Cleanup {
2924	Address Temp;
2925	Address Arg;
2926	CopyBackSwiftError(Address temp, Address arg) : Temp(temp), Arg(arg) {}
2927	void Emit(CodeGenFunction &CGF, Flags flags) override {
2928	llvm::Value *errorValue = CGF.Builder.CreateLoad(Addr: Temp);
2929	CGF.Builder.CreateStore(Val: errorValue, Addr: Arg);
2930	}
2931	};
2932	}
2933
2934	void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
2935	llvm::Function *Fn,
2936	const FunctionArgList &Args) {
2937	if (CurCodeDecl && CurCodeDecl->hasAttr<NakedAttr>())
2938	// Naked functions don't have prologues.
2939	return;
2940
2941	// If this is an implicit-return-zero function, go ahead and
2942	// initialize the return value. TODO: it might be nice to have
2943	// a more general mechanism for this that didn't require synthesized
2944	// return statements.
2945	if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(Val: CurCodeDecl)) {
2946	if (FD->hasImplicitReturnZero()) {
2947	QualType RetTy = FD->getReturnType().getUnqualifiedType();
2948	llvm::Type* LLVMTy = CGM.getTypes().ConvertType(T: RetTy);
2949	llvm::Constant* Zero = llvm::Constant::getNullValue(Ty: LLVMTy);
2950	Builder.CreateStore(Val: Zero, Addr: ReturnValue);
2951	}
2952	}
2953
2954	// FIXME: We no longer need the types from FunctionArgList; lift up and
2955	// simplify.
2956
2957	ClangToLLVMArgMapping IRFunctionArgs(CGM.getContext(), FI);
2958	assert(Fn->arg_size() == IRFunctionArgs.totalIRArgs());
2959
2960	// If we're using inalloca, all the memory arguments are GEPs off of the last
2961	// parameter, which is a pointer to the complete memory area.
2962	Address ArgStruct = Address::invalid();
2963	if (IRFunctionArgs.hasInallocaArg())
2964	ArgStruct = Address(Fn->getArg(i: IRFunctionArgs.getInallocaArgNo()),
2965	FI.getArgStruct(), FI.getArgStructAlignment());
2966
2967	// Name the struct return parameter.
2968	if (IRFunctionArgs.hasSRetArg()) {
2969	auto AI = Fn->getArg(i: IRFunctionArgs.getSRetArgNo());
2970	AI->setName("agg.result");
2971	AI->addAttr(llvm::Attribute::NoAlias);
2972	}
2973
2974	// Track if we received the parameter as a pointer (indirect, byval, or
2975	// inalloca). If already have a pointer, EmitParmDecl doesn't need to copy it
2976	// into a local alloca for us.
2977	SmallVector<ParamValue, 16> ArgVals;
2978	ArgVals.reserve(N: Args.size());
2979
2980	// Create a pointer value for every parameter declaration. This usually
2981	// entails copying one or more LLVM IR arguments into an alloca. Don't push
2982	// any cleanups or do anything that might unwind. We do that separately, so
2983	// we can push the cleanups in the correct order for the ABI.
2984	assert(FI.arg_size() == Args.size() &&
2985	"Mismatch between function signature & arguments.");
2986	unsigned ArgNo = 0;
2987	CGFunctionInfo::const_arg_iterator info_it = FI.arg_begin();
2988	for (FunctionArgList::const_iterator i = Args.begin(), e = Args.end();
2989	i != e; ++i, ++info_it, ++ArgNo) {
2990	const VarDecl *Arg = *i;
2991	const ABIArgInfo &ArgI = info_it->info;
2992
2993	bool isPromoted =
2994	isa<ParmVarDecl>(Val: Arg) && cast<ParmVarDecl>(Val: Arg)->isKNRPromoted();
2995	// We are converting from ABIArgInfo type to VarDecl type directly, unless
2996	// the parameter is promoted. In this case we convert to
2997	// CGFunctionInfo::ArgInfo type with subsequent argument demotion.
2998	QualType Ty = isPromoted ? info_it->type : Arg->getType();
2999	assert(hasScalarEvaluationKind(Ty) ==
3000	hasScalarEvaluationKind(Arg->getType()));
3001
3002	unsigned FirstIRArg, NumIRArgs;
3003	std::tie(args&: FirstIRArg, args&: NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
3004
3005	switch (ArgI.getKind()) {
3006	case ABIArgInfo::InAlloca: {
3007	assert(NumIRArgs == 0);
3008	auto FieldIndex = ArgI.getInAllocaFieldIndex();
3009	Address V =
3010	Builder.CreateStructGEP(ArgStruct, FieldIndex, Arg->getName());
3011	if (ArgI.getInAllocaIndirect())
3012	V = Address(Builder.CreateLoad(Addr: V), ConvertTypeForMem(T: Ty),
3013	getContext().getTypeAlignInChars(T: Ty));
3014	ArgVals.push_back(Elt: ParamValue::forIndirect(addr: V));
3015	break;
3016	}
3017
3018	case ABIArgInfo::Indirect:
3019	case ABIArgInfo::IndirectAliased: {
3020	assert(NumIRArgs == 1);
3021	Address ParamAddr = Address(Fn->getArg(i: FirstIRArg), ConvertTypeForMem(T: Ty),
3022	ArgI.getIndirectAlign(), KnownNonNull);
3023
3024	if (!hasScalarEvaluationKind(T: Ty)) {
3025	// Aggregates and complex variables are accessed by reference. All we
3026	// need to do is realign the value, if requested. Also, if the address
3027	// may be aliased, copy it to ensure that the parameter variable is
3028	// mutable and has a unique adress, as C requires.
3029	Address V = ParamAddr;
3030	if (ArgI.getIndirectRealign() || ArgI.isIndirectAliased()) {
3031	Address AlignedTemp = CreateMemTemp(T: Ty, Name: "coerce");
3032
3033	// Copy from the incoming argument pointer to the temporary with the
3034	// appropriate alignment.
3035	//
3036	// FIXME: We should have a common utility for generating an aggregate
3037	// copy.
3038	CharUnits Size = getContext().getTypeSizeInChars(T: Ty);
3039	Builder.CreateMemCpy(
3040	Dst: AlignedTemp.getPointer(), DstAlign: AlignedTemp.getAlignment().getAsAlign(),
3041	Src: ParamAddr.getPointer(), SrcAlign: ParamAddr.getAlignment().getAsAlign(),
3042	Size: llvm::ConstantInt::get(Ty: IntPtrTy, V: Size.getQuantity()));
3043	V = AlignedTemp;
3044	}
3045	ArgVals.push_back(Elt: ParamValue::forIndirect(addr: V));
3046	} else {
3047	// Load scalar value from indirect argument.
3048	llvm::Value *V =
3049	EmitLoadOfScalar(ParamAddr, false, Ty, Arg->getBeginLoc());
3050
3051	if (isPromoted)
3052	V = emitArgumentDemotion(CGF&: *this, var: Arg, value: V);
3053	ArgVals.push_back(Elt: ParamValue::forDirect(value: V));
3054	}
3055	break;
3056	}
3057
3058	case ABIArgInfo::Extend:
3059	case ABIArgInfo::Direct: {
3060	auto AI = Fn->getArg(i: FirstIRArg);
3061	llvm::Type *LTy = ConvertType(Arg->getType());
3062
3063	// Prepare parameter attributes. So far, only attributes for pointer
3064	// parameters are prepared. See
3065	// http://llvm.org/docs/LangRef.html#paramattrs.
3066	if (ArgI.getDirectOffset() == 0 && LTy->isPointerTy() &&
3067	ArgI.getCoerceToType()->isPointerTy()) {
3068	assert(NumIRArgs == 1);
3069
3070	if (const ParmVarDecl *PVD = dyn_cast<ParmVarDecl>(Val: Arg)) {
3071	// Set `nonnull` attribute if any.
3072	if (getNonNullAttr(CurCodeDecl, PVD, PVD->getType(),
3073	PVD->getFunctionScopeIndex()) &&
3074	!CGM.getCodeGenOpts().NullPointerIsValid)
3075	AI->addAttr(llvm::Attribute::NonNull);
3076
3077	QualType OTy = PVD->getOriginalType();
3078	if (const auto *ArrTy =
3079	getContext().getAsConstantArrayType(T: OTy)) {
3080	// A C99 array parameter declaration with the static keyword also
3081	// indicates dereferenceability, and if the size is constant we can
3082	// use the dereferenceable attribute (which requires the size in
3083	// bytes).
3084	if (ArrTy->getSizeModifier() == ArraySizeModifier::Static) {
3085	QualType ETy = ArrTy->getElementType();
3086	llvm::Align Alignment =
3087	CGM.getNaturalTypeAlignment(T: ETy).getAsAlign();
3088	AI->addAttrs(B&: llvm::AttrBuilder(getLLVMContext()).addAlignmentAttr(Align: Alignment));
3089	uint64_t ArrSize = ArrTy->getSize().getZExtValue();
3090	if (!ETy->isIncompleteType() && ETy->isConstantSizeType() &&
3091	ArrSize) {
3092	llvm::AttrBuilder Attrs(getLLVMContext());
3093	Attrs.addDereferenceableAttr(
3094	Bytes: getContext().getTypeSizeInChars(T: ETy).getQuantity() *
3095	ArrSize);
3096	AI->addAttrs(B&: Attrs);
3097	} else if (getContext().getTargetInfo().getNullPointerValue(
3098	AddrSpace: ETy.getAddressSpace()) == 0 &&
3099	!CGM.getCodeGenOpts().NullPointerIsValid) {
3100	AI->addAttr(llvm::Attribute::NonNull);
3101	}
3102	}
3103	} else if (const auto *ArrTy =
3104	getContext().getAsVariableArrayType(T: OTy)) {
3105	// For C99 VLAs with the static keyword, we don't know the size so
3106	// we can't use the dereferenceable attribute, but in addrspace(0)
3107	// we know that it must be nonnull.
3108	if (ArrTy->getSizeModifier() == ArraySizeModifier::Static) {
3109	QualType ETy = ArrTy->getElementType();
3110	llvm::Align Alignment =
3111	CGM.getNaturalTypeAlignment(T: ETy).getAsAlign();
3112	AI->addAttrs(B&: llvm::AttrBuilder(getLLVMContext()).addAlignmentAttr(Align: Alignment));
3113	if (!getTypes().getTargetAddressSpace(ETy) &&
3114	!CGM.getCodeGenOpts().NullPointerIsValid)
3115	AI->addAttr(llvm::Attribute::NonNull);
3116	}
3117	}
3118
3119	// Set `align` attribute if any.
3120	const auto *AVAttr = PVD->getAttr<AlignValueAttr>();
3121	if (!AVAttr)
3122	if (const auto *TOTy = OTy->getAs<TypedefType>())
3123	AVAttr = TOTy->getDecl()->getAttr<AlignValueAttr>();
3124	if (AVAttr && !SanOpts.has(K: SanitizerKind::Alignment)) {
3125	// If alignment-assumption sanitizer is enabled, we do *not* add
3126	// alignment attribute here, but emit normal alignment assumption,
3127	// so the UBSAN check could function.
3128	llvm::ConstantInt *AlignmentCI =
3129	cast<llvm::ConstantInt>(EmitScalarExpr(E: AVAttr->getAlignment()));
3130	uint64_t AlignmentInt =
3131	AlignmentCI->getLimitedValue(Limit: llvm::Value::MaximumAlignment);
3132	if (AI->getParamAlign().valueOrOne() < AlignmentInt) {
3133	AI->removeAttr(llvm::Attribute::AttrKind::Alignment);
3134	AI->addAttrs(B&: llvm::AttrBuilder(getLLVMContext()).addAlignmentAttr(
3135	Align: llvm::Align(AlignmentInt)));
3136	}
3137	}
3138	}
3139
3140	// Set 'noalias' if an argument type has the `restrict` qualifier.
3141	if (Arg->getType().isRestrictQualified())
3142	AI->addAttr(llvm::Attribute::NoAlias);
3143	}
3144
3145	// Prepare the argument value. If we have the trivial case, handle it
3146	// with no muss and fuss.
3147	if (!isa<llvm::StructType>(Val: ArgI.getCoerceToType()) &&
3148	ArgI.getCoerceToType() == ConvertType(T: Ty) &&
3149	ArgI.getDirectOffset() == 0) {
3150	assert(NumIRArgs == 1);
3151
3152	// LLVM expects swifterror parameters to be used in very restricted
3153	// ways. Copy the value into a less-restricted temporary.
3154	llvm::Value *V = AI;
3155	if (FI.getExtParameterInfo(argIndex: ArgNo).getABI()
3156	== ParameterABI::SwiftErrorResult) {
3157	QualType pointeeTy = Ty->getPointeeType();
3158	assert(pointeeTy->isPointerType());
3159	Address temp =
3160	CreateMemTemp(pointeeTy, getPointerAlign(), "swifterror.temp");
3161	Address arg(V, ConvertTypeForMem(T: pointeeTy),
3162	getContext().getTypeAlignInChars(T: pointeeTy));
3163	llvm::Value *incomingErrorValue = Builder.CreateLoad(Addr: arg);
3164	Builder.CreateStore(Val: incomingErrorValue, Addr: temp);
3165	V = temp.getPointer();
3166
3167	// Push a cleanup to copy the value back at the end of the function.
3168	// The convention does not guarantee that the value will be written
3169	// back if the function exits with an unwind exception.
3170	EHStack.pushCleanup<CopyBackSwiftError>(Kind: NormalCleanup, A: temp, A: arg);
3171	}
3172
3173	// Ensure the argument is the correct type.
3174	if (V->getType() != ArgI.getCoerceToType())
3175	V = Builder.CreateBitCast(V, DestTy: ArgI.getCoerceToType());
3176
3177	if (isPromoted)
3178	V = emitArgumentDemotion(CGF&: *this, var: Arg, value: V);
3179
3180	// Because of merging of function types from multiple decls it is
3181	// possible for the type of an argument to not match the corresponding
3182	// type in the function type. Since we are codegening the callee
3183	// in here, add a cast to the argument type.
3184	llvm::Type *LTy = ConvertType(Arg->getType());
3185	if (V->getType() != LTy)
3186	V = Builder.CreateBitCast(V, DestTy: LTy);
3187
3188	ArgVals.push_back(Elt: ParamValue::forDirect(value: V));
3189	break;
3190	}
3191
3192	// VLST arguments are coerced to VLATs at the function boundary for
3193	// ABI consistency. If this is a VLST that was coerced to
3194	// a VLAT at the function boundary and the types match up, use
3195	// llvm.vector.extract to convert back to the original VLST.
3196	if (auto *VecTyTo = dyn_cast<llvm::FixedVectorType>(ConvertType(Ty))) {
3197	llvm::Value *Coerced = Fn->getArg(i: FirstIRArg);
3198	if (auto *VecTyFrom =
3199	dyn_cast<llvm::ScalableVectorType>(Val: Coerced->getType())) {
3200	// If we are casting a scalable i1 predicate vector to a fixed i8
3201	// vector, bitcast the source and use a vector extract.
3202	if (VecTyFrom->getElementType()->isIntegerTy(Bitwidth: 1) &&
3203	VecTyFrom->getElementCount().isKnownMultipleOf(RHS: 8) &&
3204	VecTyTo->getElementType() == Builder.getInt8Ty()) {
3205	VecTyFrom = llvm::ScalableVectorType::get(
3206	VecTyTo->getElementType(),
3207	VecTyFrom->getElementCount().getKnownMinValue() / 8);
3208	Coerced = Builder.CreateBitCast(V: Coerced, DestTy: VecTyFrom);
3209	}
3210	if (VecTyFrom->getElementType() == VecTyTo->getElementType()) {
3211	llvm::Value *Zero = llvm::Constant::getNullValue(Ty: CGM.Int64Ty);
3212
3213	assert(NumIRArgs == 1);
3214	Coerced->setName(Arg->getName() + ".coerce");
3215	ArgVals.push_back(Elt: ParamValue::forDirect(value: Builder.CreateExtractVector(
3216	DstType: VecTyTo, SrcVec: Coerced, Idx: Zero, Name: "cast.fixed")));
3217	break;
3218	}
3219	}
3220	}
3221
3222	llvm::StructType *STy =
3223	dyn_cast<llvm::StructType>(Val: ArgI.getCoerceToType());
3224	llvm::TypeSize StructSize;
3225	llvm::TypeSize PtrElementSize;
3226	if (ArgI.isDirect() && !ArgI.getCanBeFlattened() && STy &&
3227	STy->getNumElements() > 1) {
3228	StructSize = CGM.getDataLayout().getTypeAllocSize(Ty: STy);
3229	PtrElementSize =
3230	CGM.getDataLayout().getTypeAllocSize(Ty: ConvertTypeForMem(T: Ty));
3231	if (STy->containsHomogeneousScalableVectorTypes()) {
3232	assert(StructSize == PtrElementSize &&
3233	"Only allow non-fractional movement of structure with"
3234	"homogeneous scalable vector type");
3235
3236	ArgVals.push_back(Elt: ParamValue::forDirect(value: AI));
3237	break;
3238	}
3239	}
3240
3241	Address Alloca = CreateMemTemp(Ty, getContext().getDeclAlign(Arg),
3242	Arg->getName());
3243
3244	// Pointer to store into.
3245	Address Ptr = emitAddressAtOffset(CGF&: *this, addr: Alloca, info: ArgI);
3246
3247	// Fast-isel and the optimizer generally like scalar values better than
3248	// FCAs, so we flatten them if this is safe to do for this argument.
3249	if (ArgI.isDirect() && ArgI.getCanBeFlattened() && STy &&
3250	STy->getNumElements() > 1) {
3251	llvm::TypeSize StructSize = CGM.getDataLayout().getTypeAllocSize(Ty: STy);
3252	llvm::TypeSize PtrElementSize =
3253	CGM.getDataLayout().getTypeAllocSize(Ty: Ptr.getElementType());
3254	if (StructSize.isScalable()) {
3255	assert(STy->containsHomogeneousScalableVectorTypes() &&
3256	"ABI only supports structure with homogeneous scalable vector "
3257	"type");
3258	assert(StructSize == PtrElementSize &&
3259	"Only allow non-fractional movement of structure with"
3260	"homogeneous scalable vector type");
3261	assert(STy->getNumElements() == NumIRArgs);
3262
3263	llvm::Value *LoadedStructValue = llvm::PoisonValue::get(T: STy);
3264	for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
3265	auto *AI = Fn->getArg(i: FirstIRArg + i);
3266	AI->setName(Arg->getName() + ".coerce" + Twine(i));
3267	LoadedStructValue =
3268	Builder.CreateInsertValue(Agg: LoadedStructValue, Val: AI, Idxs: i);
3269	}
3270
3271	Builder.CreateStore(Val: LoadedStructValue, Addr: Ptr);
3272	} else {
3273	uint64_t SrcSize = StructSize.getFixedValue();
3274	uint64_t DstSize = PtrElementSize.getFixedValue();
3275
3276	Address AddrToStoreInto = Address::invalid();
3277	if (SrcSize <= DstSize) {
3278	AddrToStoreInto = Ptr.withElementType(ElemTy: STy);
3279	} else {
3280	AddrToStoreInto =
3281	CreateTempAlloca(Ty: STy, align: Alloca.getAlignment(), Name: "coerce");
3282	}
3283
3284	assert(STy->getNumElements() == NumIRArgs);
3285	for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
3286	auto AI = Fn->getArg(i: FirstIRArg + i);
3287	AI->setName(Arg->getName() + ".coerce" + Twine(i));
3288	Address EltPtr = Builder.CreateStructGEP(Addr: AddrToStoreInto, Index: i);
3289	Builder.CreateStore(Val: AI, Addr: EltPtr);
3290	}
3291
3292	if (SrcSize > DstSize) {
3293	Builder.CreateMemCpy(Dest: Ptr, Src: AddrToStoreInto, Size: DstSize);
3294	}
3295	}
3296	} else {
3297	// Simple case, just do a coerced store of the argument into the alloca.
3298	assert(NumIRArgs == 1);
3299	auto AI = Fn->getArg(i: FirstIRArg);
3300	AI->setName(Arg->getName() + ".coerce");
3301	CreateCoercedStore(Src: AI, Dst: Ptr, /*DstIsVolatile=*/false, CGF&: *this);
3302	}
3303
3304	// Match to what EmitParmDecl is expecting for this type.
3305	if (CodeGenFunction::hasScalarEvaluationKind(T: Ty)) {
3306	llvm::Value *V =
3307	EmitLoadOfScalar(Alloca, false, Ty, Arg->getBeginLoc());
3308	if (isPromoted)
3309	V = emitArgumentDemotion(CGF&: *this, var: Arg, value: V);
3310	ArgVals.push_back(Elt: ParamValue::forDirect(value: V));
3311	} else {
3312	ArgVals.push_back(Elt: ParamValue::forIndirect(addr: Alloca));
3313	}
3314	break;
3315	}
3316
3317	case ABIArgInfo::CoerceAndExpand: {
3318	// Reconstruct into a temporary.
3319	Address alloca = CreateMemTemp(T: Ty, Align: getContext().getDeclAlign(Arg));
3320	ArgVals.push_back(Elt: ParamValue::forIndirect(addr: alloca));
3321
3322	auto coercionType = ArgI.getCoerceAndExpandType();
3323	alloca = alloca.withElementType(ElemTy: coercionType);
3324
3325	unsigned argIndex = FirstIRArg;
3326	for (unsigned i = 0, e = coercionType->getNumElements(); i != e; ++i) {
3327	llvm::Type *eltType = coercionType->getElementType(N: i);
3328	if (ABIArgInfo::isPaddingForCoerceAndExpand(eltType))
3329	continue;
3330
3331	auto eltAddr = Builder.CreateStructGEP(Addr: alloca, Index: i);
3332	auto elt = Fn->getArg(i: argIndex++);
3333	Builder.CreateStore(Val: elt, Addr: eltAddr);
3334	}
3335	assert(argIndex == FirstIRArg + NumIRArgs);
3336	break;
3337	}
3338
3339	case ABIArgInfo::Expand: {
3340	// If this structure was expanded into multiple arguments then
3341	// we need to create a temporary and reconstruct it from the
3342	// arguments.
3343	Address Alloca = CreateMemTemp(T: Ty, Align: getContext().getDeclAlign(Arg));
3344	LValue LV = MakeAddrLValue(Addr: Alloca, T: Ty);
3345	ArgVals.push_back(Elt: ParamValue::forIndirect(addr: Alloca));
3346
3347	auto FnArgIter = Fn->arg_begin() + FirstIRArg;
3348	ExpandTypeFromArgs(Ty, LV, AI&: FnArgIter);
3349	assert(FnArgIter == Fn->arg_begin() + FirstIRArg + NumIRArgs);
3350	for (unsigned i = 0, e = NumIRArgs; i != e; ++i) {
3351	auto AI = Fn->getArg(i: FirstIRArg + i);
3352	AI->setName(Arg->getName() + "." + Twine(i));
3353	}
3354	break;
3355	}
3356
3357	case ABIArgInfo::Ignore:
3358	assert(NumIRArgs == 0);
3359	// Initialize the local variable appropriately.
3360	if (!hasScalarEvaluationKind(T: Ty)) {
3361	ArgVals.push_back(Elt: ParamValue::forIndirect(addr: CreateMemTemp(T: Ty)));
3362	} else {
3363	llvm::Value *U = llvm::UndefValue::get(T: ConvertType(Arg->getType()));
3364	ArgVals.push_back(Elt: ParamValue::forDirect(value: U));
3365	}
3366	break;
3367	}
3368	}
3369
3370	if (getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee()) {
3371	for (int I = Args.size() - 1; I >= 0; --I)
3372	EmitParmDecl(D: *Args[I], Arg: ArgVals[I], ArgNo: I + 1);
3373	} else {
3374	for (unsigned I = 0, E = Args.size(); I != E; ++I)
3375	EmitParmDecl(D: *Args[I], Arg: ArgVals[I], ArgNo: I + 1);
3376	}
3377	}
3378
3379	static void eraseUnusedBitCasts(llvm::Instruction *insn) {
3380	while (insn->use_empty()) {
3381	llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(Val: insn);
3382	if (!bitcast) return;
3383
3384	// This is "safe" because we would have used a ConstantExpr otherwise.
3385	insn = cast<llvm::Instruction>(Val: bitcast->getOperand(i_nocapture: 0));
3386	bitcast->eraseFromParent();
3387	}
3388	}
3389
3390	/// Try to emit a fused autorelease of a return result.
3391	static llvm::Value *tryEmitFusedAutoreleaseOfResult(CodeGenFunction &CGF,
3392	llvm::Value *result) {
3393	// We must be immediately followed the cast.
3394	llvm::BasicBlock *BB = CGF.Builder.GetInsertBlock();
3395	if (BB->empty()) return nullptr;
3396	if (&BB->back() != result) return nullptr;
3397
3398	llvm::Type *resultType = result->getType();
3399
3400	// result is in a BasicBlock and is therefore an Instruction.
3401	llvm::Instruction *generator = cast<llvm::Instruction>(Val: result);
3402
3403	SmallVector<llvm::Instruction *, 4> InstsToKill;
3404
3405	// Look for:
3406	// %generator = bitcast %type1* %generator2 to %type2*
3407	while (llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(Val: generator)) {
3408	// We would have emitted this as a constant if the operand weren't
3409	// an Instruction.
3410	generator = cast<llvm::Instruction>(Val: bitcast->getOperand(i_nocapture: 0));
3411
3412	// Require the generator to be immediately followed by the cast.
3413	if (generator->getNextNode() != bitcast)
3414	return nullptr;
3415
3416	InstsToKill.push_back(Elt: bitcast);
3417	}
3418
3419	// Look for:
3420	// %generator = call i8* @objc_retain(i8* %originalResult)
3421	// or
3422	// %generator = call i8* @objc_retainAutoreleasedReturnValue(i8* %originalResult)
3423	llvm::CallInst *call = dyn_cast<llvm::CallInst>(Val: generator);
3424	if (!call) return nullptr;
3425
3426	bool doRetainAutorelease;
3427
3428	if (call->getCalledOperand() == CGF.CGM.getObjCEntrypoints().objc_retain) {
3429	doRetainAutorelease = true;
3430	} else if (call->getCalledOperand() ==
3431	CGF.CGM.getObjCEntrypoints().objc_retainAutoreleasedReturnValue) {
3432	doRetainAutorelease = false;
3433
3434	// If we emitted an assembly marker for this call (and the
3435	// ARCEntrypoints field should have been set if so), go looking
3436	// for that call. If we can't find it, we can't do this
3437	// optimization. But it should always be the immediately previous
3438	// instruction, unless we needed bitcasts around the call.
3439	if (CGF.CGM.getObjCEntrypoints().retainAutoreleasedReturnValueMarker) {
3440	llvm::Instruction *prev = call->getPrevNode();
3441	assert(prev);
3442	if (isa<llvm::BitCastInst>(Val: prev)) {
3443	prev = prev->getPrevNode();
3444	assert(prev);
3445	}
3446	assert(isa<llvm::CallInst>(prev));
3447	assert(cast<llvm::CallInst>(prev)->getCalledOperand() ==
3448	CGF.CGM.getObjCEntrypoints().retainAutoreleasedReturnValueMarker);
3449	InstsToKill.push_back(Elt: prev);
3450	}
3451	} else {
3452	return nullptr;
3453	}
3454
3455	result = call->getArgOperand(i: 0);
3456	InstsToKill.push_back(Elt: call);
3457
3458	// Keep killing bitcasts, for sanity. Note that we no longer care
3459	// about precise ordering as long as there's exactly one use.
3460	while (llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(Val: result)) {
3461	if (!bitcast->hasOneUse()) break;
3462	InstsToKill.push_back(Elt: bitcast);
3463	result = bitcast->getOperand(i_nocapture: 0);
3464	}
3465
3466	// Delete all the unnecessary instructions, from latest to earliest.
3467	for (auto *I : InstsToKill)
3468	I->eraseFromParent();
3469
3470	// Do the fused retain/autorelease if we were asked to.
3471	if (doRetainAutorelease)
3472	result = CGF.EmitARCRetainAutoreleaseReturnValue(value: result);
3473
3474	// Cast back to the result type.
3475	return CGF.Builder.CreateBitCast(V: result, DestTy: resultType);
3476	}
3477
3478	/// If this is a +1 of the value of an immutable 'self', remove it.
3479	static llvm::Value *tryRemoveRetainOfSelf(CodeGenFunction &CGF,
3480	llvm::Value *result) {
3481	// This is only applicable to a method with an immutable 'self'.
3482	const ObjCMethodDecl *method =
3483	dyn_cast_or_null<ObjCMethodDecl>(Val: CGF.CurCodeDecl);
3484	if (!method) return nullptr;
3485	const VarDecl *self = method->getSelfDecl();
3486	if (!self->getType().isConstQualified()) return nullptr;
3487
3488	// Look for a retain call. Note: stripPointerCasts looks through returned arg
3489	// functions, which would cause us to miss the retain.
3490	llvm::CallInst *retainCall = dyn_cast<llvm::CallInst>(Val: result);
3491	if (!retainCall || retainCall->getCalledOperand() !=
3492	CGF.CGM.getObjCEntrypoints().objc_retain)
3493	return nullptr;
3494
3495	// Look for an ordinary load of 'self'.
3496	llvm::Value *retainedValue = retainCall->getArgOperand(i: 0);
3497	llvm::LoadInst *load =
3498	dyn_cast<llvm::LoadInst>(Val: retainedValue->stripPointerCasts());
3499	if (!load || load->isAtomic() || load->isVolatile() ||
3500	load->getPointerOperand() != CGF.GetAddrOfLocalVar(VD: self).getPointer())
3501	return nullptr;
3502
3503	// Okay! Burn it all down. This relies for correctness on the
3504	// assumption that the retain is emitted as part of the return and
3505	// that thereafter everything is used "linearly".
3506	llvm::Type *resultType = result->getType();
3507	eraseUnusedBitCasts(insn: cast<llvm::Instruction>(Val: result));
3508	assert(retainCall->use_empty());
3509	retainCall->eraseFromParent();
3510	eraseUnusedBitCasts(insn: cast<llvm::Instruction>(Val: retainedValue));
3511
3512	return CGF.Builder.CreateBitCast(V: load, DestTy: resultType);
3513	}
3514
3515	/// Emit an ARC autorelease of the result of a function.
3516	///
3517	/// \return the value to actually return from the function
3518	static llvm::Value *emitAutoreleaseOfResult(CodeGenFunction &CGF,
3519	llvm::Value *result) {
3520	// If we're returning 'self', kill the initial retain. This is a
3521	// heuristic attempt to "encourage correctness" in the really unfortunate
3522	// case where we have a return of self during a dealloc and we desperately
3523	// need to avoid the possible autorelease.
3524	if (llvm::Value *self = tryRemoveRetainOfSelf(CGF, result))
3525	return self;
3526
3527	// At -O0, try to emit a fused retain/autorelease.
3528	if (CGF.shouldUseFusedARCCalls())
3529	if (llvm::Value *fused = tryEmitFusedAutoreleaseOfResult(CGF, result))
3530	return fused;
3531
3532	return CGF.EmitARCAutoreleaseReturnValue(value: result);
3533	}
3534
3535	/// Heuristically search for a dominating store to the return-value slot.
3536	static llvm::StoreInst *findDominatingStoreToReturnValue(CodeGenFunction &CGF) {
3537	// Check if a User is a store which pointerOperand is the ReturnValue.
3538	// We are looking for stores to the ReturnValue, not for stores of the
3539	// ReturnValue to some other location.
3540	auto GetStoreIfValid = [&CGF](llvm::User *U) -> llvm::StoreInst * {
3541	auto *SI = dyn_cast<llvm::StoreInst>(Val: U);
3542	if (!SI || SI->getPointerOperand() != CGF.ReturnValue.getPointer() ||
3543	SI->getValueOperand()->getType() != CGF.ReturnValue.getElementType())
3544	return nullptr;
3545	// These aren't actually possible for non-coerced returns, and we
3546	// only care about non-coerced returns on this code path.
3547	// All memory instructions inside __try block are volatile.
3548	assert(!SI->isAtomic() &&
3549	(!SI->isVolatile() || CGF.currentFunctionUsesSEHTry()));
3550	return SI;
3551	};
3552	// If there are multiple uses of the return-value slot, just check
3553	// for something immediately preceding the IP. Sometimes this can
3554	// happen with how we generate implicit-returns; it can also happen
3555	// with noreturn cleanups.
3556	if (!CGF.ReturnValue.getPointer()->hasOneUse()) {
3557	llvm::BasicBlock *IP = CGF.Builder.GetInsertBlock();
3558	if (IP->empty()) return nullptr;
3559
3560	// Look at directly preceding instruction, skipping bitcasts and lifetime
3561	// markers.
3562	for (llvm::Instruction &I : make_range(x: IP->rbegin(), y: IP->rend())) {
3563	if (isa<llvm::BitCastInst>(Val: &I))
3564	continue;
3565	if (auto *II = dyn_cast<llvm::IntrinsicInst>(Val: &I))
3566	if (II->getIntrinsicID() == llvm::Intrinsic::lifetime_end)
3567	continue;
3568
3569	return GetStoreIfValid(&I);
3570	}
3571	return nullptr;
3572	}
3573
3574	llvm::StoreInst *store =
3575	GetStoreIfValid(CGF.ReturnValue.getPointer()->user_back());
3576	if (!store) return nullptr;
3577
3578	// Now do a first-and-dirty dominance check: just walk up the
3579	// single-predecessors chain from the current insertion point.
3580	llvm::BasicBlock *StoreBB = store->getParent();
3581	llvm::BasicBlock *IP = CGF.Builder.GetInsertBlock();
3582	llvm::SmallPtrSet<llvm::BasicBlock *, 4> SeenBBs;
3583	while (IP != StoreBB) {
3584	if (!SeenBBs.insert(Ptr: IP).second || !(IP = IP->getSinglePredecessor()))
3585	return nullptr;
3586	}
3587
3588	// Okay, the store's basic block dominates the insertion point; we
3589	// can do our thing.
3590	return store;
3591	}
3592
3593	// Helper functions for EmitCMSEClearRecord
3594
3595	// Set the bits corresponding to a field having width `BitWidth` and located at
3596	// offset `BitOffset` (from the least significant bit) within a storage unit of
3597	// `Bits.size()` bytes. Each element of `Bits` corresponds to one target byte.
3598	// Use little-endian layout, i.e.`Bits[0]` is the LSB.
3599	static void setBitRange(SmallVectorImpl<uint64_t> &Bits, int BitOffset,
3600	int BitWidth, int CharWidth) {
3601	assert(CharWidth <= 64);
3602	assert(static_cast<unsigned>(BitWidth) <= Bits.size() * CharWidth);
3603
3604	int Pos = 0;
3605	if (BitOffset >= CharWidth) {
3606	Pos += BitOffset / CharWidth;
3607	BitOffset = BitOffset % CharWidth;
3608	}
3609
3610	const uint64_t Used = (uint64_t(1) << CharWidth) - 1;
3611	if (BitOffset + BitWidth >= CharWidth) {
3612	Bits[Pos++] |= (Used << BitOffset) & Used;
3613	BitWidth -= CharWidth - BitOffset;
3614	BitOffset = 0;
3615	}
3616
3617	while (BitWidth >= CharWidth) {
3618	Bits[Pos++] = Used;
3619	BitWidth -= CharWidth;
3620	}
3621
3622	if (BitWidth > 0)
3623	Bits[Pos++] |= (Used >> (CharWidth - BitWidth)) << BitOffset;
3624	}
3625
3626	// Set the bits corresponding to a field having width `BitWidth` and located at
3627	// offset `BitOffset` (from the least significant bit) within a storage unit of
3628	// `StorageSize` bytes, located at `StorageOffset` in `Bits`. Each element of
3629	// `Bits` corresponds to one target byte. Use target endian layout.
3630	static void setBitRange(SmallVectorImpl<uint64_t> &Bits, int StorageOffset,
3631	int StorageSize, int BitOffset, int BitWidth,
3632	int CharWidth, bool BigEndian) {
3633
3634	SmallVector<uint64_t, 8> TmpBits(StorageSize);
3635	setBitRange(Bits&: TmpBits, BitOffset, BitWidth, CharWidth);
3636
3637	if (BigEndian)
3638	std::reverse(first: TmpBits.begin(), last: TmpBits.end());
3639
3640	for (uint64_t V : TmpBits)
3641	Bits[StorageOffset++] |= V;
3642	}
3643
3644	static void setUsedBits(CodeGenModule &, QualType, int,
3645	SmallVectorImpl<uint64_t> &);
3646
3647	// Set the bits in `Bits`, which correspond to the value representations of
3648	// the actual members of the record type `RTy`. Note that this function does
3649	// not handle base classes, virtual tables, etc, since they cannot happen in
3650	// CMSE function arguments or return. The bit mask corresponds to the target
3651	// memory layout, i.e. it's endian dependent.
3652	static void setUsedBits(CodeGenModule &CGM, const RecordType *RTy, int Offset,
3653	SmallVectorImpl<uint64_t> &Bits) {
3654	ASTContext &Context = CGM.getContext();
3655	int CharWidth = Context.getCharWidth();
3656	const RecordDecl *RD = RTy->getDecl()->getDefinition();
3657	const ASTRecordLayout &ASTLayout = Context.getASTRecordLayout(D: RD);
3658	const CGRecordLayout &Layout = CGM.getTypes().getCGRecordLayout(RD);
3659
3660	int Idx = 0;
3661	for (auto I = RD->field_begin(), E = RD->field_end(); I != E; ++I, ++Idx) {
3662	const FieldDecl *F = *I;
3663
3664	if (F->isUnnamedBitfield() || F->isZeroLengthBitField(Ctx: Context) ||
3665	F->getType()->isIncompleteArrayType())
3666	continue;
3667
3668	if (F->isBitField()) {
3669	const CGBitFieldInfo &BFI = Layout.getBitFieldInfo(FD: F);
3670	setBitRange(Bits, StorageOffset: Offset + BFI.StorageOffset.getQuantity(),
3671	StorageSize: BFI.StorageSize / CharWidth, BitOffset: BFI.Offset,
3672	BitWidth: BFI.Size, CharWidth,
3673	BigEndian: CGM.getDataLayout().isBigEndian());
3674	continue;
3675	}
3676
3677	setUsedBits(CGM, F->getType(),
3678	Offset + ASTLayout.getFieldOffset(FieldNo: Idx) / CharWidth, Bits);
3679	}
3680	}
3681
3682	// Set the bits in `Bits`, which correspond to the value representations of
3683	// the elements of an array type `ATy`.
3684	static void setUsedBits(CodeGenModule &CGM, const ConstantArrayType *ATy,
3685	int Offset, SmallVectorImpl<uint64_t> &Bits) {
3686	const ASTContext &Context = CGM.getContext();
3687
3688	QualType ETy = Context.getBaseElementType(ATy);
3689	int Size = Context.getTypeSizeInChars(T: ETy).getQuantity();
3690	SmallVector<uint64_t, 4> TmpBits(Size);
3691	setUsedBits(CGM, ETy, 0, TmpBits);
3692
3693	for (int I = 0, N = Context.getConstantArrayElementCount(CA: ATy); I < N; ++I) {
3694	auto Src = TmpBits.begin();
3695	auto Dst = Bits.begin() + Offset + I * Size;
3696	for (int J = 0; J < Size; ++J)
3697	*Dst++ |= *Src++;
3698	}
3699	}
3700
3701	// Set the bits in `Bits`, which correspond to the value representations of
3702	// the type `QTy`.
3703	static void setUsedBits(CodeGenModule &CGM, QualType QTy, int Offset,
3704	SmallVectorImpl<uint64_t> &Bits) {
3705	if (const auto *RTy = QTy->getAs<RecordType>())
3706	return setUsedBits(CGM, RTy, Offset, Bits);
3707
3708	ASTContext &Context = CGM.getContext();
3709	if (const auto *ATy = Context.getAsConstantArrayType(T: QTy))
3710	return setUsedBits(CGM, ATy, Offset, Bits);
3711
3712	int Size = Context.getTypeSizeInChars(T: QTy).getQuantity();
3713	if (Size <= 0)
3714	return;
3715
3716	std::fill_n(first: Bits.begin() + Offset, n: Size,
3717	value: (uint64_t(1) << Context.getCharWidth()) - 1);
3718	}
3719
3720	static uint64_t buildMultiCharMask(const SmallVectorImpl<uint64_t> &Bits,
3721	int Pos, int Size, int CharWidth,
3722	bool BigEndian) {
3723	assert(Size > 0);
3724	uint64_t Mask = 0;
3725	if (BigEndian) {
3726	for (auto P = Bits.begin() + Pos, E = Bits.begin() + Pos + Size; P != E;
3727	++P)
3728	Mask = (Mask << CharWidth) | *P;
3729	} else {
3730	auto P = Bits.begin() + Pos + Size, End = Bits.begin() + Pos;
3731	do
3732	Mask = (Mask << CharWidth) | *--P;
3733	while (P != End);
3734	}
3735	return Mask;
3736	}
3737
3738	// Emit code to clear the bits in a record, which aren't a part of any user
3739	// declared member, when the record is a function return.
3740	llvm::Value *CodeGenFunction::EmitCMSEClearRecord(llvm::Value *Src,
3741	llvm::IntegerType *ITy,
3742	QualType QTy) {
3743	assert(Src->getType() == ITy);
3744	assert(ITy->getScalarSizeInBits() <= 64);
3745
3746	const llvm::DataLayout &DataLayout = CGM.getDataLayout();
3747	int Size = DataLayout.getTypeStoreSize(Ty: ITy);
3748	SmallVector<uint64_t, 4> Bits(Size);
3749	setUsedBits(CGM, RTy: QTy->castAs<RecordType>(), Offset: 0, Bits);
3750
3751	int CharWidth = CGM.getContext().getCharWidth();
3752	uint64_t Mask =
3753	buildMultiCharMask(Bits, Pos: 0, Size, CharWidth, BigEndian: DataLayout.isBigEndian());
3754
3755	return Builder.CreateAnd(LHS: Src, RHS: Mask, Name: "cmse.clear");
3756	}
3757
3758	// Emit code to clear the bits in a record, which aren't a part of any user
3759	// declared member, when the record is a function argument.
3760	llvm::Value *CodeGenFunction::EmitCMSEClearRecord(llvm::Value *Src,
3761	llvm::ArrayType *ATy,
3762	QualType QTy) {
3763	const llvm::DataLayout &DataLayout = CGM.getDataLayout();
3764	int Size = DataLayout.getTypeStoreSize(Ty: ATy);
3765	SmallVector<uint64_t, 16> Bits(Size);
3766	setUsedBits(CGM, RTy: QTy->castAs<RecordType>(), Offset: 0, Bits);
3767
3768	// Clear each element of the LLVM array.
3769	int CharWidth = CGM.getContext().getCharWidth();
3770	int CharsPerElt =
3771	ATy->getArrayElementType()->getScalarSizeInBits() / CharWidth;
3772	int MaskIndex = 0;
3773	llvm::Value *R = llvm::PoisonValue::get(T: ATy);
3774	for (int I = 0, N = ATy->getArrayNumElements(); I != N; ++I) {
3775	uint64_t Mask = buildMultiCharMask(Bits, Pos: MaskIndex, Size: CharsPerElt, CharWidth,
3776	BigEndian: DataLayout.isBigEndian());
3777	MaskIndex += CharsPerElt;
3778	llvm::Value *T0 = Builder.CreateExtractValue(Agg: Src, Idxs: I);
3779	llvm::Value *T1 = Builder.CreateAnd(LHS: T0, RHS: Mask, Name: "cmse.clear");
3780	R = Builder.CreateInsertValue(Agg: R, Val: T1, Idxs: I);
3781	}
3782
3783	return R;
3784	}
3785
3786	void CodeGenFunction::EmitFunctionEpilog(const CGFunctionInfo &FI,
3787	bool EmitRetDbgLoc,
3788	SourceLocation EndLoc) {
3789	if (FI.isNoReturn()) {
3790	// Noreturn functions don't return.
3791	EmitUnreachable(Loc: EndLoc);
3792	return;
3793	}
3794
3795	if (CurCodeDecl && CurCodeDecl->hasAttr<NakedAttr>()) {
3796	// Naked functions don't have epilogues.
3797	Builder.CreateUnreachable();
3798	return;
3799	}
3800
3801	// Functions with no result always return void.
3802	if (!ReturnValue.isValid()) {
3803	Builder.CreateRetVoid();
3804	return;
3805	}
3806
3807	llvm::DebugLoc RetDbgLoc;
3808	llvm::Value *RV = nullptr;
3809	QualType RetTy = FI.getReturnType();
3810	const ABIArgInfo &RetAI = FI.getReturnInfo();
3811
3812	switch (RetAI.getKind()) {
3813	case ABIArgInfo::InAlloca:
3814	// Aggregates get evaluated directly into the destination. Sometimes we
3815	// need to return the sret value in a register, though.
3816	assert(hasAggregateEvaluationKind(RetTy));
3817	if (RetAI.getInAllocaSRet()) {
3818	llvm::Function::arg_iterator EI = CurFn->arg_end();
3819	--EI;
3820	llvm::Value *ArgStruct = &*EI;
3821	llvm::Value *SRet = Builder.CreateStructGEP(
3822	Ty: FI.getArgStruct(), Ptr: ArgStruct, Idx: RetAI.getInAllocaFieldIndex());
3823	llvm::Type *Ty =
3824	cast<llvm::GetElementPtrInst>(Val: SRet)->getResultElementType();
3825	RV = Builder.CreateAlignedLoad(Ty, SRet, getPointerAlign(), "sret");
3826	}
3827	break;
3828
3829	case ABIArgInfo::Indirect: {
3830	auto AI = CurFn->arg_begin();
3831	if (RetAI.isSRetAfterThis())
3832	++AI;
3833	switch (getEvaluationKind(T: RetTy)) {
3834	case TEK_Complex: {
3835	ComplexPairTy RT =
3836	EmitLoadOfComplex(src: MakeAddrLValue(Addr: ReturnValue, T: RetTy), loc: EndLoc);
3837	EmitStoreOfComplex(V: RT, dest: MakeNaturalAlignAddrLValue(V: &*AI, T: RetTy),
3838	/*isInit*/ true);
3839	break;
3840	}
3841	case TEK_Aggregate:
3842	// Do nothing; aggregates get evaluated directly into the destination.
3843	break;
3844	case TEK_Scalar: {
3845	LValueBaseInfo BaseInfo;
3846	TBAAAccessInfo TBAAInfo;
3847	CharUnits Alignment =
3848	CGM.getNaturalTypeAlignment(T: RetTy, BaseInfo: &BaseInfo, TBAAInfo: &TBAAInfo);
3849	Address ArgAddr(&*AI, ConvertType(T: RetTy), Alignment);
3850	LValue ArgVal =
3851	LValue::MakeAddr(address: ArgAddr, type: RetTy, Context&: getContext(), BaseInfo, TBAAInfo);
3852	EmitStoreOfScalar(
3853	value: Builder.CreateLoad(Addr: ReturnValue), lvalue: ArgVal, /*isInit*/ true);
3854	break;
3855	}
3856	}
3857	break;
3858	}
3859
3860	case ABIArgInfo::Extend:
3861	case ABIArgInfo::Direct:
3862	if (RetAI.getCoerceToType() == ConvertType(T: RetTy) &&
3863	RetAI.getDirectOffset() == 0) {
3864	// The internal return value temp always will have pointer-to-return-type
3865	// type, just do a load.
3866
3867	// If there is a dominating store to ReturnValue, we can elide
3868	// the load, zap the store, and usually zap the alloca.
3869	if (llvm::StoreInst *SI =
3870	findDominatingStoreToReturnValue(CGF&: *this)) {
3871	// Reuse the debug location from the store unless there is
3872	// cleanup code to be emitted between the store and return
3873	// instruction.
3874	if (EmitRetDbgLoc && !AutoreleaseResult)
3875	RetDbgLoc = SI->getDebugLoc();
3876	// Get the stored value and nuke the now-dead store.
3877	RV = SI->getValueOperand();
3878	SI->eraseFromParent();
3879
3880	// Otherwise, we have to do a simple load.
3881	} else {
3882	RV = Builder.CreateLoad(Addr: ReturnValue);
3883	}
3884	} else {
3885	// If the value is offset in memory, apply the offset now.
3886	Address V = emitAddressAtOffset(CGF&: *this, addr: ReturnValue, info: RetAI);
3887
3888	RV = CreateCoercedLoad(Src: V, Ty: RetAI.getCoerceToType(), CGF&: *this);
3889	}
3890
3891	// In ARC, end functions that return a retainable type with a call
3892	// to objc_autoreleaseReturnValue.
3893	if (AutoreleaseResult) {
3894	#ifndef NDEBUG
3895	// Type::isObjCRetainabletype has to be called on a QualType that hasn't
3896	// been stripped of the typedefs, so we cannot use RetTy here. Get the
3897	// original return type of FunctionDecl, CurCodeDecl, and BlockDecl from
3898	// CurCodeDecl or BlockInfo.
3899	QualType RT;
3900
3901	if (auto *FD = dyn_cast<FunctionDecl>(Val: CurCodeDecl))
3902	RT = FD->getReturnType();
3903	else if (auto *MD = dyn_cast<ObjCMethodDecl>(Val: CurCodeDecl))
3904	RT = MD->getReturnType();
3905	else if (isa<BlockDecl>(Val: CurCodeDecl))
3906	RT = BlockInfo->BlockExpression->getFunctionType()->getReturnType();
3907	else
3908	llvm_unreachable("Unexpected function/method type");
3909
3910	assert(getLangOpts().ObjCAutoRefCount &&
3911	!FI.isReturnsRetained() &&
3912	RT->isObjCRetainableType());
3913	#endif
3914	RV = emitAutoreleaseOfResult(CGF&: *this, result: RV);
3915	}
3916
3917	break;
3918
3919	case ABIArgInfo::Ignore:
3920	break;
3921
3922	case ABIArgInfo::CoerceAndExpand: {
3923	auto coercionType = RetAI.getCoerceAndExpandType();
3924
3925	// Load all of the coerced elements out into results.
3926	llvm::SmallVector<llvm::Value*, 4> results;
3927	Address addr = ReturnValue.withElementType(ElemTy: coercionType);
3928	for (unsigned i = 0, e = coercionType->getNumElements(); i != e; ++i) {
3929	auto coercedEltType = coercionType->getElementType(N: i);
3930	if (ABIArgInfo::isPaddingForCoerceAndExpand(eltType: coercedEltType))
3931	continue;
3932
3933	auto eltAddr = Builder.CreateStructGEP(Addr: addr, Index: i);
3934	auto elt = Builder.CreateLoad(Addr: eltAddr);
3935	results.push_back(Elt: elt);
3936	}
3937
3938	// If we have one result, it's the single direct result type.
3939	if (results.size() == 1) {
3940	RV = results[0];
3941
3942	// Otherwise, we need to make a first-class aggregate.
3943	} else {
3944	// Construct a return type that lacks padding elements.
3945	llvm::Type *returnType = RetAI.getUnpaddedCoerceAndExpandType();
3946
3947	RV = llvm::PoisonValue::get(T: returnType);
3948	for (unsigned i = 0, e = results.size(); i != e; ++i) {
3949	RV = Builder.CreateInsertValue(Agg: RV, Val: results[i], Idxs: i);
3950	}
3951	}
3952	break;
3953	}
3954	case ABIArgInfo::Expand:
3955	case ABIArgInfo::IndirectAliased:
3956	llvm_unreachable("Invalid ABI kind for return argument");
3957	}
3958
3959	llvm::Instruction *Ret;
3960	if (RV) {
3961	if (CurFuncDecl && CurFuncDecl->hasAttr<CmseNSEntryAttr>()) {
3962	// For certain return types, clear padding bits, as they may reveal
3963	// sensitive information.
3964	// Small struct/union types are passed as integers.
3965	auto *ITy = dyn_cast<llvm::IntegerType>(Val: RV->getType());
3966	if (ITy != nullptr && isa<RecordType>(Val: RetTy.getCanonicalType()))
3967	RV = EmitCMSEClearRecord(Src: RV, ITy, QTy: RetTy);
3968	}
3969	EmitReturnValueCheck(RV);
3970	Ret = Builder.CreateRet(V: RV);
3971	} else {
3972	Ret = Builder.CreateRetVoid();
3973	}
3974
3975	if (RetDbgLoc)
3976	Ret->setDebugLoc(std::move(RetDbgLoc));
3977	}
3978
3979	void CodeGenFunction::EmitReturnValueCheck(llvm::Value *RV) {
3980	// A current decl may not be available when emitting vtable thunks.
3981	if (!CurCodeDecl)
3982	return;
3983
3984	// If the return block isn't reachable, neither is this check, so don't emit
3985	// it.
3986	if (ReturnBlock.isValid() && ReturnBlock.getBlock()->use_empty())
3987	return;
3988
3989	ReturnsNonNullAttr *RetNNAttr = nullptr;
3990	if (SanOpts.has(SanitizerKind::ReturnsNonnullAttribute))
3991	RetNNAttr = CurCodeDecl->getAttr<ReturnsNonNullAttr>();
3992
3993	if (!RetNNAttr && !requiresReturnValueNullabilityCheck())
3994	return;
3995
3996	// Prefer the returns_nonnull attribute if it's present.
3997	SourceLocation AttrLoc;
3998	SanitizerMask CheckKind;
3999	SanitizerHandler Handler;
4000	if (RetNNAttr) {
4001	assert(!requiresReturnValueNullabilityCheck() &&
4002	"Cannot check nullability and the nonnull attribute");
4003	AttrLoc = RetNNAttr->getLocation();
4004	CheckKind = SanitizerKind::ReturnsNonnullAttribute;
4005	Handler = SanitizerHandler::NonnullReturn;
4006	} else {
4007	if (auto *DD = dyn_cast<DeclaratorDecl>(Val: CurCodeDecl))
4008	if (auto *TSI = DD->getTypeSourceInfo())
4009	if (auto FTL = TSI->getTypeLoc().getAsAdjusted<FunctionTypeLoc>())
4010	AttrLoc = FTL.getReturnLoc().findNullabilityLoc();
4011	CheckKind = SanitizerKind::NullabilityReturn;
4012	Handler = SanitizerHandler::NullabilityReturn;
4013	}
4014
4015	SanitizerScope SanScope(this);
4016
4017	// Make sure the "return" source location is valid. If we're checking a
4018	// nullability annotation, make sure the preconditions for the check are met.
4019	llvm::BasicBlock *Check = createBasicBlock(name: "nullcheck");
4020	llvm::BasicBlock *NoCheck = createBasicBlock(name: "no.nullcheck");
4021	llvm::Value *SLocPtr = Builder.CreateLoad(Addr: ReturnLocation, Name: "return.sloc.load");
4022	llvm::Value *CanNullCheck = Builder.CreateIsNotNull(Arg: SLocPtr);
4023	if (requiresReturnValueNullabilityCheck())
4024	CanNullCheck =
4025	Builder.CreateAnd(LHS: CanNullCheck, RHS: RetValNullabilityPrecondition);
4026	Builder.CreateCondBr(Cond: CanNullCheck, True: Check, False: NoCheck);
4027	EmitBlock(BB: Check);
4028
4029	// Now do the null check.
4030	llvm::Value *Cond = Builder.CreateIsNotNull(Arg: RV);
4031	llvm::Constant *StaticData[] = {EmitCheckSourceLocation(Loc: AttrLoc)};
4032	llvm::Value *DynamicData[] = {SLocPtr};
4033	EmitCheck(Checked: std::make_pair(x&: Cond, y&: CheckKind), Check: Handler, StaticArgs: StaticData, DynamicArgs: DynamicData);
4034
4035	EmitBlock(BB: NoCheck);
4036
4037	#ifndef NDEBUG
4038	// The return location should not be used after the check has been emitted.
4039	ReturnLocation = Address::invalid();
4040	#endif
4041	}
4042
4043	static bool isInAllocaArgument(CGCXXABI &ABI, QualType type) {
4044	const CXXRecordDecl *RD = type->getAsCXXRecordDecl();
4045	return RD && ABI.getRecordArgABI(RD) == CGCXXABI::RAA_DirectInMemory;
4046	}
4047
4048	static AggValueSlot createPlaceholderSlot(CodeGenFunction &CGF,
4049	QualType Ty) {
4050	// FIXME: Generate IR in one pass, rather than going back and fixing up these
4051	// placeholders.
4052	llvm::Type *IRTy = CGF.ConvertTypeForMem(T: Ty);
4053	llvm::Type *IRPtrTy = llvm::PointerType::getUnqual(C&: CGF.getLLVMContext());
4054	llvm::Value *Placeholder = llvm::PoisonValue::get(T: IRPtrTy);
4055
4056	// FIXME: When we generate this IR in one pass, we shouldn't need
4057	// this win32-specific alignment hack.
4058	CharUnits Align = CharUnits::fromQuantity(Quantity: 4);
4059	Placeholder = CGF.Builder.CreateAlignedLoad(Ty: IRPtrTy, Addr: Placeholder, Align);
4060
4061	return AggValueSlot::forAddr(addr: Address(Placeholder, IRTy, Align),
4062	quals: Ty.getQualifiers(),
4063	isDestructed: AggValueSlot::IsNotDestructed,
4064	needsGC: AggValueSlot::DoesNotNeedGCBarriers,
4065	isAliased: AggValueSlot::IsNotAliased,
4066	mayOverlap: AggValueSlot::DoesNotOverlap);
4067	}
4068
4069	void CodeGenFunction::EmitDelegateCallArg(CallArgList &args,
4070	const VarDecl *param,
4071	SourceLocation loc) {
4072	// StartFunction converted the ABI-lowered parameter(s) into a
4073	// local alloca. We need to turn that into an r-value suitable
4074	// for EmitCall.
4075	Address local = GetAddrOfLocalVar(VD: param);
4076
4077	QualType type = param->getType();
4078
4079	// GetAddrOfLocalVar returns a pointer-to-pointer for references,
4080	// but the argument needs to be the original pointer.
4081	if (type->isReferenceType()) {
4082	args.add(rvalue: RValue::get(V: Builder.CreateLoad(Addr: local)), type);
4083
4084	// In ARC, move out of consumed arguments so that the release cleanup
4085	// entered by StartFunction doesn't cause an over-release. This isn't
4086	// optimal -O0 code generation, but it should get cleaned up when
4087	// optimization is enabled. This also assumes that delegate calls are
4088	// performed exactly once for a set of arguments, but that should be safe.
4089	} else if (getLangOpts().ObjCAutoRefCount &&
4090	param->hasAttr<NSConsumedAttr>() &&
4091	type->isObjCRetainableType()) {
4092	llvm::Value *ptr = Builder.CreateLoad(Addr: local);
4093	auto null =
4094	llvm::ConstantPointerNull::get(T: cast<llvm::PointerType>(Val: ptr->getType()));
4095	Builder.CreateStore(Val: null, Addr: local);
4096	args.add(rvalue: RValue::get(V: ptr), type);
4097
4098	// For the most part, we just need to load the alloca, except that
4099	// aggregate r-values are actually pointers to temporaries.
4100	} else {
4101	args.add(rvalue: convertTempToRValue(addr: local, type, Loc: loc), type);
4102	}
4103
4104	// Deactivate the cleanup for the callee-destructed param that was pushed.
4105	if (type->isRecordType() && !CurFuncIsThunk &&
4106	type->castAs<RecordType>()->getDecl()->isParamDestroyedInCallee() &&
4107	param->needsDestruction(Ctx: getContext())) {
4108	EHScopeStack::stable_iterator cleanup =
4109	CalleeDestructedParamCleanups.lookup(Val: cast<ParmVarDecl>(Val: param));
4110	assert(cleanup.isValid() &&
4111	"cleanup for callee-destructed param not recorded");
4112	// This unreachable is a temporary marker which will be removed later.
4113	llvm::Instruction *isActive = Builder.CreateUnreachable();
4114	args.addArgCleanupDeactivation(Cleanup: cleanup, IsActiveIP: isActive);
4115	}
4116	}
4117
4118	static bool isProvablyNull(llvm::Value *addr) {
4119	return isa<llvm::ConstantPointerNull>(Val: addr);
4120	}
4121
4122	/// Emit the actual writing-back of a writeback.
4123	static void emitWriteback(CodeGenFunction &CGF,
4124	const CallArgList::Writeback &writeback) {
4125	const LValue &srcLV = writeback.Source;
4126	Address srcAddr = srcLV.getAddress(CGF);
4127	assert(!isProvablyNull(srcAddr.getPointer()) &&
4128	"shouldn't have writeback for provably null argument");
4129
4130	llvm::BasicBlock *contBB = nullptr;
4131
4132	// If the argument wasn't provably non-null, we need to null check
4133	// before doing the store.
4134	bool provablyNonNull = llvm::isKnownNonZero(V: srcAddr.getPointer(),
4135	DL: CGF.CGM.getDataLayout());
4136	if (!provablyNonNull) {
4137	llvm::BasicBlock *writebackBB = CGF.createBasicBlock(name: "icr.writeback");
4138	contBB = CGF.createBasicBlock(name: "icr.done");
4139
4140	llvm::Value *isNull =
4141	CGF.Builder.CreateIsNull(Arg: srcAddr.getPointer(), Name: "icr.isnull");
4142	CGF.Builder.CreateCondBr(Cond: isNull, True: contBB, False: writebackBB);
4143	CGF.EmitBlock(BB: writebackBB);
4144	}
4145
4146	// Load the value to writeback.
4147	llvm::Value *value = CGF.Builder.CreateLoad(Addr: writeback.Temporary);
4148
4149	// Cast it back, in case we're writing an id to a Foo* or something.
4150	value = CGF.Builder.CreateBitCast(V: value, DestTy: srcAddr.getElementType(),
4151	Name: "icr.writeback-cast");
4152
4153	// Perform the writeback.
4154
4155	// If we have a "to use" value, it's something we need to emit a use
4156	// of. This has to be carefully threaded in: if it's done after the
4157	// release it's potentially undefined behavior (and the optimizer
4158	// will ignore it), and if it happens before the retain then the
4159	// optimizer could move the release there.
4160	if (writeback.ToUse) {
4161	assert(srcLV.getObjCLifetime() == Qualifiers::OCL_Strong);
4162
4163	// Retain the new value. No need to block-copy here: the block's
4164	// being passed up the stack.
4165	value = CGF.EmitARCRetainNonBlock(value);
4166
4167	// Emit the intrinsic use here.
4168	CGF.EmitARCIntrinsicUse(values: writeback.ToUse);
4169
4170	// Load the old value (primitively).
4171	llvm::Value *oldValue = CGF.EmitLoadOfScalar(lvalue: srcLV, Loc: SourceLocation());
4172
4173	// Put the new value in place (primitively).
4174	CGF.EmitStoreOfScalar(value, lvalue: srcLV, /*init*/ isInit: false);
4175
4176	// Release the old value.
4177	CGF.EmitARCRelease(value: oldValue, precise: srcLV.isARCPreciseLifetime());
4178
4179	// Otherwise, we can just do a normal lvalue store.
4180	} else {
4181	CGF.EmitStoreThroughLValue(Src: RValue::get(V: value), Dst: srcLV);
4182	}
4183
4184	// Jump to the continuation block.
4185	if (!provablyNonNull)
4186	CGF.EmitBlock(BB: contBB);
4187	}
4188
4189	static void emitWritebacks(CodeGenFunction &CGF,
4190	const CallArgList &args) {
4191	for (const auto &I : args.writebacks())
4192	emitWriteback(CGF, writeback: I);
4193	}
4194
4195	static void deactivateArgCleanupsBeforeCall(CodeGenFunction &CGF,
4196	const CallArgList &CallArgs) {
4197	ArrayRef<CallArgList::CallArgCleanup> Cleanups =
4198	CallArgs.getCleanupsToDeactivate();
4199	// Iterate in reverse to increase the likelihood of popping the cleanup.
4200	for (const auto &I : llvm::reverse(C&: Cleanups)) {
4201	CGF.DeactivateCleanupBlock(Cleanup: I.Cleanup, DominatingIP: I.IsActiveIP);
4202	I.IsActiveIP->eraseFromParent();
4203	}
4204	}
4205
4206	static const Expr *maybeGetUnaryAddrOfOperand(const Expr *E) {
4207	if (const UnaryOperator *uop = dyn_cast<UnaryOperator>(Val: E->IgnoreParens()))
4208	if (uop->getOpcode() == UO_AddrOf)
4209	return uop->getSubExpr();
4210	return nullptr;
4211	}
4212
4213	/// Emit an argument that's being passed call-by-writeback. That is,
4214	/// we are passing the address of an __autoreleased temporary; it
4215	/// might be copy-initialized with the current value of the given
4216	/// address, but it will definitely be copied out of after the call.
4217	static void emitWritebackArg(CodeGenFunction &CGF, CallArgList &args,
4218	const ObjCIndirectCopyRestoreExpr *CRE) {
4219	LValue srcLV;
4220
4221	// Make an optimistic effort to emit the address as an l-value.
4222	// This can fail if the argument expression is more complicated.
4223	if (const Expr *lvExpr = maybeGetUnaryAddrOfOperand(E: CRE->getSubExpr())) {
4224	srcLV = CGF.EmitLValue(E: lvExpr);
4225
4226	// Otherwise, just emit it as a scalar.
4227	} else {
4228	Address srcAddr = CGF.EmitPointerWithAlignment(Addr: CRE->getSubExpr());
4229
4230	QualType srcAddrType =
4231	CRE->getSubExpr()->getType()->castAs<PointerType>()->getPointeeType();
4232	srcLV = CGF.MakeAddrLValue(Addr: srcAddr, T: srcAddrType);
4233	}
4234	Address srcAddr = srcLV.getAddress(CGF);
4235
4236	// The dest and src types don't necessarily match in LLVM terms
4237	// because of the crazy ObjC compatibility rules.
4238
4239	llvm::PointerType *destType =
4240	cast<llvm::PointerType>(CGF.ConvertType(CRE->getType()));
4241	llvm::Type *destElemType =
4242	CGF.ConvertTypeForMem(T: CRE->getType()->getPointeeType());
4243
4244	// If the address is a constant null, just pass the appropriate null.
4245	if (isProvablyNull(addr: srcAddr.getPointer())) {
4246	args.add(rvalue: RValue::get(V: llvm::ConstantPointerNull::get(T: destType)),
4247	type: CRE->getType());
4248	return;
4249	}
4250
4251	// Create the temporary.
4252	Address temp =
4253	CGF.CreateTempAlloca(destElemType, CGF.getPointerAlign(), "icr.temp");
4254	// Loading an l-value can introduce a cleanup if the l-value is __weak,
4255	// and that cleanup will be conditional if we can't prove that the l-value
4256	// isn't null, so we need to register a dominating point so that the cleanups
4257	// system will make valid IR.
4258	CodeGenFunction::ConditionalEvaluation condEval(CGF);
4259
4260	// Zero-initialize it if we're not doing a copy-initialization.
4261	bool shouldCopy = CRE->shouldCopy();
4262	if (!shouldCopy) {
4263	llvm::Value *null =
4264	llvm::ConstantPointerNull::get(T: cast<llvm::PointerType>(Val: destElemType));
4265	CGF.Builder.CreateStore(Val: null, Addr: temp);
4266	}
4267
4268	llvm::BasicBlock *contBB = nullptr;
4269	llvm::BasicBlock *originBB = nullptr;
4270
4271	// If the address is *not* known to be non-null, we need to switch.
4272	llvm::Value *finalArgument;
4273
4274	bool provablyNonNull = llvm::isKnownNonZero(V: srcAddr.getPointer(),
4275	DL: CGF.CGM.getDataLayout());
4276	if (provablyNonNull) {
4277	finalArgument = temp.getPointer();
4278	} else {
4279	llvm::Value *isNull =
4280	CGF.Builder.CreateIsNull(Arg: srcAddr.getPointer(), Name: "icr.isnull");
4281
4282	finalArgument = CGF.Builder.CreateSelect(C: isNull,
4283	True: llvm::ConstantPointerNull::get(T: destType),
4284	False: temp.getPointer(), Name: "icr.argument");
4285
4286	// If we need to copy, then the load has to be conditional, which
4287	// means we need control flow.
4288	if (shouldCopy) {
4289	originBB = CGF.Builder.GetInsertBlock();
4290	contBB = CGF.createBasicBlock(name: "icr.cont");
4291	llvm::BasicBlock *copyBB = CGF.createBasicBlock(name: "icr.copy");
4292	CGF.Builder.CreateCondBr(Cond: isNull, True: contBB, False: copyBB);
4293	CGF.EmitBlock(BB: copyBB);
4294	condEval.begin(CGF);
4295	}
4296	}
4297
4298	llvm::Value *valueToUse = nullptr;
4299
4300	// Perform a copy if necessary.
4301	if (shouldCopy) {
4302	RValue srcRV = CGF.EmitLoadOfLValue(V: srcLV, Loc: SourceLocation());
4303	assert(srcRV.isScalar());
4304
4305	llvm::Value *src = srcRV.getScalarVal();
4306	src = CGF.Builder.CreateBitCast(V: src, DestTy: destElemType, Name: "icr.cast");
4307
4308	// Use an ordinary store, not a store-to-lvalue.
4309	CGF.Builder.CreateStore(Val: src, Addr: temp);
4310
4311	// If optimization is enabled, and the value was held in a
4312	// __strong variable, we need to tell the optimizer that this
4313	// value has to stay alive until we're doing the store back.
4314	// This is because the temporary is effectively unretained,
4315	// and so otherwise we can violate the high-level semantics.
4316	if (CGF.CGM.getCodeGenOpts().OptimizationLevel != 0 &&
4317	srcLV.getObjCLifetime() == Qualifiers::OCL_Strong) {
4318	valueToUse = src;
4319	}
4320	}
4321
4322	// Finish the control flow if we needed it.
4323	if (shouldCopy && !provablyNonNull) {
4324	llvm::BasicBlock *copyBB = CGF.Builder.GetInsertBlock();
4325	CGF.EmitBlock(BB: contBB);
4326
4327	// Make a phi for the value to intrinsically use.
4328	if (valueToUse) {
4329	llvm::PHINode *phiToUse = CGF.Builder.CreatePHI(Ty: valueToUse->getType(), NumReservedValues: 2,
4330	Name: "icr.to-use");
4331	phiToUse->addIncoming(V: valueToUse, BB: copyBB);
4332	phiToUse->addIncoming(V: llvm::UndefValue::get(T: valueToUse->getType()),
4333	BB: originBB);
4334	valueToUse = phiToUse;
4335	}
4336
4337	condEval.end(CGF);
4338	}
4339
4340	args.addWriteback(srcLV, temporary: temp, toUse: valueToUse);
4341	args.add(rvalue: RValue::get(V: finalArgument), type: CRE->getType());
4342	}
4343
4344	void CallArgList::allocateArgumentMemory(CodeGenFunction &CGF) {
4345	assert(!StackBase);
4346
4347	// Save the stack.
4348	StackBase = CGF.Builder.CreateStackSave(Name: "inalloca.save");
4349	}
4350
4351	void CallArgList::freeArgumentMemory(CodeGenFunction &CGF) const {
4352	if (StackBase) {
4353	// Restore the stack after the call.
4354	CGF.Builder.CreateStackRestore(Ptr: StackBase);
4355	}
4356	}
4357
4358	void CodeGenFunction::EmitNonNullArgCheck(RValue RV, QualType ArgType,
4359	SourceLocation ArgLoc,
4360	AbstractCallee AC,
4361	unsigned ParmNum) {
4362	if (!AC.getDecl() || !(SanOpts.has(K: SanitizerKind::NonnullAttribute) ||
4363	SanOpts.has(K: SanitizerKind::NullabilityArg)))
4364	return;
4365
4366	// The param decl may be missing in a variadic function.
4367	auto PVD = ParmNum < AC.getNumParams() ? AC.getParamDecl(I: ParmNum) : nullptr;
4368	unsigned ArgNo = PVD ? PVD->getFunctionScopeIndex() : ParmNum;
4369
4370	// Prefer the nonnull attribute if it's present.
4371	const NonNullAttr *NNAttr = nullptr;
4372	if (SanOpts.has(SanitizerKind::NonnullAttribute))
4373	NNAttr = getNonNullAttr(AC.getDecl(), PVD, ArgType, ArgNo);
4374
4375	bool CanCheckNullability = false;
4376	if (SanOpts.has(K: SanitizerKind::NullabilityArg) && !NNAttr && PVD) {
4377	auto Nullability = PVD->getType()->getNullability();
4378	CanCheckNullability = Nullability &&
4379	*Nullability == NullabilityKind::NonNull &&
4380	PVD->getTypeSourceInfo();
4381	}
4382
4383	if (!NNAttr && !CanCheckNullability)
4384	return;
4385
4386	SourceLocation AttrLoc;
4387	SanitizerMask CheckKind;
4388	SanitizerHandler Handler;
4389	if (NNAttr) {
4390	AttrLoc = NNAttr->getLocation();
4391	CheckKind = SanitizerKind::NonnullAttribute;
4392	Handler = SanitizerHandler::NonnullArg;
4393	} else {
4394	AttrLoc = PVD->getTypeSourceInfo()->getTypeLoc().findNullabilityLoc();
4395	CheckKind = SanitizerKind::NullabilityArg;
4396	Handler = SanitizerHandler::NullabilityArg;
4397	}
4398
4399	SanitizerScope SanScope(this);
4400	llvm::Value *Cond = EmitNonNullRValueCheck(RV, T: ArgType);
4401	llvm::Constant *StaticData[] = {
4402	EmitCheckSourceLocation(Loc: ArgLoc), EmitCheckSourceLocation(Loc: AttrLoc),
4403	llvm::ConstantInt::get(Ty: Int32Ty, V: ArgNo + 1),
4404	};
4405	EmitCheck(Checked: std::make_pair(x&: Cond, y&: CheckKind), Check: Handler, StaticArgs: StaticData, DynamicArgs: std::nullopt);
4406	}
4407
4408	// Check if the call is going to use the inalloca convention. This needs to
4409	// agree with CGFunctionInfo::usesInAlloca. The CGFunctionInfo is arranged
4410	// later, so we can't check it directly.
4411	static bool hasInAllocaArgs(CodeGenModule &CGM, CallingConv ExplicitCC,
4412	ArrayRef<QualType> ArgTypes) {
4413	// The Swift calling conventions don't go through the target-specific
4414	// argument classification, they never use inalloca.
4415	// TODO: Consider limiting inalloca use to only calling conventions supported
4416	// by MSVC.
4417	if (ExplicitCC == CC_Swift || ExplicitCC == CC_SwiftAsync)
4418	return false;
4419	if (!CGM.getTarget().getCXXABI().isMicrosoft())
4420	return false;
4421	return llvm::any_of(Range&: ArgTypes, P: [&](QualType Ty) {
4422	return isInAllocaArgument(ABI&: CGM.getCXXABI(), type: Ty);
4423	});
4424	}
4425
4426	#ifndef NDEBUG
4427	// Determine whether the given argument is an Objective-C method
4428	// that may have type parameters in its signature.
4429	static bool isObjCMethodWithTypeParams(const ObjCMethodDecl *method) {
4430	const DeclContext *dc = method->getDeclContext();
4431	if (const ObjCInterfaceDecl *classDecl = dyn_cast<ObjCInterfaceDecl>(Val: dc)) {
4432	return classDecl->getTypeParamListAsWritten();
4433	}
4434
4435	if (const ObjCCategoryDecl *catDecl = dyn_cast<ObjCCategoryDecl>(Val: dc)) {
4436	return catDecl->getTypeParamList();
4437	}
4438
4439	return false;
4440	}
4441	#endif
4442
4443	/// EmitCallArgs - Emit call arguments for a function.
4444	void CodeGenFunction::EmitCallArgs(
4445	CallArgList &Args, PrototypeWrapper Prototype,
4446	llvm::iterator_range<CallExpr::const_arg_iterator> ArgRange,
4447	AbstractCallee AC, unsigned ParamsToSkip, EvaluationOrder Order) {
4448	SmallVector<QualType, 16> ArgTypes;
4449
4450	assert((ParamsToSkip == 0 || Prototype.P) &&
4451	"Can't skip parameters if type info is not provided");
4452
4453	// This variable only captures *explicitly* written conventions, not those
4454	// applied by default via command line flags or target defaults, such as
4455	// thiscall, aapcs, stdcall via -mrtd, etc. Computing that correctly would
4456	// require knowing if this is a C++ instance method or being able to see
4457	// unprototyped FunctionTypes.
4458	CallingConv ExplicitCC = CC_C;
4459
4460	// First, if a prototype was provided, use those argument types.
4461	bool IsVariadic = false;
4462	if (Prototype.P) {
4463	const auto *MD = Prototype.P.dyn_cast<const ObjCMethodDecl *>();
4464	if (MD) {
4465	IsVariadic = MD->isVariadic();
4466	ExplicitCC = getCallingConventionForDecl(
4467	D: MD, IsWindows: CGM.getTarget().getTriple().isOSWindows());
4468	ArgTypes.assign(in_start: MD->param_type_begin() + ParamsToSkip,
4469	in_end: MD->param_type_end());
4470	} else {
4471	const auto *FPT = Prototype.P.get<const FunctionProtoType *>();
4472	IsVariadic = FPT->isVariadic();
4473	ExplicitCC = FPT->getExtInfo().getCC();
4474	ArgTypes.assign(in_start: FPT->param_type_begin() + ParamsToSkip,
4475	in_end: FPT->param_type_end());
4476	}
4477
4478	#ifndef NDEBUG
4479	// Check that the prototyped types match the argument expression types.
4480	bool isGenericMethod = MD && isObjCMethodWithTypeParams(method: MD);
4481	CallExpr::const_arg_iterator Arg = ArgRange.begin();
4482	for (QualType Ty : ArgTypes) {
4483	assert(Arg != ArgRange.end() && "Running over edge of argument list!");
4484	assert(
4485	(isGenericMethod || Ty->isVariablyModifiedType() ||
4486	Ty.getNonReferenceType()->isObjCRetainableType() ||
4487	getContext()
4488	.getCanonicalType(Ty.getNonReferenceType())
4489	.getTypePtr() ==
4490	getContext().getCanonicalType((*Arg)->getType()).getTypePtr()) &&
4491	"type mismatch in call argument!");
4492	++Arg;
4493	}
4494
4495	// Either we've emitted all the call args, or we have a call to variadic
4496	// function.
4497	assert((Arg == ArgRange.end() || IsVariadic) &&
4498	"Extra arguments in non-variadic function!");
4499	#endif
4500	}
4501
4502	// If we still have any arguments, emit them using the type of the argument.
4503	for (auto *A : llvm::drop_begin(RangeOrContainer&: ArgRange, N: ArgTypes.size()))
4504	ArgTypes.push_back(Elt: IsVariadic ? getVarArgType(A) : A->getType());
4505	assert((int)ArgTypes.size() == (ArgRange.end() - ArgRange.begin()));
4506
4507	// We must evaluate arguments from right to left in the MS C++ ABI,
4508	// because arguments are destroyed left to right in the callee. As a special
4509	// case, there are certain language constructs that require left-to-right
4510	// evaluation, and in those cases we consider the evaluation order requirement
4511	// to trump the "destruction order is reverse construction order" guarantee.
4512	bool LeftToRight =
4513	CGM.getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee()
4514	? Order == EvaluationOrder::ForceLeftToRight
4515	: Order != EvaluationOrder::ForceRightToLeft;
4516
4517	auto MaybeEmitImplicitObjectSize = [&](unsigned I, const Expr *Arg,
4518	RValue EmittedArg) {
4519	if (!AC.hasFunctionDecl() || I >= AC.getNumParams())
4520	return;
4521	auto *PS = AC.getParamDecl(I)->getAttr<PassObjectSizeAttr>();
4522	if (PS == nullptr)
4523	return;
4524
4525	const auto &Context = getContext();
4526	auto SizeTy = Context.getSizeType();
4527	auto T = Builder.getIntNTy(N: Context.getTypeSize(T: SizeTy));
4528	assert(EmittedArg.getScalarVal() && "We emitted nothing for the arg?");
4529	llvm::Value *V = evaluateOrEmitBuiltinObjectSize(E: Arg, Type: PS->getType(), ResType: T,
4530	EmittedE: EmittedArg.getScalarVal(),
4531	IsDynamic: PS->isDynamic());
4532	Args.add(rvalue: RValue::get(V), type: SizeTy);
4533	// If we're emitting args in reverse, be sure to do so with
4534	// pass_object_size, as well.
4535	if (!LeftToRight)
4536	std::swap(a&: Args.back(), b&: *(&Args.back() - 1));
4537	};
4538
4539	// Insert a stack save if we're going to need any inalloca args.
4540	if (hasInAllocaArgs(CGM, ExplicitCC, ArgTypes)) {
4541	assert(getTarget().getTriple().getArch() == llvm::Triple::x86 &&
4542	"inalloca only supported on x86");
4543	Args.allocateArgumentMemory(CGF&: *this);
4544	}
4545
4546	// Evaluate each argument in the appropriate order.
4547	size_t CallArgsStart = Args.size();
4548	for (unsigned I = 0, E = ArgTypes.size(); I != E; ++I) {
4549	unsigned Idx = LeftToRight ? I : E - I - 1;
4550	CallExpr::const_arg_iterator Arg = ArgRange.begin() + Idx;
4551	unsigned InitialArgSize = Args.size();
4552	// If *Arg is an ObjCIndirectCopyRestoreExpr, check that either the types of
4553	// the argument and parameter match or the objc method is parameterized.
4554	assert((!isa<ObjCIndirectCopyRestoreExpr>(*Arg) ||
4555	getContext().hasSameUnqualifiedType((*Arg)->getType(),
4556	ArgTypes[Idx]) ||
4557	(isa<ObjCMethodDecl>(AC.getDecl()) &&
4558	isObjCMethodWithTypeParams(cast<ObjCMethodDecl>(AC.getDecl())))) &&
4559	"Argument and parameter types don't match");
4560	EmitCallArg(Args, *Arg, ArgTypes[Idx]);
4561	// In particular, we depend on it being the last arg in Args, and the
4562	// objectsize bits depend on there only being one arg if !LeftToRight.
4563	assert(InitialArgSize + 1 == Args.size() &&
4564	"The code below depends on only adding one arg per EmitCallArg");
4565	(void)InitialArgSize;
4566	// Since pointer argument are never emitted as LValue, it is safe to emit
4567	// non-null argument check for r-value only.
4568	if (!Args.back().hasLValue()) {
4569	RValue RVArg = Args.back().getKnownRValue();
4570	EmitNonNullArgCheck(RV: RVArg, ArgType: ArgTypes[Idx], ArgLoc: (*Arg)->getExprLoc(), AC,
4571	ParmNum: ParamsToSkip + Idx);
4572	// @llvm.objectsize should never have side-effects and shouldn't need
4573	// destruction/cleanups, so we can safely "emit" it after its arg,
4574	// regardless of right-to-leftness
4575	MaybeEmitImplicitObjectSize(Idx, *Arg, RVArg);
4576	}
4577	}
4578
4579	if (!LeftToRight) {
4580	// Un-reverse the arguments we just evaluated so they match up with the LLVM
4581	// IR function.
4582	std::reverse(first: Args.begin() + CallArgsStart, last: Args.end());
4583	}
4584	}
4585
4586	namespace {
4587
4588	struct DestroyUnpassedArg final : EHScopeStack::Cleanup {
4589	DestroyUnpassedArg(Address Addr, QualType Ty)
4590	: Addr(Addr), Ty(Ty) {}
4591
4592	Address Addr;
4593	QualType Ty;
4594
4595	void Emit(CodeGenFunction &CGF, Flags flags) override {
4596	QualType::DestructionKind DtorKind = Ty.isDestructedType();
4597	if (DtorKind == QualType::DK_cxx_destructor) {
4598	const CXXDestructorDecl *Dtor = Ty->getAsCXXRecordDecl()->getDestructor();
4599	assert(!Dtor->isTrivial());
4600	CGF.EmitCXXDestructorCall(Dtor, Dtor_Complete, /*for vbase*/ false,
4601	/*Delegating=*/false, Addr, Ty);
4602	} else {
4603	CGF.callCStructDestructor(CGF.MakeAddrLValue(Addr, Ty));
4604	}
4605	}
4606	};
4607
4608	struct DisableDebugLocationUpdates {
4609	CodeGenFunction &CGF;
4610	bool disabledDebugInfo;
4611	DisableDebugLocationUpdates(CodeGenFunction &CGF, const Expr *E) : CGF(CGF) {
4612	if ((disabledDebugInfo = isa<CXXDefaultArgExpr>(Val: E) && CGF.getDebugInfo()))
4613	CGF.disableDebugInfo();
4614	}
4615	~DisableDebugLocationUpdates() {
4616	if (disabledDebugInfo)
4617	CGF.enableDebugInfo();
4618	}
4619	};
4620
4621	} // end anonymous namespace
4622
4623	RValue CallArg::getRValue(CodeGenFunction &CGF) const {
4624	if (!HasLV)
4625	return RV;
4626	LValue Copy = CGF.MakeAddrLValue(CGF.CreateMemTemp(Ty), Ty);
4627	CGF.EmitAggregateCopy(Copy, LV, Ty, AggValueSlot::DoesNotOverlap,
4628	LV.isVolatile());
4629	IsUsed = true;
4630	return RValue::getAggregate(addr: Copy.getAddress(CGF));
4631	}
4632
4633	void CallArg::copyInto(CodeGenFunction &CGF, Address Addr) const {
4634	LValue Dst = CGF.MakeAddrLValue(Addr, Ty);
4635	if (!HasLV && RV.isScalar())
4636	CGF.EmitStoreOfScalar(RV.getScalarVal(), Dst, /*isInit=*/true);
4637	else if (!HasLV && RV.isComplex())
4638	CGF.EmitStoreOfComplex(RV.getComplexVal(), Dst, /*init=*/true);
4639	else {
4640	auto Addr = HasLV ? LV.getAddress(CGF) : RV.getAggregateAddress();
4641	LValue SrcLV = CGF.MakeAddrLValue(Addr, Ty);
4642	// We assume that call args are never copied into subobjects.
4643	CGF.EmitAggregateCopy(Dst, SrcLV, Ty, AggValueSlot::DoesNotOverlap,
4644	HasLV ? LV.isVolatileQualified()
4645	: RV.isVolatileQualified());
4646	}
4647	IsUsed = true;
4648	}
4649
4650	void CodeGenFunction::EmitCallArg(CallArgList &args, const Expr *E,
4651	QualType type) {
4652	DisableDebugLocationUpdates Dis(*this, E);
4653	if (const ObjCIndirectCopyRestoreExpr *CRE
4654	= dyn_cast<ObjCIndirectCopyRestoreExpr>(Val: E)) {
4655	assert(getLangOpts().ObjCAutoRefCount);
4656	return emitWritebackArg(CGF&: *this, args, CRE);
4657	}
4658
4659	assert(type->isReferenceType() == E->isGLValue() &&
4660	"reference binding to unmaterialized r-value!");
4661
4662	if (E->isGLValue()) {
4663	assert(E->getObjectKind() == OK_Ordinary);
4664	return args.add(rvalue: EmitReferenceBindingToExpr(E), type);
4665	}
4666
4667	bool HasAggregateEvalKind = hasAggregateEvaluationKind(T: type);
4668
4669	// In the Microsoft C++ ABI, aggregate arguments are destructed by the callee.
4670	// However, we still have to push an EH-only cleanup in case we unwind before
4671	// we make it to the call.
4672	if (type->isRecordType() &&
4673	type->castAs<RecordType>()->getDecl()->isParamDestroyedInCallee()) {
4674	// If we're using inalloca, use the argument memory. Otherwise, use a
4675	// temporary.
4676	AggValueSlot Slot = args.isUsingInAlloca()
4677	? createPlaceholderSlot(CGF&: *this, Ty: type) : CreateAggTemp(T: type, Name: "agg.tmp");
4678
4679	bool DestroyedInCallee = true, NeedsEHCleanup = true;
4680	if (const auto *RD = type->getAsCXXRecordDecl())
4681	DestroyedInCallee = RD->hasNonTrivialDestructor();
4682	else
4683	NeedsEHCleanup = needsEHCleanup(kind: type.isDestructedType());
4684
4685	if (DestroyedInCallee)
4686	Slot.setExternallyDestructed();
4687
4688	EmitAggExpr(E, AS: Slot);
4689	RValue RV = Slot.asRValue();
4690	args.add(rvalue: RV, type);
4691
4692	if (DestroyedInCallee && NeedsEHCleanup) {
4693	// Create a no-op GEP between the placeholder and the cleanup so we can
4694	// RAUW it successfully. It also serves as a marker of the first
4695	// instruction where the cleanup is active.
4696	pushFullExprCleanup<DestroyUnpassedArg>(kind: EHCleanup, A: Slot.getAddress(),
4697	A: type);
4698	// This unreachable is a temporary marker which will be removed later.
4699	llvm::Instruction *IsActive = Builder.CreateUnreachable();
4700	args.addArgCleanupDeactivation(Cleanup: EHStack.stable_begin(), IsActiveIP: IsActive);
4701	}
4702	return;
4703	}
4704
4705	if (HasAggregateEvalKind && isa<ImplicitCastExpr>(Val: E) &&
4706	cast<CastExpr>(Val: E)->getCastKind() == CK_LValueToRValue) {
4707	LValue L = EmitLValue(E: cast<CastExpr>(Val: E)->getSubExpr());
4708	assert(L.isSimple());
4709	args.addUncopiedAggregate(LV: L, type);
4710	return;
4711	}
4712
4713	args.add(rvalue: EmitAnyExprToTemp(E), type);
4714	}
4715
4716	QualType CodeGenFunction::getVarArgType(const Expr *Arg) {
4717	// System headers on Windows define NULL to 0 instead of 0LL on Win64. MSVC
4718	// implicitly widens null pointer constants that are arguments to varargs
4719	// functions to pointer-sized ints.
4720	if (!getTarget().getTriple().isOSWindows())
4721	return Arg->getType();
4722
4723	if (Arg->getType()->isIntegerType() &&
4724	getContext().getTypeSize(T: Arg->getType()) <
4725	getContext().getTargetInfo().getPointerWidth(AddrSpace: LangAS::Default) &&
4726	Arg->isNullPointerConstant(Ctx&: getContext(),
4727	NPC: Expr::NPC_ValueDependentIsNotNull)) {
4728	return getContext().getIntPtrType();
4729	}
4730
4731	return Arg->getType();
4732	}
4733
4734	// In ObjC ARC mode with no ObjC ARC exception safety, tell the ARC
4735	// optimizer it can aggressively ignore unwind edges.
4736	void
4737	CodeGenFunction::AddObjCARCExceptionMetadata(llvm::Instruction *Inst) {
4738	if (CGM.getCodeGenOpts().OptimizationLevel != 0 &&
4739	!CGM.getCodeGenOpts().ObjCAutoRefCountExceptions)
4740	Inst->setMetadata(Kind: "clang.arc.no_objc_arc_exceptions",
4741	Node: CGM.getNoObjCARCExceptionsMetadata());
4742	}
4743
4744	/// Emits a call to the given no-arguments nounwind runtime function.
4745	llvm::CallInst *
4746	CodeGenFunction::EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
4747	const llvm::Twine &name) {
4748	return EmitNounwindRuntimeCall(callee, args: std::nullopt, name);
4749	}
4750
4751	/// Emits a call to the given nounwind runtime function.
4752	llvm::CallInst *
4753	CodeGenFunction::EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
4754	ArrayRef<llvm::Value *> args,
4755	const llvm::Twine &name) {
4756	llvm::CallInst *call = EmitRuntimeCall(callee, args, name);
4757	call->setDoesNotThrow();
4758	return call;
4759	}
4760
4761	/// Emits a simple call (never an invoke) to the given no-arguments
4762	/// runtime function.
4763	llvm::CallInst *CodeGenFunction::EmitRuntimeCall(llvm::FunctionCallee callee,
4764	const llvm::Twine &name) {
4765	return EmitRuntimeCall(callee, args: std::nullopt, name);
4766	}
4767
4768	// Calls which may throw must have operand bundles indicating which funclet
4769	// they are nested within.
4770	SmallVector<llvm::OperandBundleDef, 1>
4771	CodeGenFunction::getBundlesForFunclet(llvm::Value *Callee) {
4772	// There is no need for a funclet operand bundle if we aren't inside a
4773	// funclet.
4774	if (!CurrentFuncletPad)
4775	return (SmallVector<llvm::OperandBundleDef, 1>());
4776
4777	// Skip intrinsics which cannot throw (as long as they don't lower into
4778	// regular function calls in the course of IR transformations).
4779	if (auto *CalleeFn = dyn_cast<llvm::Function>(Val: Callee->stripPointerCasts())) {
4780	if (CalleeFn->isIntrinsic() && CalleeFn->doesNotThrow()) {
4781	auto IID = CalleeFn->getIntrinsicID();
4782	if (!llvm::IntrinsicInst::mayLowerToFunctionCall(IID))
4783	return (SmallVector<llvm::OperandBundleDef, 1>());
4784	}
4785	}
4786
4787	SmallVector<llvm::OperandBundleDef, 1> BundleList;
4788	BundleList.emplace_back(Args: "funclet", Args&: CurrentFuncletPad);
4789	return BundleList;
4790	}
4791
4792	/// Emits a simple call (never an invoke) to the given runtime function.
4793	llvm::CallInst *CodeGenFunction::EmitRuntimeCall(llvm::FunctionCallee callee,
4794	ArrayRef<llvm::Value *> args,
4795	const llvm::Twine &name) {
4796	llvm::CallInst *call = Builder.CreateCall(
4797	Callee: callee, Args: args, OpBundles: getBundlesForFunclet(Callee: callee.getCallee()), Name: name);
4798	call->setCallingConv(getRuntimeCC());
4799	return call;
4800	}
4801
4802	/// Emits a call or invoke to the given noreturn runtime function.
4803	void CodeGenFunction::EmitNoreturnRuntimeCallOrInvoke(
4804	llvm::FunctionCallee callee, ArrayRef<llvm::Value *> args) {
4805	SmallVector<llvm::OperandBundleDef, 1> BundleList =
4806	getBundlesForFunclet(Callee: callee.getCallee());
4807
4808	if (getInvokeDest()) {
4809	llvm::InvokeInst *invoke =
4810	Builder.CreateInvoke(Callee: callee,
4811	NormalDest: getUnreachableBlock(),
4812	UnwindDest: getInvokeDest(),
4813	Args: args,
4814	OpBundles: BundleList);
4815	invoke->setDoesNotReturn();
4816	invoke->setCallingConv(getRuntimeCC());
4817	} else {
4818	llvm::CallInst *call = Builder.CreateCall(Callee: callee, Args: args, OpBundles: BundleList);
4819	call->setDoesNotReturn();
4820	call->setCallingConv(getRuntimeCC());
4821	Builder.CreateUnreachable();
4822	}
4823	}
4824
4825	/// Emits a call or invoke instruction to the given nullary runtime function.
4826	llvm::CallBase *
4827	CodeGenFunction::EmitRuntimeCallOrInvoke(llvm::FunctionCallee callee,
4828	const Twine &name) {
4829	return EmitRuntimeCallOrInvoke(callee, args: std::nullopt, name);
4830	}
4831
4832	/// Emits a call or invoke instruction to the given runtime function.
4833	llvm::CallBase *
4834	CodeGenFunction::EmitRuntimeCallOrInvoke(llvm::FunctionCallee callee,
4835	ArrayRef<llvm::Value *> args,
4836	const Twine &name) {
4837	llvm::CallBase *call = EmitCallOrInvoke(Callee: callee, Args: args, Name: name);
4838	call->setCallingConv(getRuntimeCC());
4839	return call;
4840	}
4841
4842	/// Emits a call or invoke instruction to the given function, depending
4843	/// on the current state of the EH stack.
4844	llvm::CallBase *CodeGenFunction::EmitCallOrInvoke(llvm::FunctionCallee Callee,
4845	ArrayRef<llvm::Value *> Args,
4846	const Twine &Name) {
4847	llvm::BasicBlock *InvokeDest = getInvokeDest();
4848	SmallVector<llvm::OperandBundleDef, 1> BundleList =
4849	getBundlesForFunclet(Callee: Callee.getCallee());
4850
4851	llvm::CallBase *Inst;
4852	if (!InvokeDest)
4853	Inst = Builder.CreateCall(Callee, Args, OpBundles: BundleList, Name);
4854	else {
4855	llvm::BasicBlock *ContBB = createBasicBlock(name: "invoke.cont");
4856	Inst = Builder.CreateInvoke(Callee, NormalDest: ContBB, UnwindDest: InvokeDest, Args, OpBundles: BundleList,
4857	Name);
4858	EmitBlock(BB: ContBB);
4859	}
4860
4861	// In ObjC ARC mode with no ObjC ARC exception safety, tell the ARC
4862	// optimizer it can aggressively ignore unwind edges.
4863	if (CGM.getLangOpts().ObjCAutoRefCount)
4864	AddObjCARCExceptionMetadata(Inst);
4865
4866	return Inst;
4867	}
4868
4869	void CodeGenFunction::deferPlaceholderReplacement(llvm::Instruction *Old,
4870	llvm::Value *New) {
4871	DeferredReplacements.push_back(
4872	Elt: std::make_pair(x: llvm::WeakTrackingVH(Old), y&: New));
4873	}
4874
4875	namespace {
4876
4877	/// Specify given \p NewAlign as the alignment of return value attribute. If
4878	/// such attribute already exists, re-set it to the maximal one of two options.
4879	[[nodiscard]] llvm::AttributeList
4880	maybeRaiseRetAlignmentAttribute(llvm::LLVMContext &Ctx,
4881	const llvm::AttributeList &Attrs,
4882	llvm::Align NewAlign) {
4883	llvm::Align CurAlign = Attrs.getRetAlignment().valueOrOne();
4884	if (CurAlign >= NewAlign)
4885	return Attrs;
4886	llvm::Attribute AlignAttr = llvm::Attribute::getWithAlignment(Context&: Ctx, Alignment: NewAlign);
4887	return Attrs.removeRetAttribute(Ctx, llvm::Attribute::AttrKind::Alignment)
4888	.addRetAttribute(Ctx, AlignAttr);
4889	}
4890
4891	template <typename AlignedAttrTy> class AbstractAssumeAlignedAttrEmitter {
4892	protected:
4893	CodeGenFunction &CGF;
4894
4895	/// We do nothing if this is, or becomes, nullptr.
4896	const AlignedAttrTy *AA = nullptr;
4897
4898	llvm::Value *Alignment = nullptr; // May or may not be a constant.
4899	llvm::ConstantInt *OffsetCI = nullptr; // Constant, hopefully zero.
4900
4901	AbstractAssumeAlignedAttrEmitter(CodeGenFunction &CGF_, const Decl *FuncDecl)
4902	: CGF(CGF_) {
4903	if (!FuncDecl)
4904	return;
4905	AA = FuncDecl->getAttr<AlignedAttrTy>();
4906	}
4907
4908	public:
4909	/// If we can, materialize the alignment as an attribute on return value.
4910	[[nodiscard]] llvm::AttributeList
4911	TryEmitAsCallSiteAttribute(const llvm::AttributeList &Attrs) {
4912	if (!AA || OffsetCI || CGF.SanOpts.has(K: SanitizerKind::Alignment))
4913	return Attrs;
4914	const auto *AlignmentCI = dyn_cast<llvm::ConstantInt>(Val: Alignment);
4915	if (!AlignmentCI)
4916	return Attrs;
4917	// We may legitimately have non-power-of-2 alignment here.
4918	// If so, this is UB land, emit it via `@llvm.assume` instead.
4919	if (!AlignmentCI->getValue().isPowerOf2())
4920	return Attrs;
4921	llvm::AttributeList NewAttrs = maybeRaiseRetAlignmentAttribute(
4922	Ctx&: CGF.getLLVMContext(), Attrs,
4923	NewAlign: llvm::Align(
4924	AlignmentCI->getLimitedValue(Limit: llvm::Value::MaximumAlignment)));
4925	AA = nullptr; // We're done. Disallow doing anything else.
4926	return NewAttrs;
4927	}
4928
4929	/// Emit alignment assumption.
4930	/// This is a general fallback that we take if either there is an offset,
4931	/// or the alignment is variable or we are sanitizing for alignment.
4932	void EmitAsAnAssumption(SourceLocation Loc, QualType RetTy, RValue &Ret) {
4933	if (!AA)
4934	return;
4935	CGF.emitAlignmentAssumption(Ret.getScalarVal(), RetTy, Loc,
4936	AA->getLocation(), Alignment, OffsetCI);
4937	AA = nullptr; // We're done. Disallow doing anything else.
4938	}
4939	};
4940
4941	/// Helper data structure to emit `AssumeAlignedAttr`.
4942	class AssumeAlignedAttrEmitter final
4943	: public AbstractAssumeAlignedAttrEmitter<AssumeAlignedAttr> {
4944	public:
4945	AssumeAlignedAttrEmitter(CodeGenFunction &CGF_, const Decl *FuncDecl)
4946	: AbstractAssumeAlignedAttrEmitter(CGF_, FuncDecl) {
4947	if (!AA)
4948	return;
4949	// It is guaranteed that the alignment/offset are constants.
4950	Alignment = cast<llvm::ConstantInt>(CGF.EmitScalarExpr(AA->getAlignment()));
4951	if (Expr *Offset = AA->getOffset()) {
4952	OffsetCI = cast<llvm::ConstantInt>(CGF.EmitScalarExpr(Offset));
4953	if (OffsetCI->isNullValue()) // Canonicalize zero offset to no offset.
4954	OffsetCI = nullptr;
4955	}
4956	}
4957	};
4958
4959	/// Helper data structure to emit `AllocAlignAttr`.
4960	class AllocAlignAttrEmitter final
4961	: public AbstractAssumeAlignedAttrEmitter<AllocAlignAttr> {
4962	public:
4963	AllocAlignAttrEmitter(CodeGenFunction &CGF_, const Decl *FuncDecl,
4964	const CallArgList &CallArgs)
4965	: AbstractAssumeAlignedAttrEmitter(CGF_, FuncDecl) {
4966	if (!AA)
4967	return;
4968	// Alignment may or may not be a constant, and that is okay.
4969	Alignment = CallArgs[AA->getParamIndex().getLLVMIndex()]
4970	.getRValue(CGF)
4971	.getScalarVal();
4972	}
4973	};
4974
4975	} // namespace
4976
4977	static unsigned getMaxVectorWidth(const llvm::Type *Ty) {
4978	if (auto *VT = dyn_cast<llvm::VectorType>(Val: Ty))
4979	return VT->getPrimitiveSizeInBits().getKnownMinValue();
4980	if (auto *AT = dyn_cast<llvm::ArrayType>(Val: Ty))
4981	return getMaxVectorWidth(Ty: AT->getElementType());
4982
4983	unsigned MaxVectorWidth = 0;
4984	if (auto *ST = dyn_cast<llvm::StructType>(Val: Ty))
4985	for (auto *I : ST->elements())
4986	MaxVectorWidth = std::max(a: MaxVectorWidth, b: getMaxVectorWidth(Ty: I));
4987	return MaxVectorWidth;
4988	}
4989
4990	RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
4991	const CGCallee &Callee,
4992	ReturnValueSlot ReturnValue,
4993	const CallArgList &CallArgs,
4994	llvm::CallBase **callOrInvoke, bool IsMustTail,
4995	SourceLocation Loc) {
4996	// FIXME: We no longer need the types from CallArgs; lift up and simplify.
4997
4998	assert(Callee.isOrdinary() || Callee.isVirtual());
4999
5000	// Handle struct-return functions by passing a pointer to the
5001	// location that we would like to return into.
5002	QualType RetTy = CallInfo.getReturnType();
5003	const ABIArgInfo &RetAI = CallInfo.getReturnInfo();
5004
5005	llvm::FunctionType *IRFuncTy = getTypes().GetFunctionType(FI: CallInfo);
5006
5007	const Decl *TargetDecl = Callee.getAbstractInfo().getCalleeDecl().getDecl();
5008	if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(Val: TargetDecl)) {
5009	// We can only guarantee that a function is called from the correct
5010	// context/function based on the appropriate target attributes,
5011	// so only check in the case where we have both always_inline and target
5012	// since otherwise we could be making a conditional call after a check for
5013	// the proper cpu features (and it won't cause code generation issues due to
5014	// function based code generation).
5015	if (TargetDecl->hasAttr<AlwaysInlineAttr>() &&
5016	(TargetDecl->hasAttr<TargetAttr>() ||
5017	(CurFuncDecl && CurFuncDecl->hasAttr<TargetAttr>())))
5018	checkTargetFeatures(Loc, TargetDecl: FD);
5019
5020	// Some architectures (such as x86-64) have the ABI changed based on
5021	// attribute-target/features. Give them a chance to diagnose.
5022	CGM.getTargetCodeGenInfo().checkFunctionCallABI(
5023	CGM, CallLoc: Loc, Caller: dyn_cast_or_null<FunctionDecl>(Val: CurCodeDecl), Callee: FD, Args: CallArgs);
5024	}
5025
5026	// 1. Set up the arguments.
5027
5028	// If we're using inalloca, insert the allocation after the stack save.
5029	// FIXME: Do this earlier rather than hacking it in here!
5030	Address ArgMemory = Address::invalid();
5031	if (llvm::StructType *ArgStruct = CallInfo.getArgStruct()) {
5032	const llvm::DataLayout &DL = CGM.getDataLayout();
5033	llvm::Instruction *IP = CallArgs.getStackBase();
5034	llvm::AllocaInst *AI;
5035	if (IP) {
5036	IP = IP->getNextNode();
5037	AI = new llvm::AllocaInst(ArgStruct, DL.getAllocaAddrSpace(),
5038	"argmem", IP);
5039	} else {
5040	AI = CreateTempAlloca(Ty: ArgStruct, Name: "argmem");
5041	}
5042	auto Align = CallInfo.getArgStructAlignment();
5043	AI->setAlignment(Align.getAsAlign());
5044	AI->setUsedWithInAlloca(true);
5045	assert(AI->isUsedWithInAlloca() && !AI->isStaticAlloca());
5046	ArgMemory = Address(AI, ArgStruct, Align);
5047	}
5048
5049	ClangToLLVMArgMapping IRFunctionArgs(CGM.getContext(), CallInfo);
5050	SmallVector<llvm::Value *, 16> IRCallArgs(IRFunctionArgs.totalIRArgs());
5051
5052	// If the call returns a temporary with struct return, create a temporary
5053	// alloca to hold the result, unless one is given to us.
5054	Address SRetPtr = Address::invalid();
5055	Address SRetAlloca = Address::invalid();
5056	llvm::Value *UnusedReturnSizePtr = nullptr;
5057	if (RetAI.isIndirect() || RetAI.isInAlloca() || RetAI.isCoerceAndExpand()) {
5058	if (!ReturnValue.isNull()) {
5059	SRetPtr = ReturnValue.getValue();
5060	} else {
5061	SRetPtr = CreateMemTemp(T: RetTy, Name: "tmp", Alloca: &SRetAlloca);
5062	if (HaveInsertPoint() && ReturnValue.isUnused()) {
5063	llvm::TypeSize size =
5064	CGM.getDataLayout().getTypeAllocSize(Ty: ConvertTypeForMem(T: RetTy));
5065	UnusedReturnSizePtr = EmitLifetimeStart(Size: size, Addr: SRetAlloca.getPointer());
5066	}
5067	}
5068	if (IRFunctionArgs.hasSRetArg()) {
5069	IRCallArgs[IRFunctionArgs.getSRetArgNo()] = SRetPtr.getPointer();
5070	} else if (RetAI.isInAlloca()) {
5071	Address Addr =
5072	Builder.CreateStructGEP(Addr: ArgMemory, Index: RetAI.getInAllocaFieldIndex());
5073	Builder.CreateStore(Val: SRetPtr.getPointer(), Addr);
5074	}
5075	}
5076
5077	Address swiftErrorTemp = Address::invalid();
5078	Address swiftErrorArg = Address::invalid();
5079
5080	// When passing arguments using temporary allocas, we need to add the
5081	// appropriate lifetime markers. This vector keeps track of all the lifetime
5082	// markers that need to be ended right after the call.
5083	SmallVector<CallLifetimeEnd, 2> CallLifetimeEndAfterCall;
5084
5085	// Translate all of the arguments as necessary to match the IR lowering.
5086	assert(CallInfo.arg_size() == CallArgs.size() &&
5087	"Mismatch between function signature & arguments.");
5088	unsigned ArgNo = 0;
5089	CGFunctionInfo::const_arg_iterator info_it = CallInfo.arg_begin();
5090	for (CallArgList::const_iterator I = CallArgs.begin(), E = CallArgs.end();
5091	I != E; ++I, ++info_it, ++ArgNo) {
5092	const ABIArgInfo &ArgInfo = info_it->info;
5093
5094	// Insert a padding argument to ensure proper alignment.
5095	if (IRFunctionArgs.hasPaddingArg(ArgNo))
5096	IRCallArgs[IRFunctionArgs.getPaddingArgNo(ArgNo)] =
5097	llvm::UndefValue::get(T: ArgInfo.getPaddingType());
5098
5099	unsigned FirstIRArg, NumIRArgs;
5100	std::tie(args&: FirstIRArg, args&: NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
5101
5102	bool ArgHasMaybeUndefAttr =
5103	IsArgumentMaybeUndef(TargetDecl, NumRequiredArgs: CallInfo.getNumRequiredArgs(), ArgNo);
5104
5105	switch (ArgInfo.getKind()) {
5106	case ABIArgInfo::InAlloca: {
5107	assert(NumIRArgs == 0);
5108	assert(getTarget().getTriple().getArch() == llvm::Triple::x86);
5109	if (I->isAggregate()) {
5110	Address Addr = I->hasLValue()
5111	? I->getKnownLValue().getAddress(CGF&: *this)
5112	: I->getKnownRValue().getAggregateAddress();
5113	llvm::Instruction *Placeholder =
5114	cast<llvm::Instruction>(Val: Addr.getPointer());
5115
5116	if (!ArgInfo.getInAllocaIndirect()) {
5117	// Replace the placeholder with the appropriate argument slot GEP.
5118	CGBuilderTy::InsertPoint IP = Builder.saveIP();
5119	Builder.SetInsertPoint(Placeholder);
5120	Addr = Builder.CreateStructGEP(Addr: ArgMemory,
5121	Index: ArgInfo.getInAllocaFieldIndex());
5122	Builder.restoreIP(IP);
5123	} else {
5124	// For indirect things such as overaligned structs, replace the
5125	// placeholder with a regular aggregate temporary alloca. Store the
5126	// address of this alloca into the struct.
5127	Addr = CreateMemTemp(info_it->type, "inalloca.indirect.tmp");
5128	Address ArgSlot = Builder.CreateStructGEP(
5129	Addr: ArgMemory, Index: ArgInfo.getInAllocaFieldIndex());
5130	Builder.CreateStore(Val: Addr.getPointer(), Addr: ArgSlot);
5131	}
5132	deferPlaceholderReplacement(Old: Placeholder, New: Addr.getPointer());
5133	} else if (ArgInfo.getInAllocaIndirect()) {
5134	// Make a temporary alloca and store the address of it into the argument
5135	// struct.
5136	Address Addr = CreateMemTempWithoutCast(
5137	I->Ty, getContext().getTypeAlignInChars(I->Ty),
5138	"indirect-arg-temp");
5139	I->copyInto(CGF&: *this, Addr);
5140	Address ArgSlot =
5141	Builder.CreateStructGEP(Addr: ArgMemory, Index: ArgInfo.getInAllocaFieldIndex());
5142	Builder.CreateStore(Val: Addr.getPointer(), Addr: ArgSlot);
5143	} else {
5144	// Store the RValue into the argument struct.
5145	Address Addr =
5146	Builder.CreateStructGEP(Addr: ArgMemory, Index: ArgInfo.getInAllocaFieldIndex());
5147	Addr = Addr.withElementType(ElemTy: ConvertTypeForMem(T: I->Ty));
5148	I->copyInto(CGF&: *this, Addr);
5149	}
5150	break;
5151	}
5152
5153	case ABIArgInfo::Indirect:
5154	case ABIArgInfo::IndirectAliased: {
5155	assert(NumIRArgs == 1);
5156	if (!I->isAggregate()) {
5157	// Make a temporary alloca to pass the argument.
5158	Address Addr = CreateMemTempWithoutCast(
5159	I->Ty, ArgInfo.getIndirectAlign(), "indirect-arg-temp");
5160
5161	llvm::Value *Val = Addr.getPointer();
5162	if (ArgHasMaybeUndefAttr)
5163	Val = Builder.CreateFreeze(V: Addr.getPointer());
5164	IRCallArgs[FirstIRArg] = Val;
5165
5166	I->copyInto(CGF&: *this, Addr);
5167	} else {
5168	// We want to avoid creating an unnecessary temporary+copy here;
5169	// however, we need one in three cases:
5170	// 1. If the argument is not byval, and we are required to copy the
5171	// source. (This case doesn't occur on any common architecture.)
5172	// 2. If the argument is byval, RV is not sufficiently aligned, and
5173	// we cannot force it to be sufficiently aligned.
5174	// 3. If the argument is byval, but RV is not located in default
5175	// or alloca address space.
5176	Address Addr = I->hasLValue()
5177	? I->getKnownLValue().getAddress(CGF&: *this)
5178	: I->getKnownRValue().getAggregateAddress();
5179	llvm::Value *V = Addr.getPointer();
5180	CharUnits Align = ArgInfo.getIndirectAlign();
5181	const llvm::DataLayout *TD = &CGM.getDataLayout();
5182
5183	assert((FirstIRArg >= IRFuncTy->getNumParams() ||
5184	IRFuncTy->getParamType(FirstIRArg)->getPointerAddressSpace() ==
5185	TD->getAllocaAddrSpace()) &&
5186	"indirect argument must be in alloca address space");
5187
5188	bool NeedCopy = false;
5189	if (Addr.getAlignment() < Align &&
5190	llvm::getOrEnforceKnownAlignment(V, PrefAlign: Align.getAsAlign(), DL: *TD) <
5191	Align.getAsAlign()) {
5192	NeedCopy = true;
5193	} else if (I->hasLValue()) {
5194	auto LV = I->getKnownLValue();
5195	auto AS = LV.getAddressSpace();
5196
5197	bool isByValOrRef =
5198	ArgInfo.isIndirectAliased() || ArgInfo.getIndirectByVal();
5199
5200	if (!isByValOrRef ||
5201	(LV.getAlignment() < getContext().getTypeAlignInChars(I->Ty))) {
5202	NeedCopy = true;
5203	}
5204	if (!getLangOpts().OpenCL) {
5205	if ((isByValOrRef &&
5206	(AS != LangAS::Default &&
5207	AS != CGM.getASTAllocaAddressSpace()))) {
5208	NeedCopy = true;
5209	}
5210	}
5211	// For OpenCL even if RV is located in default or alloca address space
5212	// we don't want to perform address space cast for it.
5213	else if ((isByValOrRef &&
5214	Addr.getType()->getAddressSpace() != IRFuncTy->
5215	getParamType(i: FirstIRArg)->getPointerAddressSpace())) {
5216	NeedCopy = true;
5217	}
5218	}
5219
5220	if (NeedCopy) {
5221	// Create an aligned temporary, and copy to it.
5222	Address AI = CreateMemTempWithoutCast(
5223	I->Ty, ArgInfo.getIndirectAlign(), "byval-temp");
5224	llvm::Value *Val = AI.getPointer();
5225	if (ArgHasMaybeUndefAttr)
5226	Val = Builder.CreateFreeze(V: AI.getPointer());
5227	IRCallArgs[FirstIRArg] = Val;
5228
5229	// Emit lifetime markers for the temporary alloca.
5230	llvm::TypeSize ByvalTempElementSize =
5231	CGM.getDataLayout().getTypeAllocSize(Ty: AI.getElementType());
5232	llvm::Value *LifetimeSize =
5233	EmitLifetimeStart(Size: ByvalTempElementSize, Addr: AI.getPointer());
5234
5235	// Add cleanup code to emit the end lifetime marker after the call.
5236	if (LifetimeSize) // In case we disabled lifetime markers.
5237	CallLifetimeEndAfterCall.emplace_back(Args&: AI, Args&: LifetimeSize);
5238
5239	// Generate the copy.
5240	I->copyInto(CGF&: *this, Addr: AI);
5241	} else {
5242	// Skip the extra memcpy call.
5243	auto *T = llvm::PointerType::get(
5244	C&: CGM.getLLVMContext(), AddressSpace: CGM.getDataLayout().getAllocaAddrSpace());
5245
5246	llvm::Value *Val = getTargetHooks().performAddrSpaceCast(
5247	*this, V, LangAS::Default, CGM.getASTAllocaAddressSpace(), T,
5248	true);
5249	if (ArgHasMaybeUndefAttr)
5250	Val = Builder.CreateFreeze(V: Val);
5251	IRCallArgs[FirstIRArg] = Val;
5252	}
5253	}
5254	break;
5255	}
5256
5257	case ABIArgInfo::Ignore:
5258	assert(NumIRArgs == 0);
5259	break;
5260
5261	case ABIArgInfo::Extend:
5262	case ABIArgInfo::Direct: {
5263	if (!isa<llvm::StructType>(Val: ArgInfo.getCoerceToType()) &&
5264	ArgInfo.getCoerceToType() == ConvertType(info_it->type) &&
5265	ArgInfo.getDirectOffset() == 0) {
5266	assert(NumIRArgs == 1);
5267	llvm::Value *V;
5268	if (!I->isAggregate())
5269	V = I->getKnownRValue().getScalarVal();
5270	else
5271	V = Builder.CreateLoad(
5272	Addr: I->hasLValue() ? I->getKnownLValue().getAddress(CGF&: *this)
5273	: I->getKnownRValue().getAggregateAddress());
5274
5275	// Implement swifterror by copying into a new swifterror argument.
5276	// We'll write back in the normal path out of the call.
5277	if (CallInfo.getExtParameterInfo(argIndex: ArgNo).getABI()
5278	== ParameterABI::SwiftErrorResult) {
5279	assert(!swiftErrorTemp.isValid() && "multiple swifterror args");
5280
5281	QualType pointeeTy = I->Ty->getPointeeType();
5282	swiftErrorArg = Address(V, ConvertTypeForMem(T: pointeeTy),
5283	getContext().getTypeAlignInChars(T: pointeeTy));
5284
5285	swiftErrorTemp =
5286	CreateMemTemp(pointeeTy, getPointerAlign(), "swifterror.temp");
5287	V = swiftErrorTemp.getPointer();
5288	cast<llvm::AllocaInst>(Val: V)->setSwiftError(true);
5289
5290	llvm::Value *errorValue = Builder.CreateLoad(Addr: swiftErrorArg);
5291	Builder.CreateStore(Val: errorValue, Addr: swiftErrorTemp);
5292	}
5293
5294	// We might have to widen integers, but we should never truncate.
5295	if (ArgInfo.getCoerceToType() != V->getType() &&
5296	V->getType()->isIntegerTy())
5297	V = Builder.CreateZExt(V, DestTy: ArgInfo.getCoerceToType());
5298
5299	// If the argument doesn't match, perform a bitcast to coerce it. This
5300	// can happen due to trivial type mismatches.
5301	if (FirstIRArg < IRFuncTy->getNumParams() &&
5302	V->getType() != IRFuncTy->getParamType(i: FirstIRArg))
5303	V = Builder.CreateBitCast(V, DestTy: IRFuncTy->getParamType(i: FirstIRArg));
5304
5305	if (ArgHasMaybeUndefAttr)
5306	V = Builder.CreateFreeze(V);
5307	IRCallArgs[FirstIRArg] = V;
5308	break;
5309	}
5310
5311	llvm::StructType *STy =
5312	dyn_cast<llvm::StructType>(Val: ArgInfo.getCoerceToType());
5313	llvm::Type *SrcTy = ConvertTypeForMem(T: I->Ty);
5314	llvm::TypeSize SrcTypeSize;
5315	llvm::TypeSize DstTypeSize;
5316	if (STy && ArgInfo.isDirect() && !ArgInfo.getCanBeFlattened()) {
5317	SrcTypeSize = CGM.getDataLayout().getTypeAllocSize(Ty: SrcTy);
5318	DstTypeSize = CGM.getDataLayout().getTypeAllocSize(Ty: STy);
5319	if (STy->containsHomogeneousScalableVectorTypes()) {
5320	assert(SrcTypeSize == DstTypeSize &&
5321	"Only allow non-fractional movement of structure with "
5322	"homogeneous scalable vector type");
5323
5324	IRCallArgs[FirstIRArg] = I->getKnownRValue().getScalarVal();
5325	break;
5326	}
5327	}
5328
5329	// FIXME: Avoid the conversion through memory if possible.
5330	Address Src = Address::invalid();
5331	if (!I->isAggregate()) {
5332	Src = CreateMemTemp(I->Ty, "coerce");
5333	I->copyInto(CGF&: *this, Addr: Src);
5334	} else {
5335	Src = I->hasLValue() ? I->getKnownLValue().getAddress(CGF&: *this)
5336	: I->getKnownRValue().getAggregateAddress();
5337	}
5338
5339	// If the value is offset in memory, apply the offset now.
5340	Src = emitAddressAtOffset(CGF&: *this, addr: Src, info: ArgInfo);
5341
5342	// Fast-isel and the optimizer generally like scalar values better than
5343	// FCAs, so we flatten them if this is safe to do for this argument.
5344	if (STy && ArgInfo.isDirect() && ArgInfo.getCanBeFlattened()) {
5345	llvm::Type *SrcTy = Src.getElementType();
5346	llvm::TypeSize SrcTypeSize =
5347	CGM.getDataLayout().getTypeAllocSize(Ty: SrcTy);
5348	llvm::TypeSize DstTypeSize = CGM.getDataLayout().getTypeAllocSize(Ty: STy);
5349	if (SrcTypeSize.isScalable()) {
5350	assert(STy->containsHomogeneousScalableVectorTypes() &&
5351	"ABI only supports structure with homogeneous scalable vector "
5352	"type");
5353	assert(SrcTypeSize == DstTypeSize &&
5354	"Only allow non-fractional movement of structure with "
5355	"homogeneous scalable vector type");
5356	assert(NumIRArgs == STy->getNumElements());
5357
5358	llvm::Value *StoredStructValue =
5359	Builder.CreateLoad(Addr: Src, Name: Src.getName() + ".tuple");
5360	for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
5361	llvm::Value *Extract = Builder.CreateExtractValue(
5362	Agg: StoredStructValue, Idxs: i, Name: Src.getName() + ".extract" + Twine(i));
5363	IRCallArgs[FirstIRArg + i] = Extract;
5364	}
5365	} else {
5366	uint64_t SrcSize = SrcTypeSize.getFixedValue();
5367	uint64_t DstSize = DstTypeSize.getFixedValue();
5368
5369	// If the source type is smaller than the destination type of the
5370	// coerce-to logic, copy the source value into a temp alloca the size
5371	// of the destination type to allow loading all of it. The bits past
5372	// the source value are left undef.
5373	if (SrcSize < DstSize) {
5374	Address TempAlloca = CreateTempAlloca(Ty: STy, align: Src.getAlignment(),
5375	Name: Src.getName() + ".coerce");
5376	Builder.CreateMemCpy(Dest: TempAlloca, Src, Size: SrcSize);
5377	Src = TempAlloca;
5378	} else {
5379	Src = Src.withElementType(ElemTy: STy);
5380	}
5381
5382	assert(NumIRArgs == STy->getNumElements());
5383	for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
5384	Address EltPtr = Builder.CreateStructGEP(Addr: Src, Index: i);
5385	llvm::Value *LI = Builder.CreateLoad(Addr: EltPtr);
5386	if (ArgHasMaybeUndefAttr)
5387	LI = Builder.CreateFreeze(V: LI);
5388	IRCallArgs[FirstIRArg + i] = LI;
5389	}
5390	}
5391	} else {
5392	// In the simple case, just pass the coerced loaded value.
5393	assert(NumIRArgs == 1);
5394	llvm::Value *Load =
5395	CreateCoercedLoad(Src, Ty: ArgInfo.getCoerceToType(), CGF&: *this);
5396
5397	if (CallInfo.isCmseNSCall()) {
5398	// For certain parameter types, clear padding bits, as they may reveal
5399	// sensitive information.
5400	// Small struct/union types are passed as integer arrays.
5401	auto *ATy = dyn_cast<llvm::ArrayType>(Val: Load->getType());
5402	if (ATy != nullptr && isa<RecordType>(I->Ty.getCanonicalType()))
5403	Load = EmitCMSEClearRecord(Load, ATy, I->Ty);
5404	}
5405
5406	if (ArgHasMaybeUndefAttr)
5407	Load = Builder.CreateFreeze(V: Load);
5408	IRCallArgs[FirstIRArg] = Load;
5409	}
5410
5411	break;
5412	}
5413
5414	case ABIArgInfo::CoerceAndExpand: {
5415	auto coercionType = ArgInfo.getCoerceAndExpandType();
5416	auto layout = CGM.getDataLayout().getStructLayout(Ty: coercionType);
5417
5418	llvm::Value *tempSize = nullptr;
5419	Address addr = Address::invalid();
5420	Address AllocaAddr = Address::invalid();
5421	if (I->isAggregate()) {
5422	addr = I->hasLValue() ? I->getKnownLValue().getAddress(CGF&: *this)
5423	: I->getKnownRValue().getAggregateAddress();
5424
5425	} else {
5426	RValue RV = I->getKnownRValue();
5427	assert(RV.isScalar()); // complex should always just be direct
5428
5429	llvm::Type *scalarType = RV.getScalarVal()->getType();
5430	auto scalarSize = CGM.getDataLayout().getTypeAllocSize(Ty: scalarType);
5431	auto scalarAlign = CGM.getDataLayout().getPrefTypeAlign(Ty: scalarType);
5432
5433	// Materialize to a temporary.
5434	addr = CreateTempAlloca(
5435	Ty: RV.getScalarVal()->getType(),
5436	align: CharUnits::fromQuantity(Quantity: std::max(a: layout->getAlignment(), b: scalarAlign)),
5437	Name: "tmp",
5438	/*ArraySize=*/nullptr, Alloca: &AllocaAddr);
5439	tempSize = EmitLifetimeStart(Size: scalarSize, Addr: AllocaAddr.getPointer());
5440
5441	Builder.CreateStore(Val: RV.getScalarVal(), Addr: addr);
5442	}
5443
5444	addr = addr.withElementType(ElemTy: coercionType);
5445
5446	unsigned IRArgPos = FirstIRArg;
5447	for (unsigned i = 0, e = coercionType->getNumElements(); i != e; ++i) {
5448	llvm::Type *eltType = coercionType->getElementType(N: i);
5449	if (ABIArgInfo::isPaddingForCoerceAndExpand(eltType)) continue;
5450	Address eltAddr = Builder.CreateStructGEP(Addr: addr, Index: i);
5451	llvm::Value *elt = Builder.CreateLoad(Addr: eltAddr);
5452	if (ArgHasMaybeUndefAttr)
5453	elt = Builder.CreateFreeze(V: elt);
5454	IRCallArgs[IRArgPos++] = elt;
5455	}
5456	assert(IRArgPos == FirstIRArg + NumIRArgs);
5457
5458	if (tempSize) {
5459	EmitLifetimeEnd(Size: tempSize, Addr: AllocaAddr.getPointer());
5460	}
5461
5462	break;
5463	}
5464
5465	case ABIArgInfo::Expand: {
5466	unsigned IRArgPos = FirstIRArg;
5467	ExpandTypeToArgs(Ty: I->Ty, Arg: *I, IRFuncTy, IRCallArgs, IRCallArgPos&: IRArgPos);
5468	assert(IRArgPos == FirstIRArg + NumIRArgs);
5469	break;
5470	}
5471	}
5472	}
5473
5474	const CGCallee &ConcreteCallee = Callee.prepareConcreteCallee(CGF&: *this);
5475	llvm::Value *CalleePtr = ConcreteCallee.getFunctionPointer();
5476
5477	// If we're using inalloca, set up that argument.
5478	if (ArgMemory.isValid()) {
5479	llvm::Value *Arg = ArgMemory.getPointer();
5480	assert(IRFunctionArgs.hasInallocaArg());
5481	IRCallArgs[IRFunctionArgs.getInallocaArgNo()] = Arg;
5482	}
5483
5484	// 2. Prepare the function pointer.
5485
5486	// If the callee is a bitcast of a non-variadic function to have a
5487	// variadic function pointer type, check to see if we can remove the
5488	// bitcast. This comes up with unprototyped functions.
5489	//
5490	// This makes the IR nicer, but more importantly it ensures that we
5491	// can inline the function at -O0 if it is marked always_inline.
5492	auto simplifyVariadicCallee = [](llvm::FunctionType *CalleeFT,
5493	llvm::Value *Ptr) -> llvm::Function * {
5494	if (!CalleeFT->isVarArg())
5495	return nullptr;
5496
5497	// Get underlying value if it's a bitcast
5498	if (llvm::ConstantExpr *CE = dyn_cast<llvm::ConstantExpr>(Val: Ptr)) {
5499	if (CE->getOpcode() == llvm::Instruction::BitCast)
5500	Ptr = CE->getOperand(i_nocapture: 0);
5501	}
5502
5503	llvm::Function *OrigFn = dyn_cast<llvm::Function>(Val: Ptr);
5504	if (!OrigFn)
5505	return nullptr;
5506
5507	llvm::FunctionType *OrigFT = OrigFn->getFunctionType();
5508
5509	// If the original type is variadic, or if any of the component types
5510	// disagree, we cannot remove the cast.
5511	if (OrigFT->isVarArg() ||
5512	OrigFT->getNumParams() != CalleeFT->getNumParams() ||
5513	OrigFT->getReturnType() != CalleeFT->getReturnType())
5514	return nullptr;
5515
5516	for (unsigned i = 0, e = OrigFT->getNumParams(); i != e; ++i)
5517	if (OrigFT->getParamType(i) != CalleeFT->getParamType(i))
5518	return nullptr;
5519
5520	return OrigFn;
5521	};
5522
5523	if (llvm::Function *OrigFn = simplifyVariadicCallee(IRFuncTy, CalleePtr)) {
5524	CalleePtr = OrigFn;
5525	IRFuncTy = OrigFn->getFunctionType();
5526	}
5527
5528	// 3. Perform the actual call.
5529
5530	// Deactivate any cleanups that we're supposed to do immediately before
5531	// the call.
5532	if (!CallArgs.getCleanupsToDeactivate().empty())
5533	deactivateArgCleanupsBeforeCall(CGF&: *this, CallArgs);
5534
5535	// Assert that the arguments we computed match up. The IR verifier
5536	// will catch this, but this is a common enough source of problems
5537	// during IRGen changes that it's way better for debugging to catch
5538	// it ourselves here.
5539	#ifndef NDEBUG
5540	assert(IRCallArgs.size() == IRFuncTy->getNumParams() || IRFuncTy->isVarArg());
5541	for (unsigned i = 0; i < IRCallArgs.size(); ++i) {
5542	// Inalloca argument can have different type.
5543	if (IRFunctionArgs.hasInallocaArg() &&
5544	i == IRFunctionArgs.getInallocaArgNo())
5545	continue;
5546	if (i < IRFuncTy->getNumParams())
5547	assert(IRCallArgs[i]->getType() == IRFuncTy->getParamType(i));
5548	}
5549	#endif
5550
5551	// Update the largest vector width if any arguments have vector types.
5552	for (unsigned i = 0; i < IRCallArgs.size(); ++i)
5553	LargestVectorWidth = std::max(a: LargestVectorWidth,
5554	b: getMaxVectorWidth(Ty: IRCallArgs[i]->getType()));
5555
5556	// Compute the calling convention and attributes.
5557	unsigned CallingConv;
5558	llvm::AttributeList Attrs;
5559	CGM.ConstructAttributeList(Name: CalleePtr->getName(), FI: CallInfo,
5560	CalleeInfo: Callee.getAbstractInfo(), AttrList&: Attrs, CallingConv,
5561	/*AttrOnCallSite=*/true,
5562	/*IsThunk=*/false);
5563
5564	if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(Val: CurFuncDecl)) {
5565	if (FD->hasAttr<StrictFPAttr>())
5566	// All calls within a strictfp function are marked strictfp
5567	Attrs = Attrs.addFnAttribute(getLLVMContext(), llvm::Attribute::StrictFP);
5568
5569	// If -ffast-math is enabled and the function is guarded by an
5570	// '__attribute__((optnone)) adjust the memory attribute so the BE emits the
5571	// library call instead of the intrinsic.
5572	if (FD->hasAttr<OptimizeNoneAttr>() && getLangOpts().FastMath)
5573	CGM.AdjustMemoryAttribute(Name: CalleePtr->getName(), CalleeInfo: Callee.getAbstractInfo(),
5574	Attrs);
5575	}
5576	// Add call-site nomerge attribute if exists.
5577	if (InNoMergeAttributedStmt)
5578	Attrs = Attrs.addFnAttribute(getLLVMContext(), llvm::Attribute::NoMerge);
5579
5580	// Add call-site noinline attribute if exists.
5581	if (InNoInlineAttributedStmt)
5582	Attrs = Attrs.addFnAttribute(getLLVMContext(), llvm::Attribute::NoInline);
5583
5584	// Add call-site always_inline attribute if exists.
5585	if (InAlwaysInlineAttributedStmt)
5586	Attrs =
5587	Attrs.addFnAttribute(getLLVMContext(), llvm::Attribute::AlwaysInline);
5588
5589	// Apply some call-site-specific attributes.
5590	// TODO: work this into building the attribute set.
5591
5592	// Apply always_inline to all calls within flatten functions.
5593	// FIXME: should this really take priority over __try, below?
5594	if (CurCodeDecl && CurCodeDecl->hasAttr<FlattenAttr>() &&
5595	!InNoInlineAttributedStmt &&
5596	!(TargetDecl && TargetDecl->hasAttr<NoInlineAttr>())) {
5597	Attrs =
5598	Attrs.addFnAttribute(getLLVMContext(), llvm::Attribute::AlwaysInline);
5599	}
5600
5601	// Disable inlining inside SEH __try blocks.
5602	if (isSEHTryScope()) {
5603	Attrs = Attrs.addFnAttribute(getLLVMContext(), llvm::Attribute::NoInline);
5604	}
5605
5606	// Decide whether to use a call or an invoke.
5607	bool CannotThrow;
5608	if (currentFunctionUsesSEHTry()) {
5609	// SEH cares about asynchronous exceptions, so everything can "throw."
5610	CannotThrow = false;
5611	} else if (isCleanupPadScope() &&
5612	EHPersonality::get(CGF&: *this).isMSVCXXPersonality()) {
5613	// The MSVC++ personality will implicitly terminate the program if an
5614	// exception is thrown during a cleanup outside of a try/catch.
5615	// We don't need to model anything in IR to get this behavior.
5616	CannotThrow = true;
5617	} else {
5618	// Otherwise, nounwind call sites will never throw.
5619	CannotThrow = Attrs.hasFnAttr(llvm::Attribute::NoUnwind);
5620
5621	if (auto *FPtr = dyn_cast<llvm::Function>(Val: CalleePtr))
5622	if (FPtr->hasFnAttribute(llvm::Attribute::NoUnwind))
5623	CannotThrow = true;
5624	}
5625
5626	// If we made a temporary, be sure to clean up after ourselves. Note that we
5627	// can't depend on being inside of an ExprWithCleanups, so we need to manually
5628	// pop this cleanup later on. Being eager about this is OK, since this
5629	// temporary is 'invisible' outside of the callee.
5630	if (UnusedReturnSizePtr)
5631	pushFullExprCleanup<CallLifetimeEnd>(kind: NormalEHLifetimeMarker, A: SRetAlloca,
5632	A: UnusedReturnSizePtr);
5633
5634	llvm::BasicBlock *InvokeDest = CannotThrow ? nullptr : getInvokeDest();
5635
5636	SmallVector<llvm::OperandBundleDef, 1> BundleList =
5637	getBundlesForFunclet(Callee: CalleePtr);
5638
5639	if (SanOpts.has(K: SanitizerKind::KCFI) &&
5640	!isa_and_nonnull<FunctionDecl>(Val: TargetDecl))
5641	EmitKCFIOperandBundle(Callee: ConcreteCallee, Bundles&: BundleList);
5642
5643	if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(CurFuncDecl))
5644	if (FD->hasAttr<StrictFPAttr>())
5645	// All calls within a strictfp function are marked strictfp
5646	Attrs = Attrs.addFnAttribute(getLLVMContext(), llvm::Attribute::StrictFP);
5647
5648	AssumeAlignedAttrEmitter AssumeAlignedAttrEmitter(*this, TargetDecl);
5649	Attrs = AssumeAlignedAttrEmitter.TryEmitAsCallSiteAttribute(Attrs);
5650
5651	AllocAlignAttrEmitter AllocAlignAttrEmitter(*this, TargetDecl, CallArgs);
5652	Attrs = AllocAlignAttrEmitter.TryEmitAsCallSiteAttribute(Attrs);
5653
5654	// Emit the actual call/invoke instruction.
5655	llvm::CallBase *CI;
5656	if (!InvokeDest) {
5657	CI = Builder.CreateCall(FTy: IRFuncTy, Callee: CalleePtr, Args: IRCallArgs, OpBundles: BundleList);
5658	} else {
5659	llvm::BasicBlock *Cont = createBasicBlock(name: "invoke.cont");
5660	CI = Builder.CreateInvoke(Ty: IRFuncTy, Callee: CalleePtr, NormalDest: Cont, UnwindDest: InvokeDest, Args: IRCallArgs,
5661	OpBundles: BundleList);
5662	EmitBlock(BB: Cont);
5663	}
5664	if (CI->getCalledFunction() && CI->getCalledFunction()->hasName() &&
5665	CI->getCalledFunction()->getName().starts_with(Prefix: "_Z4sqrt")) {
5666	SetSqrtFPAccuracy(CI);
5667	}
5668	if (callOrInvoke)
5669	*callOrInvoke = CI;
5670
5671	// If this is within a function that has the guard(nocf) attribute and is an
5672	// indirect call, add the "guard_nocf" attribute to this call to indicate that
5673	// Control Flow Guard checks should not be added, even if the call is inlined.
5674	if (const auto *FD = dyn_cast_or_null<FunctionDecl>(Val: CurFuncDecl)) {
5675	if (const auto *A = FD->getAttr<CFGuardAttr>()) {
5676	if (A->getGuard() == CFGuardAttr::GuardArg::nocf && !CI->getCalledFunction())
5677	Attrs = Attrs.addFnAttribute(C&: getLLVMContext(), Kind: "guard_nocf");
5678	}
5679	}
5680
5681	// Apply the attributes and calling convention.
5682	CI->setAttributes(Attrs);
5683	CI->setCallingConv(static_cast<llvm::CallingConv::ID>(CallingConv));
5684
5685	// Apply various metadata.
5686
5687	if (!CI->getType()->isVoidTy())
5688	CI->setName("call");
5689
5690	// Update largest vector width from the return type.
5691	LargestVectorWidth =
5692	std::max(a: LargestVectorWidth, b: getMaxVectorWidth(Ty: CI->getType()));
5693
5694	// Insert instrumentation or attach profile metadata at indirect call sites.
5695	// For more details, see the comment before the definition of
5696	// IPVK_IndirectCallTarget in InstrProfData.inc.
5697	if (!CI->getCalledFunction())
5698	PGO.valueProfile(Builder, ValueKind: llvm::IPVK_IndirectCallTarget,
5699	ValueSite: CI, ValuePtr: CalleePtr);
5700
5701	// In ObjC ARC mode with no ObjC ARC exception safety, tell the ARC
5702	// optimizer it can aggressively ignore unwind edges.
5703	if (CGM.getLangOpts().ObjCAutoRefCount)
5704	AddObjCARCExceptionMetadata(Inst: CI);
5705
5706	// Set tail call kind if necessary.
5707	if (llvm::CallInst *Call = dyn_cast<llvm::CallInst>(Val: CI)) {
5708	if (TargetDecl && TargetDecl->hasAttr<NotTailCalledAttr>())
5709	Call->setTailCallKind(llvm::CallInst::TCK_NoTail);
5710	else if (IsMustTail)
5711	Call->setTailCallKind(llvm::CallInst::TCK_MustTail);
5712	}
5713
5714	// Add metadata for calls to MSAllocator functions
5715	if (getDebugInfo() && TargetDecl &&
5716	TargetDecl->hasAttr<MSAllocatorAttr>())
5717	getDebugInfo()->addHeapAllocSiteMetadata(CallSite: CI, AllocatedTy: RetTy->getPointeeType(), Loc);
5718
5719	// Add metadata if calling an __attribute__((error(""))) or warning fn.
5720	if (TargetDecl && TargetDecl->hasAttr<ErrorAttr>()) {
5721	llvm::ConstantInt *Line =
5722	llvm::ConstantInt::get(Ty: Int32Ty, V: Loc.getRawEncoding());
5723	llvm::ConstantAsMetadata *MD = llvm::ConstantAsMetadata::get(C: Line);
5724	llvm::MDTuple *MDT = llvm::MDNode::get(Context&: getLLVMContext(), MDs: {MD});
5725	CI->setMetadata(Kind: "srcloc", Node: MDT);
5726	}
5727
5728	// 4. Finish the call.
5729
5730	// If the call doesn't return, finish the basic block and clear the
5731	// insertion point; this allows the rest of IRGen to discard
5732	// unreachable code.
5733	if (CI->doesNotReturn()) {
5734	if (UnusedReturnSizePtr)
5735	PopCleanupBlock();
5736
5737	// Strip away the noreturn attribute to better diagnose unreachable UB.
5738	if (SanOpts.has(K: SanitizerKind::Unreachable)) {
5739	// Also remove from function since CallBase::hasFnAttr additionally checks
5740	// attributes of the called function.
5741	if (auto *F = CI->getCalledFunction())
5742	F->removeFnAttr(llvm::Attribute::NoReturn);
5743	CI->removeFnAttr(llvm::Attribute::NoReturn);
5744
5745	// Avoid incompatibility with ASan which relies on the `noreturn`
5746	// attribute to insert handler calls.
5747	if (SanOpts.hasOneOf(K: SanitizerKind::Address |
5748	SanitizerKind::KernelAddress)) {
5749	SanitizerScope SanScope(this);
5750	llvm::IRBuilder<>::InsertPointGuard IPGuard(Builder);
5751	Builder.SetInsertPoint(CI);
5752	auto *FnType = llvm::FunctionType::get(Result: CGM.VoidTy, /*isVarArg=*/false);
5753	llvm::FunctionCallee Fn =
5754	CGM.CreateRuntimeFunction(Ty: FnType, Name: "__asan_handle_no_return");
5755	EmitNounwindRuntimeCall(callee: Fn);
5756	}
5757	}
5758
5759	EmitUnreachable(Loc);
5760	Builder.ClearInsertionPoint();
5761
5762	// FIXME: For now, emit a dummy basic block because expr emitters in
5763	// generally are not ready to handle emitting expressions at unreachable
5764	// points.
5765	EnsureInsertPoint();
5766
5767	// Return a reasonable RValue.
5768	return GetUndefRValue(Ty: RetTy);
5769	}
5770
5771	// If this is a musttail call, return immediately. We do not branch to the
5772	// epilogue in this case.
5773	if (IsMustTail) {
5774	for (auto it = EHStack.find(sp: CurrentCleanupScopeDepth); it != EHStack.end();
5775	++it) {
5776	EHCleanupScope *Cleanup = dyn_cast<EHCleanupScope>(Val: &*it);
5777	if (!(Cleanup && Cleanup->getCleanup()->isRedundantBeforeReturn()))
5778	CGM.ErrorUnsupported(MustTailCall, "tail call skipping over cleanups");
5779	}
5780	if (CI->getType()->isVoidTy())
5781	Builder.CreateRetVoid();
5782	else
5783	Builder.CreateRet(V: CI);
5784	Builder.ClearInsertionPoint();
5785	EnsureInsertPoint();
5786	return GetUndefRValue(Ty: RetTy);
5787	}
5788
5789	// Perform the swifterror writeback.
5790	if (swiftErrorTemp.isValid()) {
5791	llvm::Value *errorResult = Builder.CreateLoad(Addr: swiftErrorTemp);
5792	Builder.CreateStore(Val: errorResult, Addr: swiftErrorArg);
5793	}
5794
5795	// Emit any call-associated writebacks immediately. Arguably this
5796	// should happen after any return-value munging.
5797	if (CallArgs.hasWritebacks())
5798	emitWritebacks(CGF&: *this, args: CallArgs);
5799
5800	// The stack cleanup for inalloca arguments has to run out of the normal
5801	// lexical order, so deactivate it and run it manually here.
5802	CallArgs.freeArgumentMemory(CGF&: *this);
5803
5804	// Extract the return value.
5805	RValue Ret = [&] {
5806	switch (RetAI.getKind()) {
5807	case ABIArgInfo::CoerceAndExpand: {
5808	auto coercionType = RetAI.getCoerceAndExpandType();
5809
5810	Address addr = SRetPtr.withElementType(ElemTy: coercionType);
5811
5812	assert(CI->getType() == RetAI.getUnpaddedCoerceAndExpandType());
5813	bool requiresExtract = isa<llvm::StructType>(Val: CI->getType());
5814
5815	unsigned unpaddedIndex = 0;
5816	for (unsigned i = 0, e = coercionType->getNumElements(); i != e; ++i) {
5817	llvm::Type *eltType = coercionType->getElementType(N: i);
5818	if (ABIArgInfo::isPaddingForCoerceAndExpand(eltType)) continue;
5819	Address eltAddr = Builder.CreateStructGEP(Addr: addr, Index: i);
5820	llvm::Value *elt = CI;
5821	if (requiresExtract)
5822	elt = Builder.CreateExtractValue(Agg: elt, Idxs: unpaddedIndex++);
5823	else
5824	assert(unpaddedIndex == 0);
5825	Builder.CreateStore(Val: elt, Addr: eltAddr);
5826	}
5827	[[fallthrough]];
5828	}
5829
5830	case ABIArgInfo::InAlloca:
5831	case ABIArgInfo::Indirect: {
5832	RValue ret = convertTempToRValue(addr: SRetPtr, type: RetTy, Loc: SourceLocation());
5833	if (UnusedReturnSizePtr)
5834	PopCleanupBlock();
5835	return ret;
5836	}
5837
5838	case ABIArgInfo::Ignore:
5839	// If we are ignoring an argument that had a result, make sure to
5840	// construct the appropriate return value for our caller.
5841	return GetUndefRValue(Ty: RetTy);
5842
5843	case ABIArgInfo::Extend:
5844	case ABIArgInfo::Direct: {
5845	llvm::Type *RetIRTy = ConvertType(T: RetTy);
5846	if (RetAI.getCoerceToType() == RetIRTy && RetAI.getDirectOffset() == 0) {
5847	switch (getEvaluationKind(T: RetTy)) {
5848	case TEK_Complex: {
5849	llvm::Value *Real = Builder.CreateExtractValue(Agg: CI, Idxs: 0);
5850	llvm::Value *Imag = Builder.CreateExtractValue(Agg: CI, Idxs: 1);
5851	return RValue::getComplex(C: std::make_pair(x&: Real, y&: Imag));
5852	}
5853	case TEK_Aggregate: {
5854	Address DestPtr = ReturnValue.getValue();
5855	bool DestIsVolatile = ReturnValue.isVolatile();
5856
5857	if (!DestPtr.isValid()) {
5858	DestPtr = CreateMemTemp(T: RetTy, Name: "agg.tmp");
5859	DestIsVolatile = false;
5860	}
5861	EmitAggregateStore(Val: CI, Dest: DestPtr, DestIsVolatile);
5862	return RValue::getAggregate(addr: DestPtr);
5863	}
5864	case TEK_Scalar: {
5865	// If the argument doesn't match, perform a bitcast to coerce it. This
5866	// can happen due to trivial type mismatches.
5867	llvm::Value *V = CI;
5868	if (V->getType() != RetIRTy)
5869	V = Builder.CreateBitCast(V, DestTy: RetIRTy);
5870	return RValue::get(V);
5871	}
5872	}
5873	llvm_unreachable("bad evaluation kind");
5874	}
5875
5876	// If coercing a fixed vector from a scalable vector for ABI
5877	// compatibility, and the types match, use the llvm.vector.extract
5878	// intrinsic to perform the conversion.
5879	if (auto *FixedDstTy = dyn_cast<llvm::FixedVectorType>(Val: RetIRTy)) {
5880	llvm::Value *V = CI;
5881	if (auto *ScalableSrcTy =
5882	dyn_cast<llvm::ScalableVectorType>(Val: V->getType())) {
5883	if (FixedDstTy->getElementType() == ScalableSrcTy->getElementType()) {
5884	llvm::Value *Zero = llvm::Constant::getNullValue(Ty: CGM.Int64Ty);
5885	V = Builder.CreateExtractVector(DstType: FixedDstTy, SrcVec: V, Idx: Zero, Name: "cast.fixed");
5886	return RValue::get(V);
5887	}
5888	}
5889	}
5890
5891	Address DestPtr = ReturnValue.getValue();
5892	bool DestIsVolatile = ReturnValue.isVolatile();
5893
5894	if (!DestPtr.isValid()) {
5895	DestPtr = CreateMemTemp(T: RetTy, Name: "coerce");
5896	DestIsVolatile = false;
5897	}
5898
5899	// An empty record can overlap other data (if declared with
5900	// no_unique_address); omit the store for such types - as there is no
5901	// actual data to store.
5902	if (!isEmptyRecord(Context&: getContext(), T: RetTy, AllowArrays: true)) {
5903	// If the value is offset in memory, apply the offset now.
5904	Address StorePtr = emitAddressAtOffset(CGF&: *this, addr: DestPtr, info: RetAI);
5905	CreateCoercedStore(Src: CI, Dst: StorePtr, DstIsVolatile: DestIsVolatile, CGF&: *this);
5906	}
5907
5908	return convertTempToRValue(addr: DestPtr, type: RetTy, Loc: SourceLocation());
5909	}
5910
5911	case ABIArgInfo::Expand:
5912	case ABIArgInfo::IndirectAliased:
5913	llvm_unreachable("Invalid ABI kind for return argument");
5914	}
5915
5916	llvm_unreachable("Unhandled ABIArgInfo::Kind");
5917	} ();
5918
5919	// Emit the assume_aligned check on the return value.
5920	if (Ret.isScalar() && TargetDecl) {
5921	AssumeAlignedAttrEmitter.EmitAsAnAssumption(Loc, RetTy, Ret);
5922	AllocAlignAttrEmitter.EmitAsAnAssumption(Loc, RetTy, Ret);
5923	}
5924
5925	// Explicitly call CallLifetimeEnd::Emit just to re-use the code even though
5926	// we can't use the full cleanup mechanism.
5927	for (CallLifetimeEnd &LifetimeEnd : CallLifetimeEndAfterCall)
5928	LifetimeEnd.Emit(CGF&: *this, /*Flags=*/flags: {});
5929
5930	if (!ReturnValue.isExternallyDestructed() &&
5931	RetTy.isDestructedType() == QualType::DK_nontrivial_c_struct)
5932	pushDestroy(dtorKind: QualType::DK_nontrivial_c_struct, addr: Ret.getAggregateAddress(),
5933	type: RetTy);
5934
5935	return Ret;
5936	}
5937
5938	CGCallee CGCallee::prepareConcreteCallee(CodeGenFunction &CGF) const {
5939	if (isVirtual()) {
5940	const CallExpr *CE = getVirtualCallExpr();
5941	return CGF.CGM.getCXXABI().getVirtualFunctionPointer(
5942	CGF, GD: getVirtualMethodDecl(), This: getThisAddress(), Ty: getVirtualFunctionType(),
5943	Loc: CE ? CE->getBeginLoc() : SourceLocation());
5944	}
5945
5946	return *this;
5947	}
5948
5949	/* VarArg handling */
5950
5951	Address CodeGenFunction::EmitVAArg(VAArgExpr *VE, Address &VAListAddr) {
5952	VAListAddr = VE->isMicrosoftABI()
5953	? EmitMSVAListRef(E: VE->getSubExpr())
5954	: EmitVAListRef(E: VE->getSubExpr());
5955	QualType Ty = VE->getType();
5956	if (VE->isMicrosoftABI())
5957	return CGM.getTypes().getABIInfo().EmitMSVAArg(CGF&: *this, VAListAddr, Ty);
5958	return CGM.getTypes().getABIInfo().EmitVAArg(CGF&: *this, VAListAddr, Ty);
5959	}
5960