CGCall.cpp source code [clang/lib/CodeGen/CGCall.cpp]

1	//===--- CGCall.cpp - Encapsulate calling convention details --------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// These classes wrap the information about a call or function
10	// definition used to handle ABI compliancy.
11	//
12	//===----------------------------------------------------------------------===//
13
14	#include "CGCall.h"
15	#include "ABIInfo.h"
16	#include "ABIInfoImpl.h"
17	#include "CGBlocks.h"
18	#include "CGCXXABI.h"
19	#include "CGCleanup.h"
20	#include "CGDebugInfo.h"
21	#include "CGRecordLayout.h"
22	#include "CodeGenFunction.h"
23	#include "CodeGenModule.h"
24	#include "CodeGenPGO.h"
25	#include "TargetInfo.h"
26	#include "clang/AST/Attr.h"
27	#include "clang/AST/Decl.h"
28	#include "clang/AST/DeclCXX.h"
29	#include "clang/AST/DeclObjC.h"
30	#include "clang/Basic/CodeGenOptions.h"
31	#include "clang/Basic/TargetInfo.h"
32	#include "clang/CodeGen/CGFunctionInfo.h"
33	#include "clang/CodeGen/SwiftCallingConv.h"
34	#include "llvm/ADT/StringExtras.h"
35	#include "llvm/Analysis/ValueTracking.h"
36	#include "llvm/IR/Assumptions.h"
37	#include "llvm/IR/AttributeMask.h"
38	#include "llvm/IR/Attributes.h"
39	#include "llvm/IR/CallingConv.h"
40	#include "llvm/IR/DataLayout.h"
41	#include "llvm/IR/InlineAsm.h"
42	#include "llvm/IR/IntrinsicInst.h"
43	#include "llvm/IR/Intrinsics.h"
44	#include "llvm/IR/Type.h"
45	#include "llvm/Transforms/Utils/Local.h"
46	#include <optional>
47	using namespace clang;
48	using namespace CodeGen;
49
50	/*/
51
52	unsigned CodeGenTypes::ClangCallConvToLLVMCallConv(CallingConv CC) {
53	switch (CC) {
54	default:
55	return llvm::CallingConv::C;
56	case CC_X86StdCall:
57	return llvm::CallingConv::X86_StdCall;
58	case CC_X86FastCall:
59	return llvm::CallingConv::X86_FastCall;
60	case CC_X86RegCall:
61	return llvm::CallingConv::X86_RegCall;
62	case CC_X86ThisCall:
63	return llvm::CallingConv::X86_ThisCall;
64	case CC_Win64:
65	return llvm::CallingConv::Win64;
66	case CC_X86_64SysV:
67	return llvm::CallingConv::X86_64_SysV;
68	case CC_AAPCS:
69	return llvm::CallingConv::ARM_AAPCS;
70	case CC_AAPCS_VFP:
71	return llvm::CallingConv::ARM_AAPCS_VFP;
72	case CC_IntelOclBicc:
73	return llvm::CallingConv::Intel_OCL_BI;
74	// TODO: Add support for __pascal to LLVM.
75	case CC_X86Pascal:
76	return llvm::CallingConv::C;
77	// TODO: Add support for __vectorcall to LLVM.
78	case CC_X86VectorCall:
79	return llvm::CallingConv::X86_VectorCall;
80	case CC_AArch64VectorCall:
81	return llvm::CallingConv::AArch64_VectorCall;
82	case CC_AArch64SVEPCS:
83	return llvm::CallingConv::AArch64_SVE_VectorCall;
84	case CC_SpirFunction:
85	return llvm::CallingConv::SPIR_FUNC;
86	case CC_DeviceKernel: {
87	if (CGM.getLangOpts().OpenCL)
88	return CGM.getTargetCodeGenInfo().getOpenCLKernelCallingConv();
89	if (CGM.getTriple().isSPIROrSPIRV())
90	return llvm::CallingConv::SPIR_KERNEL;
91	if (CGM.getTriple().isAMDGPU())
92	return llvm::CallingConv::AMDGPU_KERNEL;
93	if (CGM.getTriple().isNVPTX())
94	return llvm::CallingConv::PTX_Kernel;
95	llvm_unreachable("Unknown kernel calling convention");
96	}
97	case CC_PreserveMost:
98	return llvm::CallingConv::PreserveMost;
99	case CC_PreserveAll:
100	return llvm::CallingConv::PreserveAll;
101	case CC_Swift:
102	return llvm::CallingConv::Swift;
103	case CC_SwiftAsync:
104	return llvm::CallingConv::SwiftTail;
105	case CC_M68kRTD:
106	return llvm::CallingConv::M68k_RTD;
107	case CC_PreserveNone:
108	return llvm::CallingConv::PreserveNone;
109	// clang-format off
110	case CC_RISCVVectorCall: return llvm::CallingConv::RISCV_VectorCall;
111	// clang-format on
112	#define CC_VLS_CASE(ABI_VLEN) \
113	case CC_RISCVVLSCall_##ABI_VLEN: \
114	return llvm::CallingConv::RISCV_VLSCall_##ABI_VLEN;
115	CC_VLS_CASE(`32`)
116	CC_VLS_CASE(`64`)
117	CC_VLS_CASE(`128`)
118	CC_VLS_CASE(`256`)
119	CC_VLS_CASE(`512`)
120	CC_VLS_CASE(`1024`)
121	CC_VLS_CASE(`2048`)
122	CC_VLS_CASE(`4096`)
123	CC_VLS_CASE(`8192`)
124	CC_VLS_CASE(`16384`)
125	CC_VLS_CASE(`32768`)
126	CC_VLS_CASE(`65536`)
127	#undef CC_VLS_CASE
128	}
129	}
130
131	/// Derives the 'this' type for codegen purposes, i.e. ignoring method CVR
132	/// qualification. Either or both of RD and MD may be null. A null RD indicates
133	/// that there is no meaningful 'this' type, and a null MD can occur when
134	/// calling a method pointer.
135	CanQualType CodeGenTypes::DeriveThisType(const CXXRecordDecl *RD,
136	const CXXMethodDecl *MD) {
137	QualType RecTy;
138	if (RD)
139	RecTy = Context.getTagDeclType(RD)->getCanonicalTypeInternal();
140	else
141	RecTy = Context.VoidTy;
142
143	if (MD)
144	RecTy = Context.getAddrSpaceQualType(
145	T: RecTy, AddressSpace: MD->getMethodQualifiers().getAddressSpace());
146	return Context.getPointerType(T: CanQualType::CreateUnsafe(Other: RecTy));
147	}
148
149	/// Returns the canonical formal type of the given C++ method.
150	static CanQual<FunctionProtoType> GetFormalType(const CXXMethodDecl *MD) {
151	return MD->getType()
152	->getCanonicalTypeUnqualified()
153	.getAs<FunctionProtoType>();
154	}
155
156	/// Returns the "extra-canonicalized" return type, which discards
157	/// qualifiers on the return type. Codegen doesn't care about them,
158	/// and it makes ABI code a little easier to be able to assume that
159	/// all parameter and return types are top-level unqualified.
160	static CanQualType GetReturnType(QualType RetTy) {
161	return RetTy->getCanonicalTypeUnqualified().getUnqualifiedType();
162	}
163
164	/// Arrange the argument and result information for a value of the given
165	/// unprototyped freestanding function type.
166	const CGFunctionInfo &
167	CodeGenTypes::arrangeFreeFunctionType(CanQual<FunctionNoProtoType> FTNP) {
168	// When translating an unprototyped function type, always use a
169	// variadic type.
170	return arrangeLLVMFunctionInfo(FTNP->getReturnType().getUnqualifiedType(),
171	FnInfoOpts::None, {}, FTNP->getExtInfo(), {},
172	RequiredArgs(`0`));
173	}
174
175	static void addExtParameterInfosForCall(
176	llvm::SmallVectorImpl<FunctionProtoType::ExtParameterInfo> &paramInfos,
177	const FunctionProtoType proto, unsigned* prefixArgs, unsigned totalArgs) {
178	assert(proto->hasExtParameterInfos());
179	assert(paramInfos.size() <= prefixArgs);
180	assert(proto->getNumParams() + prefixArgs <= totalArgs);
181
182	paramInfos.reserve(N: totalArgs);
183
184	// Add default infos for any prefix args that don't already have infos.
185	paramInfos.resize(N: prefixArgs);
186
187	// Add infos for the prototype.
188	for (const auto &ParamInfo : proto->getExtParameterInfos()) {
189	paramInfos.push_back(Elt: ParamInfo);
190	// pass_object_size params have no parameter info.
191	if (ParamInfo.hasPassObjectSize())
192	paramInfos.emplace_back();
193	}
194
195	assert(paramInfos.size() <= totalArgs &&
196	"Did we forget to insert pass_object_size args?");
197	// Add default infos for the variadic and/or suffix arguments.
198	paramInfos.resize(N: totalArgs);
199	}
200
201	/// Adds the formal parameters in FPT to the given prefix. If any parameter in
202	/// FPT has pass_object_size attrs, then we'll add parameters for those, too.
203	static void appendParameterTypes(
204	const CodeGenTypes &CGT, SmallVectorImpl<CanQualType> &prefix,
205	SmallVectorImpl<FunctionProtoType::ExtParameterInfo> &paramInfos,
206	CanQual<FunctionProtoType> FPT) {
207	// Fast path: don't touch param info if we don't need to.
208	if (!FPT->hasExtParameterInfos()) {
209	assert(paramInfos.empty() &&
210	"We have paramInfos, but the prototype doesn't?");
211	prefix.append(FPT->param_type_begin(), FPT->param_type_end());
212	return;
213	}
214
215	unsigned PrefixSize = prefix.size();
216	// In the vast majority of cases, we'll have precisely FPT->getNumParams()
217	// parameters; the only thing that can change this is the presence of
218	// pass_object_size. So, we preallocate for the common case.
219	prefix.reserve(prefix.size() + FPT->getNumParams());
220
221	auto ExtInfos = FPT->getExtParameterInfos();
222	assert(ExtInfos.size() == FPT->getNumParams());
223	for (unsigned I = `0`, E = FPT->getNumParams(); I != E; ++I) {
224	prefix.push_back(FPT->getParamType(I));
225	if (ExtInfos[I].hasPassObjectSize())
226	prefix.push_back(Elt: CGT.getContext().getSizeType());
227	}
228
229	addExtParameterInfosForCall(paramInfos, proto: FPT.getTypePtr(), prefixArgs: PrefixSize,
230	totalArgs: prefix.size());
231	}
232
233	using ExtParameterInfoList =
234	SmallVector<FunctionProtoType::ExtParameterInfo, `16`>;
235
236	/// Arrange the LLVM function layout for a value of the given function
237	/// type, on top of any implicit parameters already stored.
238	static const CGFunctionInfo &
239	arrangeLLVMFunctionInfo(CodeGenTypes &CGT, bool instanceMethod,
240	SmallVectorImpl<CanQualType> &prefix,
241	CanQual<FunctionProtoType> FTP) {
242	ExtParameterInfoList paramInfos;
243	RequiredArgs Required = RequiredArgs::forPrototypePlus(prototype: FTP, additional: prefix.size());
244	appendParameterTypes(CGT, prefix, paramInfos, FPT: FTP);
245	CanQualType resultType = FTP->getReturnType().getUnqualifiedType();
246
247	FnInfoOpts opts =
248	instanceMethod ? FnInfoOpts::IsInstanceMethod : FnInfoOpts::None;
249	return CGT.arrangeLLVMFunctionInfo(resultType, opts, prefix,
250	FTP->getExtInfo(), paramInfos, Required);
251	}
252
253	using CanQualTypeList = SmallVector<CanQualType, `16`>;
254
255	/// Arrange the argument and result information for a value of the
256	/// given freestanding function type.
257	const CGFunctionInfo &
258	CodeGenTypes::arrangeFreeFunctionType(CanQual<FunctionProtoType> FTP) {
259	CanQualTypeList argTypes;
260	return ::arrangeLLVMFunctionInfo(CGT&: *this, /instanceMethod=/false, prefix&: argTypes,
261	FTP);
262	}
263
264	static CallingConv getCallingConventionForDecl(const ObjCMethodDecl *D,
265	bool IsTargetDefaultMSABI) {
266	// Set the appropriate calling convention for the Function.
267	if (D->hasAttr<StdCallAttr>())
268	return CC_X86StdCall;
269
270	if (D->hasAttr<FastCallAttr>())
271	return CC_X86FastCall;
272
273	if (D->hasAttr<RegCallAttr>())
274	return CC_X86RegCall;
275
276	if (D->hasAttr<ThisCallAttr>())
277	return CC_X86ThisCall;
278
279	if (D->hasAttr<VectorCallAttr>())
280	return CC_X86VectorCall;
281
282	if (D->hasAttr<PascalAttr>())
283	return CC_X86Pascal;
284
285	if (PcsAttr *PCS = D->getAttr<PcsAttr>())
286	return (PCS->getPCS() == PcsAttr::AAPCS ? CC_AAPCS : CC_AAPCS_VFP);
287
288	if (D->hasAttr<AArch64VectorPcsAttr>())
289	return CC_AArch64VectorCall;
290
291	if (D->hasAttr<AArch64SVEPcsAttr>())
292	return CC_AArch64SVEPCS;
293
294	if (D->hasAttr<DeviceKernelAttr>())
295	return CC_DeviceKernel;
296
297	if (D->hasAttr<IntelOclBiccAttr>())
298	return CC_IntelOclBicc;
299
300	if (D->hasAttr<MSABIAttr>())
301	return IsTargetDefaultMSABI ? CC_C : CC_Win64;
302
303	if (D->hasAttr<SysVABIAttr>())
304	return IsTargetDefaultMSABI ? CC_X86_64SysV : CC_C;
305
306	if (D->hasAttr<PreserveMostAttr>())
307	return CC_PreserveMost;
308
309	if (D->hasAttr<PreserveAllAttr>())
310	return CC_PreserveAll;
311
312	if (D->hasAttr<M68kRTDAttr>())
313	return CC_M68kRTD;
314
315	if (D->hasAttr<PreserveNoneAttr>())
316	return CC_PreserveNone;
317
318	if (D->hasAttr<RISCVVectorCCAttr>())
319	return CC_RISCVVectorCall;
320
321	if (RISCVVLSCCAttr *PCS = D->getAttr<RISCVVLSCCAttr>()) {
322	switch (PCS->getVectorWidth()) {
323	default:
324	llvm_unreachable("Invalid RISC-V VLS ABI VLEN");
325	#define CC_VLS_CASE(ABI_VLEN) \
326	case ABI_VLEN: \
327	return CC_RISCVVLSCall_##ABI_VLEN;
328	CC_VLS_CASE(`32`)
329	CC_VLS_CASE(`64`)
330	CC_VLS_CASE(`128`)
331	CC_VLS_CASE(`256`)
332	CC_VLS_CASE(`512`)
333	CC_VLS_CASE(`1024`)
334	CC_VLS_CASE(`2048`)
335	CC_VLS_CASE(`4096`)
336	CC_VLS_CASE(`8192`)
337	CC_VLS_CASE(`16384`)
338	CC_VLS_CASE(`32768`)
339	CC_VLS_CASE(`65536`)
340	#undef CC_VLS_CASE
341	}
342	}
343
344	return CC_C;
345	}
346
347	/// Arrange the argument and result information for a call to an
348	/// unknown C++ non-static member function of the given abstract type.
349	/// (A null RD means we don't have any meaningful "this" argument type,
350	/// so fall back to a generic pointer type).
351	/// The member function must be an ordinary function, i.e. not a
352	/// constructor or destructor.
353	const CGFunctionInfo &
354	CodeGenTypes::arrangeCXXMethodType(const CXXRecordDecl *RD,
355	const FunctionProtoType *FTP,
356	const CXXMethodDecl *MD) {
357	CanQualTypeList argTypes;
358
359	// Add the 'this' pointer.
360	argTypes.push_back(Elt: DeriveThisType(RD, MD));
361
362	return ::arrangeLLVMFunctionInfo(
363	CGT&: *this, /instanceMethod=/true, prefix&: argTypes,
364	FTP: FTP->getCanonicalTypeUnqualified().getAs<FunctionProtoType>());
365	}
366
367	/// Set calling convention for CUDA/HIP kernel.
368	static void setCUDAKernelCallingConvention(CanQualType &FTy, CodeGenModule &CGM,
369	const FunctionDecl *FD) {
370	if (FD->hasAttr<CUDAGlobalAttr>()) {
371	const FunctionType *FT = FTy->getAs<FunctionType>();
372	CGM.getTargetCodeGenInfo().setCUDAKernelCallingConvention(FT);
373	FTy = FT->getCanonicalTypeUnqualified();
374	}
375	}
376
377	/// Arrange the argument and result information for a declaration or
378	/// definition of the given C++ non-static member function. The
379	/// member function must be an ordinary function, i.e. not a
380	/// constructor or destructor.
381	const CGFunctionInfo &
382	CodeGenTypes::arrangeCXXMethodDeclaration(const CXXMethodDecl *MD) {
383	assert(!isa<CXXConstructorDecl>(MD) && "wrong method for constructors!");
384	assert(!isa<CXXDestructorDecl>(MD) && "wrong method for destructors!");
385
386	CanQualType FT = GetFormalType(MD).getAs<Type>();
387	setCUDAKernelCallingConvention(FT, CGM, MD);
388	auto prototype = FT.getAs<FunctionProtoType>();
389
390	if (MD->isImplicitObjectMemberFunction()) {
391	// The abstract case is perfectly fine.
392	const CXXRecordDecl *ThisType =
393	getCXXABI().getThisArgumentTypeForMethod(MD);
394	return arrangeCXXMethodType(RD: ThisType, FTP: prototype.getTypePtr(), MD);
395	}
396
397	return arrangeFreeFunctionType(FTP: prototype);
398	}
399
400	bool CodeGenTypes::inheritingCtorHasParams(
401	const InheritedConstructor &Inherited, CXXCtorType Type) {
402	// Parameters are unnecessary if we're constructing a base class subobject
403	// and the inherited constructor lives in a virtual base.
404	return Type == Ctor_Complete \|\|
405	!Inherited.getShadowDecl()->constructsVirtualBase() \|\|
406	!Target.getCXXABI().hasConstructorVariants();
407	}
408
409	const CGFunctionInfo &
410	CodeGenTypes::arrangeCXXStructorDeclaration(GlobalDecl GD) {
411	auto *MD = cast<CXXMethodDecl>(Val: GD.getDecl());
412
413	CanQualTypeList argTypes;
414	ExtParameterInfoList paramInfos;
415
416	const CXXRecordDecl *ThisType = getCXXABI().getThisArgumentTypeForMethod(GD);
417	argTypes.push_back(Elt: DeriveThisType(RD: ThisType, MD));
418
419	bool PassParams = true;
420
421	if (auto *CD = dyn_cast<CXXConstructorDecl>(Val: MD)) {
422	// A base class inheriting constructor doesn't get forwarded arguments
423	// needed to construct a virtual base (or base class thereof).
424	if (auto Inherited = CD->getInheritedConstructor())
425	PassParams = inheritingCtorHasParams(Inherited, Type: GD.getCtorType());
426	}
427
428	CanQual<FunctionProtoType> FTP = GetFormalType(MD);
429
430	// Add the formal parameters.
431	if (PassParams)
432	appendParameterTypes(CGT: *this, prefix&: argTypes, paramInfos, FPT: FTP);
433
434	CGCXXABI::AddedStructorArgCounts AddedArgs =
435	getCXXABI().buildStructorSignature(GD, ArgTys&: argTypes);
436	if (!paramInfos.empty()) {
437	// Note: prefix implies after the first param.
438	if (AddedArgs.Prefix)
439	paramInfos.insert(I: paramInfos.begin() + `1`, NumToInsert: AddedArgs.Prefix,
440	Elt: FunctionProtoType::ExtParameterInfo {});
441	if (AddedArgs.Suffix)
442	paramInfos.append(NumInputs: AddedArgs.Suffix,
443	Elt: FunctionProtoType::ExtParameterInfo {});
444	}
445
446	RequiredArgs required =
447	(PassParams && MD->isVariadic() ? RequiredArgs (argTypes.size())
448	: RequiredArgs::All);
449
450	FunctionType::ExtInfo extInfo = FTP->getExtInfo();
451	CanQualType resultType = getCXXABI().HasThisReturn(GD) ? argTypes.front()
452	: getCXXABI().hasMostDerivedReturn(GD)
453	? CGM.getContext().VoidPtrTy
454	: Context.VoidTy;
455	return arrangeLLVMFunctionInfo(returnType: resultType, opts: FnInfoOpts::IsInstanceMethod,
456	argTypes, info: extInfo, paramInfos, args: required);
457	}
458
459	static CanQualTypeList getArgTypesForCall(ASTContext &ctx,
460	const CallArgList &args) {
461	CanQualTypeList argTypes;
462	for (auto &arg : args)
463	argTypes.push_back(Elt: ctx.getCanonicalParamType(T: arg.Ty));
464	return argTypes;
465	}
466
467	static CanQualTypeList getArgTypesForDeclaration(ASTContext &ctx,
468	const FunctionArgList &args) {
469	CanQualTypeList argTypes;
470	for (auto &arg : args)
471	argTypes.push_back(Elt: ctx.getCanonicalParamType(T: arg->getType()));
472	return argTypes;
473	}
474
475	static ExtParameterInfoList
476	getExtParameterInfosForCall(const FunctionProtoType proto, unsigned* prefixArgs,
477	unsigned totalArgs) {
478	ExtParameterInfoList result;
479	if (proto->hasExtParameterInfos()) {
480	addExtParameterInfosForCall(paramInfos&: result, proto, prefixArgs, totalArgs);
481	}
482	return result;
483	}
484
485	/// Arrange a call to a C++ method, passing the given arguments.
486	///
487	/// ExtraPrefixArgs is the number of ABI-specific args passed after the `this`
488	/// parameter.
489	/// ExtraSuffixArgs is the number of ABI-specific args passed at the end of
490	/// args.
491	/// PassProtoArgs indicates whether `args` has args for the parameters in the
492	/// given CXXConstructorDecl.
493	const CGFunctionInfo &CodeGenTypes::arrangeCXXConstructorCall(
494	const CallArgList &args, const CXXConstructorDecl *D, CXXCtorType CtorKind,
495	unsigned ExtraPrefixArgs, unsigned ExtraSuffixArgs, bool PassProtoArgs) {
496	CanQualTypeList ArgTypes;
497	for (const auto &Arg : args)
498	ArgTypes.push_back(Elt: Context.getCanonicalParamType(T: Arg.Ty));
499
500	// +1 for implicit this, which should always be args[0].
501	unsigned TotalPrefixArgs = `1` + ExtraPrefixArgs;
502
503	CanQual<FunctionProtoType> FPT = GetFormalType(D);
504	RequiredArgs Required = PassProtoArgs
505	? RequiredArgs::forPrototypePlus(
506	prototype: FPT, additional: TotalPrefixArgs + ExtraSuffixArgs)
507	: RequiredArgs::All;
508
509	GlobalDecl GD(D, CtorKind);
510	CanQualType ResultType = getCXXABI().HasThisReturn(GD) ? ArgTypes.front()
511	: getCXXABI().hasMostDerivedReturn(GD)
512	? CGM.getContext().VoidPtrTy
513	: Context.VoidTy;
514
515	FunctionType::ExtInfo Info = FPT->getExtInfo();
516	ExtParameterInfoList ParamInfos;
517	// If the prototype args are elided, we should only have ABI-specific args,
518	// which never have param info.
519	if (PassProtoArgs && FPT->hasExtParameterInfos()) {
520	// ABI-specific suffix arguments are treated the same as variadic arguments.
521	addExtParameterInfosForCall(paramInfos&: ParamInfos, proto: FPT.getTypePtr(), prefixArgs: TotalPrefixArgs,
522	totalArgs: ArgTypes.size());
523	}
524
525	return arrangeLLVMFunctionInfo(returnType: ResultType, opts: FnInfoOpts::IsInstanceMethod,
526	argTypes: ArgTypes, info: Info, paramInfos: ParamInfos, args: Required);
527	}
528
529	/// Arrange the argument and result information for the declaration or
530	/// definition of the given function.
531	const CGFunctionInfo &
532	CodeGenTypes::arrangeFunctionDeclaration(const GlobalDecl GD) {
533	const FunctionDecl *FD = cast<FunctionDecl>(Val: GD.getDecl());
534	if (const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(Val: FD))
535	if (MD->isImplicitObjectMemberFunction())
536	return arrangeCXXMethodDeclaration(MD);
537
538	CanQualType FTy = FD->getType()->getCanonicalTypeUnqualified();
539
540	assert(isa<FunctionType>(FTy));
541	setCUDAKernelCallingConvention(FTy, CGM, FD);
542
543	if (DeviceKernelAttr::isOpenCLSpelling(FD->getAttr<DeviceKernelAttr>()) &&
544	GD.getKernelReferenceKind() == KernelReferenceKind::Stub) {
545	const FunctionType *FT = FTy->getAs<FunctionType>();
546	CGM.getTargetCodeGenInfo().setOCLKernelStubCallingConvention(FT);
547	FTy = FT->getCanonicalTypeUnqualified();
548	}
549
550	// When declaring a function without a prototype, always use a
551	// non-variadic type.
552	if (CanQual<FunctionNoProtoType> noProto = FTy.getAs<FunctionNoProtoType>()) {
553	return arrangeLLVMFunctionInfo(noProto->getReturnType(), FnInfoOpts::None,
554	{}, noProto->getExtInfo(), {},
555	RequiredArgs::All);
556	}
557
558	return arrangeFreeFunctionType(FTP: FTy.castAs<FunctionProtoType>());
559	}
560
561	/// Arrange the argument and result information for the declaration or
562	/// definition of an Objective-C method.
563	const CGFunctionInfo &
564	CodeGenTypes::arrangeObjCMethodDeclaration(const ObjCMethodDecl *MD) {
565	// It happens that this is the same as a call with no optional
566	// arguments, except also using the formal 'self' type.
567	return arrangeObjCMessageSendSignature(MD, receiverType: MD->getSelfDecl()->getType());
568	}
569
570	/// Arrange the argument and result information for the function type
571	/// through which to perform a send to the given Objective-C method,
572	/// using the given receiver type. The receiver type is not always
573	/// the 'self' type of the method or even an Objective-C pointer type.
574	/// This is not* the right method for actually performing such a*
575	/// message send, due to the possibility of optional arguments.
576	const CGFunctionInfo &
577	CodeGenTypes::arrangeObjCMessageSendSignature(const ObjCMethodDecl *MD,
578	QualType receiverType) {
579	CanQualTypeList argTys;
580	ExtParameterInfoList extParamInfos(MD->isDirectMethod() ? `1` : `2`);
581	argTys.push_back(Elt: Context.getCanonicalParamType(T: receiverType));
582	if (!MD->isDirectMethod())
583	argTys.push_back(Elt: Context.getCanonicalParamType(T: Context.getObjCSelType()));
584	for (const auto *I : MD->parameters()) {
585	argTys.push_back(Elt: Context.getCanonicalParamType(T: I->getType()));
586	auto extParamInfo = FunctionProtoType::ExtParameterInfo().withIsNoEscape(
587	I->hasAttr<NoEscapeAttr>());
588	extParamInfos.push_back(Elt: extParamInfo);
589	}
590
591	FunctionType::ExtInfo einfo;
592	bool IsTargetDefaultMSABI =
593	getContext().getTargetInfo().getTriple().isOSWindows() \|\|
594	getContext().getTargetInfo().getTriple().isUEFI();
595	einfo = einfo.withCallingConv(
596	cc: getCallingConventionForDecl(D: MD, IsTargetDefaultMSABI));
597
598	if (getContext().getLangOpts().ObjCAutoRefCount &&
599	MD->hasAttr<NSReturnsRetainedAttr>())
600	einfo = einfo.withProducesResult(producesResult: true);
601
602	RequiredArgs required =
603	(MD->isVariadic() ? RequiredArgs (argTys.size()) : RequiredArgs::All);
604
605	return arrangeLLVMFunctionInfo(returnType: GetReturnType(RetTy: MD->getReturnType()),
606	opts: FnInfoOpts::None, argTypes: argTys, info: einfo, paramInfos: extParamInfos,
607	args: required);
608	}
609
610	const CGFunctionInfo &
611	CodeGenTypes::arrangeUnprototypedObjCMessageSend(QualType returnType,
612	const CallArgList &args) {
613	CanQualTypeList argTypes = getArgTypesForCall(ctx&: Context, args);
614	FunctionType::ExtInfo einfo;
615
616	return arrangeLLVMFunctionInfo(returnType: GetReturnType(RetTy: returnType), opts: FnInfoOpts::None,
617	argTypes, info: einfo, paramInfos: {}, args: RequiredArgs::All);
618	}
619
620	const CGFunctionInfo &CodeGenTypes::arrangeGlobalDeclaration(GlobalDecl GD) {
621	// FIXME: Do we need to handle ObjCMethodDecl?
622	if (isa<CXXConstructorDecl>(Val: GD.getDecl()) \|\|
623	isa<CXXDestructorDecl>(Val: GD.getDecl()))
624	return arrangeCXXStructorDeclaration(GD);
625
626	return arrangeFunctionDeclaration(GD);
627	}
628
629	/// Arrange a thunk that takes 'this' as the first parameter followed by
630	/// varargs. Return a void pointer, regardless of the actual return type.
631	/// The body of the thunk will end in a musttail call to a function of the
632	/// correct type, and the caller will bitcast the function to the correct
633	/// prototype.
634	const CGFunctionInfo &
635	CodeGenTypes::arrangeUnprototypedMustTailThunk(const CXXMethodDecl *MD) {
636	assert(MD->isVirtual() && "only methods have thunks");
637	CanQual<FunctionProtoType> FTP = GetFormalType(MD);
638	CanQualType ArgTys[] = {DeriveThisType(RD: MD->getParent(), MD)};
639	return arrangeLLVMFunctionInfo(Context.VoidTy, FnInfoOpts::None, ArgTys,
640	FTP->getExtInfo(), {}, RequiredArgs(`1`));
641	}
642
643	const CGFunctionInfo &
644	CodeGenTypes::arrangeMSCtorClosure(const CXXConstructorDecl *CD,
645	CXXCtorType CT) {
646	assert(CT == Ctor_CopyingClosure \|\| CT == Ctor_DefaultClosure);
647
648	CanQual<FunctionProtoType> FTP = GetFormalType(CD);
649	SmallVector<CanQualType, `2`> ArgTys;
650	const CXXRecordDecl *RD = CD->getParent();
651	ArgTys.push_back(Elt: DeriveThisType(RD, CD));
652	if (CT == Ctor_CopyingClosure)
653	ArgTys.push_back(*FTP->param_type_begin());
654	if (RD->getNumVBases() > `0`)
655	ArgTys.push_back(Elt: Context.IntTy);
656	CallingConv CC = Context.getDefaultCallingConvention(
657	/IsVariadic=/false, /IsCXXMethod=/true);
658	return arrangeLLVMFunctionInfo(returnType: Context.VoidTy, opts: FnInfoOpts::IsInstanceMethod,
659	argTypes: ArgTys, info: FunctionType::ExtInfo (CC), paramInfos: {},
660	args: RequiredArgs::All);
661	}
662
663	/// Arrange a call as unto a free function, except possibly with an
664	/// additional number of formal parameters considered required.
665	static const CGFunctionInfo &
666	arrangeFreeFunctionLikeCall(CodeGenTypes &CGT, CodeGenModule &CGM,
667	const CallArgList &args, const FunctionType *fnType,
668	unsigned numExtraRequiredArgs, bool chainCall) {
669	assert(args.size() >= numExtraRequiredArgs);
670
671	ExtParameterInfoList paramInfos;
672
673	// In most cases, there are no optional arguments.
674	RequiredArgs required = RequiredArgs::All;
675
676	// If we have a variadic prototype, the required arguments are the
677	// extra prefix plus the arguments in the prototype.
678	if (const FunctionProtoType *proto = dyn_cast<FunctionProtoType>(Val: fnType)) {
679	if (proto->isVariadic())
680	required = RequiredArgs::forPrototypePlus(prototype: proto, additional: numExtraRequiredArgs);
681
682	if (proto->hasExtParameterInfos())
683	addExtParameterInfosForCall(paramInfos, proto, prefixArgs: numExtraRequiredArgs,
684	totalArgs: args.size());
685
686	// If we don't have a prototype at all, but we're supposed to
687	// explicitly use the variadic convention for unprototyped calls,
688	// treat all of the arguments as required but preserve the nominal
689	// possibility of variadics.
690	} else if (CGM.getTargetCodeGenInfo().isNoProtoCallVariadic(
691	args, fnType: cast<FunctionNoProtoType>(Val: fnType))) {
692	required = RequiredArgs (args.size());
693	}
694
695	CanQualTypeList argTypes;
696	for (const auto &arg : args)
697	argTypes.push_back(Elt: CGT.getContext().getCanonicalParamType(T: arg.Ty));
698	FnInfoOpts opts = chainCall ? FnInfoOpts::IsChainCall : FnInfoOpts::None;
699	return CGT.arrangeLLVMFunctionInfo(returnType: GetReturnType(RetTy: fnType->getReturnType()),
700	opts, argTypes, info: fnType->getExtInfo(),
701	paramInfos, args: required);
702	}
703
704	/// Figure out the rules for calling a function with the given formal
705	/// type using the given arguments. The arguments are necessary
706	/// because the function might be unprototyped, in which case it's
707	/// target-dependent in crazy ways.
708	const CGFunctionInfo &CodeGenTypes::arrangeFreeFunctionCall(
709	const CallArgList &args, const FunctionType fnType, bool* chainCall) {
710	return arrangeFreeFunctionLikeCall(CGT&: *this, CGM, args, fnType,
711	numExtraRequiredArgs: chainCall ? `1` : `0`, chainCall);
712	}
713
714	/// A block function is essentially a free function with an
715	/// extra implicit argument.
716	const CGFunctionInfo &
717	CodeGenTypes::arrangeBlockFunctionCall(const CallArgList &args,
718	const FunctionType *fnType) {
719	return arrangeFreeFunctionLikeCall(CGT&: *this, CGM, args, fnType, numExtraRequiredArgs: `1`,
720	/chainCall=/false);
721	}
722
723	const CGFunctionInfo &
724	CodeGenTypes::arrangeBlockFunctionDeclaration(const FunctionProtoType *proto,
725	const FunctionArgList &params) {
726	ExtParameterInfoList paramInfos =
727	getExtParameterInfosForCall(proto, prefixArgs: `1`, totalArgs: params.size());
728	CanQualTypeList argTypes = getArgTypesForDeclaration(ctx&: Context, args: params);
729
730	return arrangeLLVMFunctionInfo(returnType: GetReturnType(proto->getReturnType()),
731	opts: FnInfoOpts::None, argTypes,
732	info: proto->getExtInfo(), paramInfos,
733	args: RequiredArgs::forPrototypePlus(prototype: proto, additional: `1`));
734	}
735
736	const CGFunctionInfo &
737	CodeGenTypes::arrangeBuiltinFunctionCall(QualType resultType,
738	const CallArgList &args) {
739	CanQualTypeList argTypes;
740	for (const auto &Arg : args)
741	argTypes.push_back(Elt: Context.getCanonicalParamType(T: Arg.Ty));
742	return arrangeLLVMFunctionInfo(returnType: GetReturnType(RetTy: resultType), opts: FnInfoOpts::None,
743	argTypes, info: FunctionType::ExtInfo (),
744	/paramInfos=/{}, args: RequiredArgs::All);
745	}
746
747	const CGFunctionInfo &
748	CodeGenTypes::arrangeBuiltinFunctionDeclaration(QualType resultType,
749	const FunctionArgList &args) {
750	CanQualTypeList argTypes = getArgTypesForDeclaration(ctx&: Context, args);
751
752	return arrangeLLVMFunctionInfo(returnType: GetReturnType(RetTy: resultType), opts: FnInfoOpts::None,
753	argTypes, info: FunctionType::ExtInfo (), paramInfos: {},
754	args: RequiredArgs::All);
755	}
756
757	const CGFunctionInfo &CodeGenTypes::arrangeBuiltinFunctionDeclaration(
758	CanQualType resultType, ArrayRef<CanQualType> argTypes) {
759	return arrangeLLVMFunctionInfo(returnType: resultType, opts: FnInfoOpts::None, argTypes,
760	info: FunctionType::ExtInfo (), paramInfos: {},
761	args: RequiredArgs::All);
762	}
763
764	const CGFunctionInfo &
765	CodeGenTypes::arrangeSYCLKernelCallerDeclaration(QualType resultType,
766	const FunctionArgList &args) {
767	CanQualTypeList argTypes = getArgTypesForDeclaration(ctx&: Context, args);
768
769	return arrangeLLVMFunctionInfo(returnType: GetReturnType(RetTy: resultType), opts: FnInfoOpts::None,
770	argTypes,
771	info: FunctionType::ExtInfo (CC_DeviceKernel),
772	/paramInfos=/{}, args: RequiredArgs::All);
773	}
774
775	/// Arrange a call to a C++ method, passing the given arguments.
776	///
777	/// numPrefixArgs is the number of ABI-specific prefix arguments we have. It
778	/// does not count `this`.
779	const CGFunctionInfo &CodeGenTypes::arrangeCXXMethodCall(
780	const CallArgList &args, const FunctionProtoType *proto,
781	RequiredArgs required, unsigned numPrefixArgs) {
782	assert(numPrefixArgs + `1` <= args.size() &&
783	"Emitting a call with less args than the required prefix?");
784	// Add one to account for `this`. It's a bit awkward here, but we don't count
785	// `this` in similar places elsewhere.
786	ExtParameterInfoList paramInfos =
787	getExtParameterInfosForCall(proto, prefixArgs: numPrefixArgs + `1`, totalArgs: args.size());
788
789	CanQualTypeList argTypes = getArgTypesForCall(ctx&: Context, args);
790
791	FunctionType::ExtInfo info = proto->getExtInfo();
792	return arrangeLLVMFunctionInfo(returnType: GetReturnType(proto->getReturnType()),
793	opts: FnInfoOpts::IsInstanceMethod, argTypes, info,
794	paramInfos, args: required);
795	}
796
797	const CGFunctionInfo &CodeGenTypes::arrangeNullaryFunction() {
798	return arrangeLLVMFunctionInfo(returnType: getContext().VoidTy, opts: FnInfoOpts::None, argTypes: {},
799	info: FunctionType::ExtInfo (), paramInfos: {},
800	args: RequiredArgs::All);
801	}
802
803	const CGFunctionInfo &CodeGenTypes::arrangeCall(const CGFunctionInfo &signature,
804	const CallArgList &args) {
805	assert(signature.arg_size() <= args.size());
806	if (signature.arg_size() == args.size())
807	return signature;
808
809	ExtParameterInfoList paramInfos;
810	auto sigParamInfos = signature.getExtParameterInfos();
811	if (!sigParamInfos.empty()) {
812	paramInfos.append(in_start: sigParamInfos.begin(), in_end: sigParamInfos.end());
813	paramInfos.resize(N: args.size());
814	}
815
816	CanQualTypeList argTypes = getArgTypesForCall(ctx&: Context, args);
817
818	assert(signature.getRequiredArgs().allowsOptionalArgs());
819	FnInfoOpts opts = FnInfoOpts::None;
820	if (signature.isInstanceMethod())
821	opts \|= FnInfoOpts::IsInstanceMethod;
822	if (signature.isChainCall())
823	opts \|= FnInfoOpts::IsChainCall;
824	if (signature.isDelegateCall())
825	opts \|= FnInfoOpts::IsDelegateCall;
826	return arrangeLLVMFunctionInfo(returnType: signature.getReturnType(), opts, argTypes,
827	info: signature.getExtInfo(), paramInfos,
828	args: signature.getRequiredArgs());
829	}
830
831	namespace clang {
832	namespace CodeGen {
833	void computeSPIRKernelABIInfo(CodeGenModule &CGM, CGFunctionInfo &FI);
834	}
835	} // namespace clang
836
837	/// Arrange the argument and result information for an abstract value
838	/// of a given function type. This is the method which all of the
839	/// above functions ultimately defer to.
840	const CGFunctionInfo &CodeGenTypes::arrangeLLVMFunctionInfo(
841	CanQualType resultType, FnInfoOpts opts, ArrayRef<CanQualType> argTypes,
842	FunctionType::ExtInfo info,
843	ArrayRef<FunctionProtoType::ExtParameterInfo> paramInfos,
844	RequiredArgs required) {
845	assert(llvm::all_of(argTypes,
846	[](CanQualType T) { return T.isCanonicalAsParam(); }));
847
848	// Lookup or create unique function info.
849	llvm::FoldingSetNodeID ID;
850	bool isInstanceMethod =
851	(opts & FnInfoOpts::IsInstanceMethod) == FnInfoOpts::IsInstanceMethod;
852	bool isChainCall =
853	(opts & FnInfoOpts::IsChainCall) == FnInfoOpts::IsChainCall;
854	bool isDelegateCall =
855	(opts & FnInfoOpts::IsDelegateCall) == FnInfoOpts::IsDelegateCall;
856	CGFunctionInfo::Profile(ID, InstanceMethod: isInstanceMethod, ChainCall: isChainCall, IsDelegateCall: isDelegateCall,
857	info, paramInfos, required, resultType, argTypes);
858
859	void insertPos = nullptr*;
860	CGFunctionInfo *FI = FunctionInfos.FindNodeOrInsertPos(ID, InsertPos&: insertPos);
861	if (FI)
862	return *FI;
863
864	unsigned CC = ClangCallConvToLLVMCallConv(CC: info.getCC());
865
866	// Construct the function info. We co-allocate the ArgInfos.
867	FI = CGFunctionInfo::create(llvmCC: CC, instanceMethod: isInstanceMethod, chainCall: isChainCall, delegateCall: isDelegateCall,
868	extInfo: info, paramInfos, resultType, argTypes, required);
869	FunctionInfos.InsertNode(N: FI, InsertPos: insertPos);
870
871	bool inserted = FunctionsBeingProcessed.insert(Ptr: FI).second;
872	(void)inserted;
873	assert(inserted && "Recursively being processed?");
874
875	// Compute ABI information.
876	if (CC == llvm::CallingConv::SPIR_KERNEL) {
877	// Force target independent argument handling for the host visible
878	// kernel functions.
879	computeSPIRKernelABIInfo(CGM, FI&: *FI);
880	} else if (info.getCC() == CC_Swift \|\| info.getCC() == CC_SwiftAsync) {
881	swiftcall::computeABIInfo(CGM, FI&: *FI);
882	} else {
883	CGM.getABIInfo().computeInfo(FI&: *FI);
884	}
885
886	// Loop over all of the computed argument and return value info. If any of
887	// them are direct or extend without a specified coerce type, specify the
888	// default now.
889	ABIArgInfo &retInfo = FI->getReturnInfo();
890	if (retInfo.canHaveCoerceToType() && retInfo.getCoerceToType() == nullptr)
891	retInfo.setCoerceToType(ConvertType(T: FI->getReturnType()));
892
893	for (auto &I : FI->arguments())
894	if (I.info.canHaveCoerceToType() && I.info.getCoerceToType() == nullptr)
895	I.info.setCoerceToType(ConvertType(T: I.type));
896
897	bool erased = FunctionsBeingProcessed.erase(Ptr: FI);
898	(void)erased;
899	assert(erased && "Not in set?");
900
901	return *FI;
902	}
903
904	CGFunctionInfo CGFunctionInfo::create(unsigned* llvmCC, bool instanceMethod,
905	bool chainCall, bool delegateCall,
906	const FunctionType::ExtInfo &info,
907	ArrayRef<ExtParameterInfo> paramInfos,
908	CanQualType resultType,
909	ArrayRef<CanQualType> argTypes,
910	RequiredArgs required) {
911	assert(paramInfos.empty() \|\| paramInfos.size() == argTypes.size());
912	assert(!required.allowsOptionalArgs() \|\|
913	required.getNumRequiredArgs() <= argTypes.size());
914
915	void buffer = operator* new(totalSizeToAlloc<ArgInfo, ExtParameterInfo>(
916	Counts: argTypes.size() + `1`, Counts: paramInfos.size()));
917
918	CGFunctionInfo FI = new* (buffer) CGFunctionInfo ();
919	FI->CallingConvention = llvmCC;
920	FI->EffectiveCallingConvention = llvmCC;
921	FI->ASTCallingConvention = info.getCC();
922	FI->InstanceMethod = instanceMethod;
923	FI->ChainCall = chainCall;
924	FI->DelegateCall = delegateCall;
925	FI->CmseNSCall = info.getCmseNSCall();
926	FI->NoReturn = info.getNoReturn();
927	FI->ReturnsRetained = info.getProducesResult();
928	FI->NoCallerSavedRegs = info.getNoCallerSavedRegs();
929	FI->NoCfCheck = info.getNoCfCheck();
930	FI->Required = required;
931	FI->HasRegParm = info.getHasRegParm();
932	FI->RegParm = info.getRegParm();
933	FI->ArgStruct = nullptr;
934	FI->ArgStructAlign = `0`;
935	FI->NumArgs = argTypes.size();
936	FI->HasExtParameterInfos = !paramInfos.empty();
937	FI->getArgsBuffer()[`0`].type = resultType;
938	FI->MaxVectorWidth = `0`;
939	for (unsigned i = `0`, e = argTypes.size(); i != e; ++i)
940	FI->getArgsBuffer()[i + `1`].type = argTypes [i];
941	for (unsigned i = `0`, e = paramInfos.size(); i != e; ++i)
942	FI->getExtParameterInfosBuffer()[i] = paramInfos [i];
943	return FI;
944	}
945
946	/*/
947
948	namespace {
949	// ABIArgInfo::Expand implementation.
950
951	// Specifies the way QualType passed as ABIArgInfo::Expand is expanded.
952	struct TypeExpansion {
953	enum TypeExpansionKind {
954	// Elements of constant arrays are expanded recursively.
955	TEK_ConstantArray,
956	// Record fields are expanded recursively (but if record is a union, only
957	// the field with the largest size is expanded).
958	TEK_Record,
959	// For complex types, real and imaginary parts are expanded recursively.
960	TEK_Complex,
961	// All other types are not expandable.
962	TEK_None
963	};
964
965	const TypeExpansionKind Kind;
966
967	TypeExpansion(TypeExpansionKind K) : Kind(K) {}
968	virtual ~TypeExpansion() {}
969	};
970
971	struct ConstantArrayExpansion : TypeExpansion {
972	QualType EltTy;
973	uint64_t NumElts;
974
975	ConstantArrayExpansion(QualType EltTy, uint64_t NumElts)
976	: TypeExpansion (TEK_ConstantArray), EltTy(EltTy), NumElts(NumElts) {}
977	static bool classof(const TypeExpansion *TE) {
978	return TE->Kind == TEK_ConstantArray;
979	}
980	};
981
982	struct RecordExpansion : TypeExpansion {
983	SmallVector<const CXXBaseSpecifier *, `1`> Bases;
984
985	SmallVector<const FieldDecl *, `1`> Fields;
986
987	RecordExpansion(SmallVector<const CXXBaseSpecifier *, `1`> &&Bases,
988	SmallVector<const FieldDecl *, `1`> &&Fields)
989	: TypeExpansion (TEK_Record), Bases (std::move(Bases)),
990	Fields (std::move(Fields)) {}
991	static bool classof(const TypeExpansion *TE) {
992	return TE->Kind == TEK_Record;
993	}
994	};
995
996	struct ComplexExpansion : TypeExpansion {
997	QualType EltTy;
998
999	ComplexExpansion(QualType EltTy) : TypeExpansion (TEK_Complex), EltTy(EltTy) {}
1000	static bool classof(const TypeExpansion *TE) {
1001	return TE->Kind == TEK_Complex;
1002	}
1003	};
1004
1005	struct NoExpansion : TypeExpansion {
1006	NoExpansion() : TypeExpansion (TEK_None) {}
1007	static bool classof(const TypeExpansion TE) { return* TE->Kind == TEK_None; }
1008	};
1009	} // namespace
1010
1011	static std::unique_ptr<TypeExpansion>
1012	getTypeExpansion(QualType Ty, const ASTContext &Context) {
1013	if (const ConstantArrayType *AT = Context.getAsConstantArrayType(T: Ty)) {
1014	return std::make_unique<ConstantArrayExpansion>(AT->getElementType(),
1015	AT->getZExtSize());
1016	}
1017	if (const RecordType *RT = Ty ->getAs<RecordType>()) {
1018	SmallVector<const CXXBaseSpecifier *, `1`> Bases;
1019	SmallVector<const FieldDecl *, `1`> Fields;
1020	const RecordDecl *RD = RT->getDecl();
1021	assert(!RD->hasFlexibleArrayMember() &&
1022	"Cannot expand structure with flexible array.");
1023	if (RD->isUnion()) {
1024	// Unions can be here only in degenerative cases - all the fields are same
1025	// after flattening. Thus we have to use the "largest" field.
1026	const FieldDecl LargestFD = nullptr*;
1027	CharUnits UnionSize = CharUnits::Zero();
1028
1029	for (const auto *FD : RD->fields()) {
1030	if (FD->isZeroLengthBitField())
1031	continue;
1032	assert(!FD->isBitField() &&
1033	"Cannot expand structure with bit-field members.");
1034	CharUnits FieldSize = Context.getTypeSizeInChars(FD->getType());
1035	if (UnionSize < FieldSize) {
1036	UnionSize = FieldSize;
1037	LargestFD = FD;
1038	}
1039	}
1040	if (LargestFD)
1041	Fields.push_back(Elt: LargestFD);
1042	} else {
1043	if (const auto *CXXRD = dyn_cast<CXXRecordDecl>(Val: RD)) {
1044	assert(!CXXRD->isDynamicClass() &&
1045	"cannot expand vtable pointers in dynamic classes");
1046	llvm::append_range(C&: Bases, R: llvm::make_pointer_range(Range: CXXRD->bases()));
1047	}
1048
1049	for (const auto *FD : RD->fields()) {
1050	if (FD->isZeroLengthBitField())
1051	continue;
1052	assert(!FD->isBitField() &&
1053	"Cannot expand structure with bit-field members.");
1054	Fields.push_back(Elt: FD);
1055	}
1056	}
1057	return std::make_unique<RecordExpansion>(args: std::move(Bases),
1058	args: std::move(Fields));
1059	}
1060	if (const ComplexType *CT = Ty ->getAs<ComplexType>()) {
1061	return std::make_unique<ComplexExpansion>(args: CT->getElementType());
1062	}
1063	return std::make_unique<NoExpansion>();
1064	}
1065
1066	static int getExpansionSize(QualType Ty, const ASTContext &Context) {
1067	auto Exp = getTypeExpansion(Ty, Context);
1068	if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Val: Exp.get())) {
1069	return CAExp->NumElts * getExpansionSize(CAExp->EltTy, Context);
1070	}
1071	if (auto RExp = dyn_cast<RecordExpansion>(Val: Exp.get())) {
1072	int Res = `0`;
1073	for (auto BS : RExp->Bases)
1074	Res += getExpansionSize(Ty: BS->getType(), Context);
1075	for (auto FD : RExp->Fields)
1076	Res += getExpansionSize(FD->getType(), Context);
1077	return Res;
1078	}
1079	if (isa<ComplexExpansion>(Val: Exp.get()))
1080	return `2`;
1081	assert(isa<NoExpansion>(Exp.get()));
1082	return `1`;
1083	}
1084
1085	void CodeGenTypes::getExpandedTypes(
1086	QualType Ty, SmallVectorImpl<llvm::Type *>::iterator &TI) {
1087	auto Exp = getTypeExpansion(Ty, Context);
1088	if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Val: Exp.get())) {
1089	for (int i = `0`, n = CAExp->NumElts; i < n; i++) {
1090	getExpandedTypes(Ty: CAExp->EltTy, TI);
1091	}
1092	} else if (auto RExp = dyn_cast<RecordExpansion>(Val: Exp.get())) {
1093	for (auto BS : RExp->Bases)
1094	getExpandedTypes(Ty: BS->getType(), TI);
1095	for (auto FD : RExp->Fields)
1096	getExpandedTypes(Ty: FD->getType(), TI);
1097	} else if (auto CExp = dyn_cast<ComplexExpansion>(Val: Exp.get())) {
1098	llvm::Type *EltTy = ConvertType(T: CExp->EltTy);
1099	*TI++ = EltTy;
1100	*TI++ = EltTy;
1101	} else {
1102	assert(isa<NoExpansion>(Exp.get()));
1103	*TI++ = ConvertType(T: Ty);
1104	}
1105	}
1106
1107	static void forConstantArrayExpansion(CodeGenFunction &CGF,
1108	ConstantArrayExpansion *CAE,
1109	Address BaseAddr,
1110	llvm::function_ref<void(Address)> Fn) {
1111	for (int i = `0`, n = CAE->NumElts; i < n; i++) {
1112	Address EltAddr = CGF.Builder.CreateConstGEP2_32(Addr: BaseAddr, Idx0: `0`, Idx1: i);
1113	Fn (EltAddr);
1114	}
1115	}
1116
1117	void CodeGenFunction::ExpandTypeFromArgs(QualType Ty, LValue LV,
1118	llvm::Function::arg_iterator &AI) {
1119	assert(LV.isSimple() &&
1120	"Unexpected non-simple lvalue during struct expansion.");
1121
1122	auto Exp = getTypeExpansion(Ty, Context: getContext());
1123	if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Val: Exp.get())) {
1124	forConstantArrayExpansion(
1125	CGF&: *this, CAE: CAExp, BaseAddr: LV.getAddress(), Fn: [&](Address EltAddr) {
1126	LValue LV = MakeAddrLValue(EltAddr, CAExp->EltTy);
1127	ExpandTypeFromArgs(Ty: CAExp->EltTy, LV, AI);
1128	});
1129	} else if (auto RExp = dyn_cast<RecordExpansion>(Val: Exp.get())) {
1130	Address This = LV.getAddress();
1131	for (const CXXBaseSpecifier *BS : RExp->Bases) {
1132	// Perform a single step derived-to-base conversion.
1133	Address Base =
1134	GetAddressOfBaseClass(Value: This, Derived: Ty ->getAsCXXRecordDecl(), PathBegin: &BS, PathEnd: &BS + `1`,
1135	/NullCheckValue=/false, Loc: SourceLocation ());
1136	LValue SubLV = MakeAddrLValue(Addr: Base, T: BS->getType());
1137
1138	// Recurse onto bases.
1139	ExpandTypeFromArgs(Ty: BS->getType(), LV: SubLV, AI);
1140	}
1141	for (auto FD : RExp->Fields) {
1142	// FIXME: What are the right qualifiers here?
1143	LValue SubLV = EmitLValueForFieldInitialization(Base: LV, Field: FD);
1144	ExpandTypeFromArgs(Ty: FD->getType(), LV: SubLV, AI);
1145	}
1146	} else if (isa<ComplexExpansion>(Val: Exp.get())) {
1147	auto realValue = &*AI++;
1148	auto imagValue = &*AI++;
1149	EmitStoreOfComplex(V: ComplexPairTy (realValue, imagValue), dest: LV, /init/ isInit: true);
1150	} else {
1151	// Call EmitStoreOfScalar except when the lvalue is a bitfield to emit a
1152	// primitive store.
1153	assert(isa<NoExpansion>(Exp.get()));
1154	llvm::Value Arg = &AI++;
1155	if (LV.isBitField()) {
1156	EmitStoreThroughLValue(Src: RValue::get(V: Arg), Dst: LV);
1157	} else {
1158	// TODO: currently there are some places are inconsistent in what LLVM
1159	// pointer type they use (see D118744). Once clang uses opaque pointers
1160	// all LLVM pointer types will be the same and we can remove this check.
1161	if (Arg->getType()->isPointerTy()) {
1162	Address Addr = LV.getAddress();
1163	Arg = Builder.CreateBitCast(V: Arg, DestTy: Addr.getElementType());
1164	}
1165	EmitStoreOfScalar(value: Arg, lvalue: LV);
1166	}
1167	}
1168	}
1169
1170	void CodeGenFunction::ExpandTypeToArgs(
1171	QualType Ty, CallArg Arg, llvm::FunctionType *IRFuncTy,
1172	SmallVectorImpl<llvm::Value > &IRCallArgs, unsigned* &IRCallArgPos) {
1173	auto Exp = getTypeExpansion(Ty, Context: getContext());
1174	if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Val: Exp.get())) {
1175	Address Addr = Arg.hasLValue() ? Arg.getKnownLValue().getAddress()
1176	: Arg.getKnownRValue().getAggregateAddress();
1177	forConstantArrayExpansion(CGF&: *this, CAE: CAExp, BaseAddr: Addr, Fn: [&](Address EltAddr) {
1178	CallArg EltArg =
1179	CallArg(convertTempToRValue(addr: EltAddr, type: CAExp->EltTy, Loc: SourceLocation ()),
1180	CAExp->EltTy);
1181	ExpandTypeToArgs(Ty: CAExp->EltTy, Arg: EltArg, IRFuncTy, IRCallArgs,
1182	IRCallArgPos);
1183	});
1184	} else if (auto RExp = dyn_cast<RecordExpansion>(Val: Exp.get())) {
1185	Address This = Arg.hasLValue() ? Arg.getKnownLValue().getAddress()
1186	: Arg.getKnownRValue().getAggregateAddress();
1187	for (const CXXBaseSpecifier *BS : RExp->Bases) {
1188	// Perform a single step derived-to-base conversion.
1189	Address Base =
1190	GetAddressOfBaseClass(Value: This, Derived: Ty ->getAsCXXRecordDecl(), PathBegin: &BS, PathEnd: &BS + `1`,
1191	/NullCheckValue=/false, Loc: SourceLocation ());
1192	CallArg BaseArg = CallArg (RValue::getAggregate(addr: Base), BS->getType());
1193
1194	// Recurse onto bases.
1195	ExpandTypeToArgs(Ty: BS->getType(), Arg: BaseArg, IRFuncTy, IRCallArgs,
1196	IRCallArgPos);
1197	}
1198
1199	LValue LV = MakeAddrLValue(Addr: This, T: Ty);
1200	for (auto FD : RExp->Fields) {
1201	CallArg FldArg =
1202	CallArg(EmitRValueForField(LV, FD, Loc: SourceLocation ()), FD->getType());
1203	ExpandTypeToArgs(Ty: FD->getType(), Arg: FldArg, IRFuncTy, IRCallArgs,
1204	IRCallArgPos);
1205	}
1206	} else if (isa<ComplexExpansion>(Val: Exp.get())) {
1207	ComplexPairTy CV = Arg.getKnownRValue().getComplexVal();
1208	IRCallArgs [IRCallArgPos++] = CV.first;
1209	IRCallArgs [IRCallArgPos++] = CV.second;
1210	} else {
1211	assert(isa<NoExpansion>(Exp.get()));
1212	auto RV = Arg.getKnownRValue();
1213	assert(RV.isScalar() &&
1214	"Unexpected non-scalar rvalue during struct expansion.");
1215
1216	// Insert a bitcast as needed.
1217	llvm::Value *V = RV.getScalarVal();
1218	if (IRCallArgPos < IRFuncTy->getNumParams() &&
1219	V->getType() != IRFuncTy->getParamType(i: IRCallArgPos))
1220	V = Builder.CreateBitCast(V, DestTy: IRFuncTy->getParamType(i: IRCallArgPos));
1221
1222	IRCallArgs [IRCallArgPos++] = V;
1223	}
1224	}
1225
1226	/// Create a temporary allocation for the purposes of coercion.
1227	static RawAddress CreateTempAllocaForCoercion(CodeGenFunction &CGF,
1228	llvm::Type *Ty,
1229	CharUnits MinAlign,
1230	const Twine &Name = "tmp") {
1231	// Don't use an alignment that's worse than what LLVM would prefer.
1232	auto PrefAlign = CGF.CGM.getDataLayout().getPrefTypeAlign(Ty);
1233	CharUnits Align = std::max(a: MinAlign, b: CharUnits::fromQuantity(Quantity: PrefAlign));
1234
1235	return CGF.CreateTempAlloca(Ty, align: Align, Name: Name + ".coerce");
1236	}
1237
1238	/// EnterStructPointerForCoercedAccess - Given a struct pointer that we are
1239	/// accessing some number of bytes out of it, try to gep into the struct to get
1240	/// at its inner goodness. Dive as deep as possible without entering an element
1241	/// with an in-memory size smaller than DstSize.
1242	static Address EnterStructPointerForCoercedAccess(Address SrcPtr,
1243	llvm::StructType *SrcSTy,
1244	uint64_t DstSize,
1245	CodeGenFunction &CGF) {
1246	// We can't dive into a zero-element struct.
1247	if (SrcSTy->getNumElements() == `0`)
1248	return SrcPtr;
1249
1250	llvm::Type *FirstElt = SrcSTy->getElementType(N: `0`);
1251
1252	// If the first elt is at least as large as what we're looking for, or if the
1253	// first element is the same size as the whole struct, we can enter it. The
1254	// comparison must be made on the store size and not the alloca size. Using
1255	// the alloca size may overstate the size of the load.
1256	uint64_t FirstEltSize = CGF.CGM.getDataLayout().getTypeStoreSize(Ty: FirstElt);
1257	if (FirstEltSize < DstSize &&
1258	FirstEltSize < CGF.CGM.getDataLayout().getTypeStoreSize(Ty: SrcSTy))
1259	return SrcPtr;
1260
1261	// GEP into the first element.
1262	SrcPtr = CGF.Builder.CreateStructGEP(Addr: SrcPtr, Index: `0`, Name: "coerce.dive");
1263
1264	// If the first element is a struct, recurse.
1265	llvm::Type *SrcTy = SrcPtr.getElementType();
1266	if (llvm::StructType *SrcSTy = dyn_cast<llvm::StructType>(Val: SrcTy))
1267	return EnterStructPointerForCoercedAccess(SrcPtr, SrcSTy, DstSize, CGF);
1268
1269	return SrcPtr;
1270	}
1271
1272	/// CoerceIntOrPtrToIntOrPtr - Convert a value Val to the specific Ty where both
1273	/// are either integers or pointers. This does a truncation of the value if it
1274	/// is too large or a zero extension if it is too small.
1275	///
1276	/// This behaves as if the value were coerced through memory, so on big-endian
1277	/// targets the high bits are preserved in a truncation, while little-endian
1278	/// targets preserve the low bits.
1279	static llvm::Value CoerceIntOrPtrToIntOrPtr(llvm::Value Val, llvm::Type *Ty,
1280	CodeGenFunction &CGF) {
1281	if (Val->getType() == Ty)
1282	return Val;
1283
1284	if (isa<llvm::PointerType>(Val: Val->getType())) {
1285	// If this is Pointer->Pointer avoid conversion to and from int.
1286	if (isa<llvm::PointerType>(Val: Ty))
1287	return CGF.Builder.CreateBitCast(V: Val, DestTy: Ty, Name: "coerce.val");
1288
1289	// Convert the pointer to an integer so we can play with its width.
1290	Val = CGF.Builder.CreatePtrToInt(V: Val, DestTy: CGF.IntPtrTy, Name: "coerce.val.pi");
1291	}
1292
1293	llvm::Type *DestIntTy = Ty;
1294	if (isa<llvm::PointerType>(Val: DestIntTy))
1295	DestIntTy = CGF.IntPtrTy;
1296
1297	if (Val->getType() != DestIntTy) {
1298	const llvm::DataLayout &DL = CGF.CGM.getDataLayout();
1299	if (DL.isBigEndian()) {
1300	// Preserve the high bits on big-endian targets.
1301	// That is what memory coercion does.
1302	uint64_t SrcSize = DL.getTypeSizeInBits(Ty: Val->getType());
1303	uint64_t DstSize = DL.getTypeSizeInBits(Ty: DestIntTy);
1304
1305	if (SrcSize > DstSize) {
1306	Val = CGF.Builder.CreateLShr(LHS: Val, RHS: SrcSize - DstSize, Name: "coerce.highbits");
1307	Val = CGF.Builder.CreateTrunc(V: Val, DestTy: DestIntTy, Name: "coerce.val.ii");
1308	} else {
1309	Val = CGF.Builder.CreateZExt(V: Val, DestTy: DestIntTy, Name: "coerce.val.ii");
1310	Val = CGF.Builder.CreateShl(LHS: Val, RHS: DstSize - SrcSize, Name: "coerce.highbits");
1311	}
1312	} else {
1313	// Little-endian targets preserve the low bits. No shifts required.
1314	Val = CGF.Builder.CreateIntCast(V: Val, DestTy: DestIntTy, isSigned: false, Name: "coerce.val.ii");
1315	}
1316	}
1317
1318	if (isa<llvm::PointerType>(Val: Ty))
1319	Val = CGF.Builder.CreateIntToPtr(V: Val, DestTy: Ty, Name: "coerce.val.ip");
1320	return Val;
1321	}
1322
1323	/// CreateCoercedLoad - Create a load from \arg SrcPtr interpreted as
1324	/// a pointer to an object of type \arg Ty, known to be aligned to
1325	/// \arg SrcAlign bytes.
1326	///
1327	/// This safely handles the case when the src type is smaller than the
1328	/// destination type; in this situation the values of bits which not
1329	/// present in the src are undefined.
1330	static llvm::Value CreateCoercedLoad(Address Src, llvm::Type Ty,
1331	CodeGenFunction &CGF) {
1332	llvm::Type *SrcTy = Src.getElementType();
1333
1334	// If SrcTy and Ty are the same, just do a load.
1335	if (SrcTy == Ty)
1336	return CGF.Builder.CreateLoad(Addr: Src);
1337
1338	llvm::TypeSize DstSize = CGF.CGM.getDataLayout().getTypeAllocSize(Ty);
1339
1340	if (llvm::StructType *SrcSTy = dyn_cast<llvm::StructType>(Val: SrcTy)) {
1341	Src = EnterStructPointerForCoercedAccess(SrcPtr: Src, SrcSTy,
1342	DstSize: DstSize.getFixedValue(), CGF);
1343	SrcTy = Src.getElementType();
1344	}
1345
1346	llvm::TypeSize SrcSize = CGF.CGM.getDataLayout().getTypeAllocSize(Ty: SrcTy);
1347
1348	// If the source and destination are integer or pointer types, just do an
1349	// extension or truncation to the desired type.
1350	if ((isa<llvm::IntegerType>(Val: Ty) \|\| isa<llvm::PointerType>(Val: Ty)) &&
1351	(isa<llvm::IntegerType>(Val: SrcTy) \|\| isa<llvm::PointerType>(Val: SrcTy))) {
1352	llvm::Value *Load = CGF.Builder.CreateLoad(Addr: Src);
1353	return CoerceIntOrPtrToIntOrPtr(Val: Load, Ty, CGF);
1354	}
1355
1356	// If load is legal, just bitcast the src pointer.
1357	if (!SrcSize.isScalable() && !DstSize.isScalable() &&
1358	SrcSize.getFixedValue() >= DstSize.getFixedValue()) {
1359	// Generally SrcSize is never greater than DstSize, since this means we are
1360	// losing bits. However, this can happen in cases where the structure has
1361	// additional padding, for example due to a user specified alignment.
1362	//
1363	// FIXME: Assert that we aren't truncating non-padding bits when have access
1364	// to that information.
1365	Src = Src.withElementType(ElemTy: Ty);
1366	return CGF.Builder.CreateLoad(Addr: Src);
1367	}
1368
1369	// If coercing a fixed vector to a scalable vector for ABI compatibility, and
1370	// the types match, use the llvm.vector.insert intrinsic to perform the
1371	// conversion.
1372	if (auto *ScalableDstTy = dyn_cast<llvm::ScalableVectorType>(Val: Ty)) {
1373	if (auto *FixedSrcTy = dyn_cast<llvm::FixedVectorType>(Val: SrcTy)) {
1374	// If we are casting a fixed i8 vector to a scalable i1 predicate
1375	// vector, use a vector insert and bitcast the result.
1376	if (ScalableDstTy->getElementType()->isIntegerTy(Bitwidth: `1`) &&
1377	FixedSrcTy->getElementType()->isIntegerTy(Bitwidth: `8`)) {
1378	ScalableDstTy = llvm::ScalableVectorType::get(
1379	ElementType: FixedSrcTy->getElementType(),
1380	MinNumElts: llvm::divideCeil(
1381	Numerator: ScalableDstTy->getElementCount().getKnownMinValue(), Denominator: `8`));
1382	}
1383	if (ScalableDstTy->getElementType() == FixedSrcTy->getElementType()) {
1384	auto *Load = CGF.Builder.CreateLoad(Addr: Src);
1385	auto *PoisonVec = llvm::PoisonValue::get(T: ScalableDstTy);
1386	llvm::Value *Result = CGF.Builder.CreateInsertVector(
1387	DstType: ScalableDstTy, SrcVec: PoisonVec, SubVec: Load, Idx: uint64_t(`0`), Name: "cast.scalable");
1388	ScalableDstTy = cast<llvm::ScalableVectorType>(
1389	Val: llvm::VectorType::getWithSizeAndScalar(SizeTy: ScalableDstTy, EltTy: Ty));
1390	if (Result->getType() != ScalableDstTy)
1391	Result = CGF.Builder.CreateBitCast(V: Result, DestTy: ScalableDstTy);
1392	if (Result->getType() != Ty)
1393	Result = CGF.Builder.CreateExtractVector(DstType: Ty, SrcVec: Result, Idx: uint64_t(`0`));
1394	return Result;
1395	}
1396	}
1397	}
1398
1399	// Otherwise do coercion through memory. This is stupid, but simple.
1400	RawAddress Tmp =
1401	CreateTempAllocaForCoercion(CGF, Ty, MinAlign: Src.getAlignment(), Name: Src.getName());
1402	CGF.Builder.CreateMemCpy(
1403	Dst: Tmp.getPointer(), DstAlign: Tmp.getAlignment().getAsAlign(),
1404	Src: Src.emitRawPointer(CGF), SrcAlign: Src.getAlignment().getAsAlign(),
1405	Size: llvm::ConstantInt::get(Ty: CGF.IntPtrTy, V: SrcSize.getKnownMinValue()));
1406	return CGF.Builder.CreateLoad(Addr: Tmp);
1407	}
1408
1409	void CodeGenFunction::CreateCoercedStore(llvm::Value *Src, Address Dst,
1410	llvm::TypeSize DstSize,
1411	bool DstIsVolatile) {
1412	if (!DstSize)
1413	return;
1414
1415	llvm::Type *SrcTy = Src->getType();
1416	llvm::TypeSize SrcSize = CGM.getDataLayout().getTypeAllocSize(Ty: SrcTy);
1417
1418	// GEP into structs to try to make types match.
1419	// FIXME: This isn't really that useful with opaque types, but it impacts a
1420	// lot of regression tests.
1421	if (SrcTy != Dst.getElementType()) {
1422	if (llvm::StructType *DstSTy =
1423	dyn_cast<llvm::StructType>(Val: Dst.getElementType())) {
1424	assert(!SrcSize.isScalable());
1425	Dst = EnterStructPointerForCoercedAccess(SrcPtr: Dst, SrcSTy: DstSTy,
1426	DstSize: SrcSize.getFixedValue(), CGF&: *this);
1427	}
1428	}
1429
1430	if (SrcSize.isScalable() \|\| SrcSize <= DstSize) {
1431	if (SrcTy->isIntegerTy() && Dst.getElementType()->isPointerTy() &&
1432	SrcSize == CGM.getDataLayout().getTypeAllocSize(Ty: Dst.getElementType())) {
1433	// If the value is supposed to be a pointer, convert it before storing it.
1434	Src = CoerceIntOrPtrToIntOrPtr(Val: Src, Ty: Dst.getElementType(), CGF&: *this);
1435	auto *I = Builder.CreateStore(Val: Src, Addr: Dst, IsVolatile: DstIsVolatile);
1436	addInstToCurrentSourceAtom(KeyInstruction: I, Backup: Src);
1437	} else if (llvm::StructType *STy =
1438	dyn_cast<llvm::StructType>(Val: Src->getType())) {
1439	// Prefer scalar stores to first-class aggregate stores.
1440	Dst = Dst.withElementType(ElemTy: SrcTy);
1441	for (unsigned i = `0`, e = STy->getNumElements(); i != e; ++i) {
1442	Address EltPtr = Builder.CreateStructGEP(Addr: Dst, Index: i);
1443	llvm::Value *Elt = Builder.CreateExtractValue(Agg: Src, Idxs: i);
1444	auto *I = Builder.CreateStore(Val: Elt, Addr: EltPtr, IsVolatile: DstIsVolatile);
1445	addInstToCurrentSourceAtom(KeyInstruction: I, Backup: Elt);
1446	}
1447	} else {
1448	auto *I =
1449	Builder.CreateStore(Val: Src, Addr: Dst.withElementType(ElemTy: SrcTy), IsVolatile: DstIsVolatile);
1450	addInstToCurrentSourceAtom(KeyInstruction: I, Backup: Src);
1451	}
1452	} else if (SrcTy->isIntegerTy()) {
1453	// If the source is a simple integer, coerce it directly.
1454	llvm::Type DstIntTy = Builder.getIntNTy(N: DstSize.getFixedValue() `8`);
1455	Src = CoerceIntOrPtrToIntOrPtr(Val: Src, Ty: DstIntTy, CGF&: *this);
1456	auto *I =
1457	Builder.CreateStore(Val: Src, Addr: Dst.withElementType(ElemTy: DstIntTy), IsVolatile: DstIsVolatile);
1458	addInstToCurrentSourceAtom(KeyInstruction: I, Backup: Src);
1459	} else {
1460	// Otherwise do coercion through memory. This is stupid, but
1461	// simple.
1462
1463	// Generally SrcSize is never greater than DstSize, since this means we are
1464	// losing bits. However, this can happen in cases where the structure has
1465	// additional padding, for example due to a user specified alignment.
1466	//
1467	// FIXME: Assert that we aren't truncating non-padding bits when have access
1468	// to that information.
1469	RawAddress Tmp =
1470	CreateTempAllocaForCoercion(CGF&: *this, Ty: SrcTy, MinAlign: Dst.getAlignment());
1471	Builder.CreateStore(Val: Src, Addr: Tmp);
1472	auto *I = Builder.CreateMemCpy(
1473	Dst: Dst.emitRawPointer(CGF&: *this), DstAlign: Dst.getAlignment().getAsAlign(),
1474	Src: Tmp.getPointer(), SrcAlign: Tmp.getAlignment().getAsAlign(),
1475	Size: Builder.CreateTypeSize(Ty: IntPtrTy, Size: DstSize));
1476	addInstToCurrentSourceAtom(KeyInstruction: I, Backup: Src);
1477	}
1478	}
1479
1480	static Address emitAddressAtOffset(CodeGenFunction &CGF, Address addr,
1481	const ABIArgInfo &info) {
1482	if (unsigned offset = info.getDirectOffset()) {
1483	addr = addr.withElementType(ElemTy: CGF.Int8Ty);
1484	addr = CGF.Builder.CreateConstInBoundsByteGEP(
1485	Addr: addr, Offset: CharUnits::fromQuantity(Quantity: offset));
1486	addr = addr.withElementType(ElemTy: info.getCoerceToType());
1487	}
1488	return addr;
1489	}
1490
1491	static std::pair<llvm::Value , bool*>
1492	CoerceScalableToFixed(CodeGenFunction &CGF, llvm::FixedVectorType *ToTy,
1493	llvm::ScalableVectorType FromTy, llvm::Value V,
1494	StringRef Name = "") {
1495	// If we are casting a scalable i1 predicate vector to a fixed i8
1496	// vector, first bitcast the source.
1497	if (FromTy->getElementType()->isIntegerTy(Bitwidth: `1`) &&
1498	ToTy->getElementType() == CGF.Builder.getInt8Ty()) {
1499	if (!FromTy->getElementCount().isKnownMultipleOf(RHS: `8`)) {
1500	FromTy = llvm::ScalableVectorType::get(
1501	ElementType: FromTy->getElementType(),
1502	MinNumElts: llvm::alignTo<`8`>(Value: FromTy->getElementCount().getKnownMinValue()));
1503	llvm::Value *ZeroVec = llvm::Constant::getNullValue(Ty: FromTy);
1504	V = CGF.Builder.CreateInsertVector(DstType: FromTy, SrcVec: ZeroVec, SubVec: V, Idx: uint64_t(`0`));
1505	}
1506	FromTy = llvm::ScalableVectorType::get(
1507	ElementType: ToTy->getElementType(),
1508	MinNumElts: FromTy->getElementCount().getKnownMinValue() / `8`);
1509	V = CGF.Builder.CreateBitCast(V, DestTy: FromTy);
1510	}
1511	if (FromTy->getElementType() == ToTy->getElementType()) {
1512	V->setName(Name + ".coerce");
1513	V = CGF.Builder.CreateExtractVector(DstType: ToTy, SrcVec: V, Idx: uint64_t(`0`), Name: "cast.fixed");
1514	return {V, true};
1515	}
1516	return {V, false};
1517	}
1518
1519	namespace {
1520
1521	/// Encapsulates information about the way function arguments from
1522	/// CGFunctionInfo should be passed to actual LLVM IR function.
1523	class ClangToLLVMArgMapping {
1524	static const unsigned InvalidIndex = ~`0U`;
1525	unsigned InallocaArgNo;
1526	unsigned SRetArgNo;
1527	unsigned TotalIRArgs;
1528
1529	/// Arguments of LLVM IR function corresponding to single Clang argument.
1530	struct IRArgs {
1531	unsigned PaddingArgIndex;
1532	// Argument is expanded to IR arguments at positions
1533	// [FirstArgIndex, FirstArgIndex + NumberOfArgs).
1534	unsigned FirstArgIndex;
1535	unsigned NumberOfArgs;
1536
1537	IRArgs()
1538	: PaddingArgIndex(InvalidIndex), FirstArgIndex(InvalidIndex),
1539	NumberOfArgs(`0`) {}
1540	};
1541
1542	SmallVector<IRArgs, `8`> ArgInfo;
1543
1544	public:
1545	ClangToLLVMArgMapping(const ASTContext &Context, const CGFunctionInfo &FI,
1546	bool OnlyRequiredArgs = false)
1547	: InallocaArgNo(InvalidIndex), SRetArgNo(InvalidIndex), TotalIRArgs(`0`),
1548	ArgInfo (OnlyRequiredArgs ? FI.getNumRequiredArgs() : FI.arg_size()) {
1549	construct(Context, FI, OnlyRequiredArgs);
1550	}
1551
1552	bool hasInallocaArg() const { return InallocaArgNo != InvalidIndex; }
1553	unsigned getInallocaArgNo() const {
1554	assert(hasInallocaArg());
1555	return InallocaArgNo;
1556	}
1557
1558	bool hasSRetArg() const { return SRetArgNo != InvalidIndex; }
1559	unsigned getSRetArgNo() const {
1560	assert(hasSRetArg());
1561	return SRetArgNo;
1562	}
1563
1564	unsigned totalIRArgs() const { return TotalIRArgs; }
1565
1566	bool hasPaddingArg(unsigned ArgNo) const {
1567	assert(ArgNo < ArgInfo.size());
1568	return ArgInfo [ArgNo].PaddingArgIndex != InvalidIndex;
1569	}
1570	unsigned getPaddingArgNo(unsigned ArgNo) const {
1571	assert(hasPaddingArg(ArgNo));
1572	return ArgInfo [ArgNo].PaddingArgIndex;
1573	}
1574
1575	/// Returns index of first IR argument corresponding to ArgNo, and their
1576	/// quantity.
1577	std::pair<unsigned, unsigned> getIRArgs(unsigned ArgNo) const {
1578	assert(ArgNo < ArgInfo.size());
1579	return std::make_pair(x: ArgInfo [ArgNo].FirstArgIndex,
1580	y: ArgInfo [ArgNo].NumberOfArgs);
1581	}
1582
1583	private:
1584	void construct(const ASTContext &Context, const CGFunctionInfo &FI,
1585	bool OnlyRequiredArgs);
1586	};
1587
1588	void ClangToLLVMArgMapping::construct(const ASTContext &Context,
1589	const CGFunctionInfo &FI,
1590	bool OnlyRequiredArgs) {
1591	unsigned IRArgNo = `0`;
1592	bool SwapThisWithSRet = false;
1593	const ABIArgInfo &RetAI = FI.getReturnInfo();
1594
1595	if (RetAI.getKind() == ABIArgInfo::Indirect) {
1596	SwapThisWithSRet = RetAI.isSRetAfterThis();
1597	SRetArgNo = SwapThisWithSRet ? `1` : IRArgNo++;
1598	}
1599
1600	unsigned ArgNo = `0`;
1601	unsigned NumArgs = OnlyRequiredArgs ? FI.getNumRequiredArgs() : FI.arg_size();
1602	for (CGFunctionInfo::const_arg_iterator I = FI.arg_begin(); ArgNo < NumArgs;
1603	++I, ++ArgNo) {
1604	assert(I != FI.arg_end());
1605	QualType ArgType = I->type;
1606	const ABIArgInfo &AI = I->info;
1607	// Collect data about IR arguments corresponding to Clang argument ArgNo.
1608	auto &IRArgs = ArgInfo [ArgNo];
1609
1610	if (AI.getPaddingType())
1611	IRArgs.PaddingArgIndex = IRArgNo++;
1612
1613	switch (AI.getKind()) {
1614	case ABIArgInfo::Extend:
1615	case ABIArgInfo::Direct: {
1616	// FIXME: handle sseregparm someday...
1617	llvm::StructType *STy = dyn_cast<llvm::StructType>(Val: AI.getCoerceToType());
1618	if (AI.isDirect() && AI.getCanBeFlattened() && STy) {
1619	IRArgs.NumberOfArgs = STy->getNumElements();
1620	} else {
1621	IRArgs.NumberOfArgs = `1`;
1622	}
1623	break;
1624	}
1625	case ABIArgInfo::Indirect:
1626	case ABIArgInfo::IndirectAliased:
1627	IRArgs.NumberOfArgs = `1`;
1628	break;
1629	case ABIArgInfo::Ignore:
1630	case ABIArgInfo::InAlloca:
1631	// ignore and inalloca doesn't have matching LLVM parameters.
1632	IRArgs.NumberOfArgs = `0`;
1633	break;
1634	case ABIArgInfo::CoerceAndExpand:
1635	IRArgs.NumberOfArgs = AI.getCoerceAndExpandTypeSequence().size();
1636	break;
1637	case ABIArgInfo::Expand:
1638	IRArgs.NumberOfArgs = getExpansionSize(Ty: ArgType, Context);
1639	break;
1640	}
1641
1642	if (IRArgs.NumberOfArgs > `0`) {
1643	IRArgs.FirstArgIndex = IRArgNo;
1644	IRArgNo += IRArgs.NumberOfArgs;
1645	}
1646
1647	// Skip over the sret parameter when it comes second. We already handled it
1648	// above.
1649	if (IRArgNo == `1` && SwapThisWithSRet)
1650	IRArgNo++;
1651	}
1652	assert(ArgNo == ArgInfo.size());
1653
1654	if (FI.usesInAlloca())
1655	InallocaArgNo = IRArgNo++;
1656
1657	TotalIRArgs = IRArgNo;
1658	}
1659	} // namespace
1660
1661	/*/
1662
1663	bool CodeGenModule::ReturnTypeUsesSRet(const CGFunctionInfo &FI) {
1664	const auto &RI = FI.getReturnInfo();
1665	return RI.isIndirect() \|\| (RI.isInAlloca() && RI.getInAllocaSRet());
1666	}
1667
1668	bool CodeGenModule::ReturnTypeHasInReg(const CGFunctionInfo &FI) {
1669	const auto &RI = FI.getReturnInfo();
1670	return RI.getInReg();
1671	}
1672
1673	bool CodeGenModule::ReturnSlotInterferesWithArgs(const CGFunctionInfo &FI) {
1674	return ReturnTypeUsesSRet(FI) &&
1675	getTargetCodeGenInfo().doesReturnSlotInterfereWithArgs();
1676	}
1677
1678	bool CodeGenModule::ReturnTypeUsesFPRet(QualType ResultType) {
1679	if (const BuiltinType *BT = ResultType ->getAs<BuiltinType>()) {
1680	switch (BT->getKind()) {
1681	default:
1682	return false;
1683	case BuiltinType::Float:
1684	return getTarget().useObjCFPRetForRealType(T: FloatModeKind::Float);
1685	case BuiltinType::Double:
1686	return getTarget().useObjCFPRetForRealType(T: FloatModeKind::Double);
1687	case BuiltinType::LongDouble:
1688	return getTarget().useObjCFPRetForRealType(T: FloatModeKind::LongDouble);
1689	}
1690	}
1691
1692	return false;
1693	}
1694
1695	bool CodeGenModule::ReturnTypeUsesFP2Ret(QualType ResultType) {
1696	if (const ComplexType *CT = ResultType ->getAs<ComplexType>()) {
1697	if (const BuiltinType *BT = CT->getElementType()->getAs<BuiltinType>()) {
1698	if (BT->getKind() == BuiltinType::LongDouble)
1699	return getTarget().useObjCFP2RetForComplexLongDouble();
1700	}
1701	}
1702
1703	return false;
1704	}
1705
1706	llvm::FunctionType *CodeGenTypes::GetFunctionType(GlobalDecl GD) {
1707	const CGFunctionInfo &FI = arrangeGlobalDeclaration(GD);
1708	return GetFunctionType(Info: FI);
1709	}
1710
1711	llvm::FunctionType CodeGenTypes::GetFunctionType(const* CGFunctionInfo &FI) {
1712
1713	bool Inserted = FunctionsBeingProcessed.insert(Ptr: &FI).second;
1714	(void)Inserted;
1715	assert(Inserted && "Recursively being processed?");
1716
1717	llvm::Type resultType = nullptr*;
1718	const ABIArgInfo &retAI = FI.getReturnInfo();
1719	switch (retAI.getKind()) {
1720	case ABIArgInfo::Expand:
1721	case ABIArgInfo::IndirectAliased:
1722	llvm_unreachable("Invalid ABI kind for return argument");
1723
1724	case ABIArgInfo::Extend:
1725	case ABIArgInfo::Direct:
1726	resultType = retAI.getCoerceToType();
1727	break;
1728
1729	case ABIArgInfo::InAlloca:
1730	if (retAI.getInAllocaSRet()) {
1731	// sret things on win32 aren't void, they return the sret pointer.
1732	QualType ret = FI.getReturnType();
1733	unsigned addressSpace = CGM.getTypes().getTargetAddressSpace(T: ret);
1734	resultType = llvm::PointerType::get(C&: getLLVMContext(), AddressSpace: addressSpace);
1735	} else {
1736	resultType = llvm::Type::getVoidTy(C&: getLLVMContext());
1737	}
1738	break;
1739
1740	case ABIArgInfo::Indirect:
1741	case ABIArgInfo::Ignore:
1742	resultType = llvm::Type::getVoidTy(C&: getLLVMContext());
1743	break;
1744
1745	case ABIArgInfo::CoerceAndExpand:
1746	resultType = retAI.getUnpaddedCoerceAndExpandType();
1747	break;
1748	}
1749
1750	ClangToLLVMArgMapping IRFunctionArgs(getContext(), FI, true);
1751	SmallVector<llvm::Type *, `8`> ArgTypes(IRFunctionArgs.totalIRArgs());
1752
1753	// Add type for sret argument.
1754	if (IRFunctionArgs.hasSRetArg()) {
1755	ArgTypes [IRFunctionArgs.getSRetArgNo()] = llvm::PointerType::get(
1756	C&: getLLVMContext(), AddressSpace: FI.getReturnInfo().getIndirectAddrSpace());
1757	}
1758
1759	// Add type for inalloca argument.
1760	if (IRFunctionArgs.hasInallocaArg())
1761	ArgTypes [IRFunctionArgs.getInallocaArgNo()] =
1762	llvm::PointerType::getUnqual(C&: getLLVMContext());
1763
1764	// Add in all of the required arguments.
1765	unsigned ArgNo = `0`;
1766	CGFunctionInfo::const_arg_iterator it = FI.arg_begin(),
1767	ie = it + FI.getNumRequiredArgs();
1768	for (; it != ie; ++it, ++ArgNo) {
1769	const ABIArgInfo &ArgInfo = it->info;
1770
1771	// Insert a padding type to ensure proper alignment.
1772	if (IRFunctionArgs.hasPaddingArg(ArgNo))
1773	ArgTypes [IRFunctionArgs.getPaddingArgNo(ArgNo)] =
1774	ArgInfo.getPaddingType();
1775
1776	unsigned FirstIRArg, NumIRArgs;
1777	std::tie(args&: FirstIRArg, args&: NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
1778
1779	switch (ArgInfo.getKind()) {
1780	case ABIArgInfo::Ignore:
1781	case ABIArgInfo::InAlloca:
1782	assert(NumIRArgs == `0`);
1783	break;
1784
1785	case ABIArgInfo::Indirect:
1786	assert(NumIRArgs == `1`);
1787	// indirect arguments are always on the stack, which is alloca addr space.
1788	ArgTypes [FirstIRArg] = llvm::PointerType::get(
1789	C&: getLLVMContext(), AddressSpace: CGM.getDataLayout().getAllocaAddrSpace());
1790	break;
1791	case ABIArgInfo::IndirectAliased:
1792	assert(NumIRArgs == `1`);
1793	ArgTypes [FirstIRArg] = llvm::PointerType::get(
1794	C&: getLLVMContext(), AddressSpace: ArgInfo.getIndirectAddrSpace());
1795	break;
1796	case ABIArgInfo::Extend:
1797	case ABIArgInfo::Direct: {
1798	// Fast-isel and the optimizer generally like scalar values better than
1799	// FCAs, so we flatten them if this is safe to do for this argument.
1800	llvm::Type *argType = ArgInfo.getCoerceToType();
1801	llvm::StructType *st = dyn_cast<llvm::StructType>(Val: argType);
1802	if (st && ArgInfo.isDirect() && ArgInfo.getCanBeFlattened()) {
1803	assert(NumIRArgs == st->getNumElements());
1804	for (unsigned i = `0`, e = st->getNumElements(); i != e; ++i)
1805	ArgTypes [FirstIRArg + i] = st->getElementType(N: i);
1806	} else {
1807	assert(NumIRArgs == `1`);
1808	ArgTypes [FirstIRArg] = argType;
1809	}
1810	break;
1811	}
1812
1813	case ABIArgInfo::CoerceAndExpand: {
1814	auto ArgTypesIter = ArgTypes.begin() + FirstIRArg;
1815	for (auto *EltTy : ArgInfo.getCoerceAndExpandTypeSequence()) {
1816	*ArgTypesIter++ = EltTy;
1817	}
1818	assert(ArgTypesIter == ArgTypes.begin() + FirstIRArg + NumIRArgs);
1819	break;
1820	}
1821
1822	case ABIArgInfo::Expand:
1823	auto ArgTypesIter = ArgTypes.begin() + FirstIRArg;
1824	getExpandedTypes(Ty: it->type, TI&: ArgTypesIter);
1825	assert(ArgTypesIter == ArgTypes.begin() + FirstIRArg + NumIRArgs);
1826	break;
1827	}
1828	}
1829
1830	bool Erased = FunctionsBeingProcessed.erase(Ptr: &FI);
1831	(void)Erased;
1832	assert(Erased && "Not in set?");
1833
1834	return llvm::FunctionType::get(Result: resultType, Params: ArgTypes, isVarArg: FI.isVariadic());
1835	}
1836
1837	llvm::Type *CodeGenTypes::GetFunctionTypeForVTable(GlobalDecl GD) {
1838	const CXXMethodDecl *MD = cast<CXXMethodDecl>(Val: GD.getDecl());
1839	const FunctionProtoType *FPT = MD->getType()->castAs<FunctionProtoType>();
1840
1841	if (!isFuncTypeConvertible(FPT))
1842	return llvm::StructType::get(Context&: getLLVMContext());
1843
1844	return GetFunctionType(GD);
1845	}
1846
1847	static void AddAttributesFromFunctionProtoType(ASTContext &Ctx,
1848	llvm::AttrBuilder &FuncAttrs,
1849	const FunctionProtoType *FPT) {
1850	if (!FPT)
1851	return;
1852
1853	if (!isUnresolvedExceptionSpec(FPT->getExceptionSpecType()) &&
1854	FPT->isNothrow())
1855	FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
1856
1857	unsigned SMEBits = FPT->getAArch64SMEAttributes();
1858	if (SMEBits & FunctionType::SME_PStateSMEnabledMask)
1859	FuncAttrs.addAttribute(A: "aarch64_pstate_sm_enabled");
1860	if (SMEBits & FunctionType::SME_PStateSMCompatibleMask)
1861	FuncAttrs.addAttribute(A: "aarch64_pstate_sm_compatible");
1862	if (SMEBits & FunctionType::SME_AgnosticZAStateMask)
1863	FuncAttrs.addAttribute(A: "aarch64_za_state_agnostic");
1864
1865	// ZA
1866	if (FunctionType::getArmZAState(AttrBits: SMEBits) == FunctionType::ARM_Preserves)
1867	FuncAttrs.addAttribute(A: "aarch64_preserves_za");
1868	if (FunctionType::getArmZAState(AttrBits: SMEBits) == FunctionType::ARM_In)
1869	FuncAttrs.addAttribute(A: "aarch64_in_za");
1870	if (FunctionType::getArmZAState(AttrBits: SMEBits) == FunctionType::ARM_Out)
1871	FuncAttrs.addAttribute(A: "aarch64_out_za");
1872	if (FunctionType::getArmZAState(AttrBits: SMEBits) == FunctionType::ARM_InOut)
1873	FuncAttrs.addAttribute(A: "aarch64_inout_za");
1874
1875	// ZT0
1876	if (FunctionType::getArmZT0State(AttrBits: SMEBits) == FunctionType::ARM_Preserves)
1877	FuncAttrs.addAttribute(A: "aarch64_preserves_zt0");
1878	if (FunctionType::getArmZT0State(AttrBits: SMEBits) == FunctionType::ARM_In)
1879	FuncAttrs.addAttribute(A: "aarch64_in_zt0");
1880	if (FunctionType::getArmZT0State(AttrBits: SMEBits) == FunctionType::ARM_Out)
1881	FuncAttrs.addAttribute(A: "aarch64_out_zt0");
1882	if (FunctionType::getArmZT0State(AttrBits: SMEBits) == FunctionType::ARM_InOut)
1883	FuncAttrs.addAttribute(A: "aarch64_inout_zt0");
1884	}
1885
1886	static void AddAttributesFromOMPAssumes(llvm::AttrBuilder &FuncAttrs,
1887	const Decl *Callee) {
1888	if (!Callee)
1889	return;
1890
1891	SmallVector<StringRef, `4`> Attrs;
1892
1893	for (const OMPAssumeAttr *AA : Callee->specific_attrs<OMPAssumeAttr>())
1894	AA->getAssumption().split(Attrs, ",");
1895
1896	if (!Attrs.empty())
1897	FuncAttrs.addAttribute(A: llvm::AssumptionAttrKey,
1898	V: llvm::join(Begin: Attrs.begin(), End: Attrs.end(), Separator: ","));
1899	}
1900
1901	bool CodeGenModule::MayDropFunctionReturn(const ASTContext &Context,
1902	QualType ReturnType) const {
1903	// We can't just discard the return value for a record type with a
1904	// complex destructor or a non-trivially copyable type.
1905	if (const RecordType *RT =
1906	ReturnType.getCanonicalType()->getAs<RecordType>()) {
1907	if (const auto *ClassDecl = dyn_cast<CXXRecordDecl>(Val: RT->getDecl()))
1908	return ClassDecl->hasTrivialDestructor();
1909	}
1910	return ReturnType.isTriviallyCopyableType(Context);
1911	}
1912
1913	static bool HasStrictReturn(const CodeGenModule &Module, QualType RetTy,
1914	const Decl *TargetDecl) {
1915	// As-is msan can not tolerate noundef mismatch between caller and
1916	// implementation. Mismatch is possible for e.g. indirect calls from C-caller
1917	// into C++. Such mismatches lead to confusing false reports. To avoid
1918	// expensive workaround on msan we enforce initialization event in uncommon
1919	// cases where it's allowed.
1920	if (Module.getLangOpts().Sanitize.has(K: SanitizerKind::Memory))
1921	return true;
1922	// C++ explicitly makes returning undefined values UB. C's rule only applies
1923	// to used values, so we never mark them noundef for now.
1924	if (!Module.getLangOpts().CPlusPlus)
1925	return false;
1926	if (TargetDecl) {
1927	if (const FunctionDecl *FDecl = dyn_cast<FunctionDecl>(Val: TargetDecl)) {
1928	if (FDecl->isExternC())
1929	return false;
1930	} else if (const VarDecl *VDecl = dyn_cast<VarDecl>(Val: TargetDecl)) {
1931	// Function pointer.
1932	if (VDecl->isExternC())
1933	return false;
1934	}
1935	}
1936
1937	// We don't want to be too aggressive with the return checking, unless
1938	// it's explicit in the code opts or we're using an appropriate sanitizer.
1939	// Try to respect what the programmer intended.
1940	return Module.getCodeGenOpts().StrictReturn \|\|
1941	!Module.MayDropFunctionReturn(Context: Module.getContext(), ReturnType: RetTy) \|\|
1942	Module.getLangOpts().Sanitize.has(K: SanitizerKind::Return);
1943	}
1944
1945	/// Add denormal-fp-math and denormal-fp-math-f32 as appropriate for the
1946	/// requested denormal behavior, accounting for the overriding behavior of the
1947	/// -f32 case.
1948	static void addDenormalModeAttrs(llvm::DenormalMode FPDenormalMode,
1949	llvm::DenormalMode FP32DenormalMode,
1950	llvm::AttrBuilder &FuncAttrs) {
1951	if (FPDenormalMode != llvm::DenormalMode::getDefault())
1952	FuncAttrs.addAttribute(A: "denormal-fp-math", V: FPDenormalMode.str());
1953
1954	if (FP32DenormalMode != FPDenormalMode && FP32DenormalMode.isValid())
1955	FuncAttrs.addAttribute(A: "denormal-fp-math-f32", V: FP32DenormalMode.str());
1956	}
1957
1958	/// Add default attributes to a function, which have merge semantics under
1959	/// -mlink-builtin-bitcode and should not simply overwrite any existing
1960	/// attributes in the linked library.
1961	static void
1962	addMergableDefaultFunctionAttributes(const CodeGenOptions &CodeGenOpts,
1963	llvm::AttrBuilder &FuncAttrs) {
1964	addDenormalModeAttrs(FPDenormalMode: CodeGenOpts.FPDenormalMode, FP32DenormalMode: CodeGenOpts.FP32DenormalMode,
1965	FuncAttrs);
1966	}
1967
1968	static void getTrivialDefaultFunctionAttributes(
1969	StringRef Name, bool HasOptnone, const CodeGenOptions &CodeGenOpts,
1970	const LangOptions &LangOpts, bool AttrOnCallSite,
1971	llvm::AttrBuilder &FuncAttrs) {
1972	// OptimizeNoneAttr takes precedence over -Os or -Oz. No warning needed.
1973	if (!HasOptnone) {
1974	if (CodeGenOpts.OptimizeSize)
1975	FuncAttrs.addAttribute(llvm::Attribute::OptimizeForSize);
1976	if (CodeGenOpts.OptimizeSize == `2`)
1977	FuncAttrs.addAttribute(llvm::Attribute::MinSize);
1978	}
1979
1980	if (CodeGenOpts.DisableRedZone)
1981	FuncAttrs.addAttribute(llvm::Attribute::NoRedZone);
1982	if (CodeGenOpts.IndirectTlsSegRefs)
1983	FuncAttrs.addAttribute(A: "indirect-tls-seg-refs");
1984	if (CodeGenOpts.NoImplicitFloat)
1985	FuncAttrs.addAttribute(llvm::Attribute::NoImplicitFloat);
1986
1987	if (AttrOnCallSite) {
1988	// Attributes that should go on the call site only.
1989	// FIXME: Look for 'BuiltinAttr' on the function rather than re-checking
1990	// the -fno-builtin-foo list.
1991	if (!CodeGenOpts.SimplifyLibCalls \|\| LangOpts.isNoBuiltinFunc(Name))
1992	FuncAttrs.addAttribute(llvm::Attribute::NoBuiltin);
1993	if (!CodeGenOpts.TrapFuncName.empty())
1994	FuncAttrs.addAttribute(A: "trap-func-name", V: CodeGenOpts.TrapFuncName);
1995	} else {
1996	switch (CodeGenOpts.getFramePointer()) {
1997	case CodeGenOptions::FramePointerKind::None:
1998	// This is the default behavior.
1999	break;
2000	case CodeGenOptions::FramePointerKind::Reserved:
2001	case CodeGenOptions::FramePointerKind::NonLeaf:
2002	case CodeGenOptions::FramePointerKind::All:
2003	FuncAttrs.addAttribute(A: "frame-pointer",
2004	V: CodeGenOptions::getFramePointerKindName(
2005	Kind: CodeGenOpts.getFramePointer()));
2006	}
2007
2008	if (CodeGenOpts.LessPreciseFPMAD)
2009	FuncAttrs.addAttribute(A: "less-precise-fpmad", V: "true");
2010
2011	if (CodeGenOpts.NullPointerIsValid)
2012	FuncAttrs.addAttribute(llvm::Attribute::NullPointerIsValid);
2013
2014	if (LangOpts.getDefaultExceptionMode() == LangOptions::FPE_Ignore)
2015	FuncAttrs.addAttribute(A: "no-trapping-math", V: "true");
2016
2017	// TODO: Are these all needed?
2018	// unsafe/inf/nan/nsz are handled by instruction-level FastMathFlags.
2019	if (LangOpts.NoHonorInfs)
2020	FuncAttrs.addAttribute(A: "no-infs-fp-math", V: "true");
2021	if (LangOpts.NoHonorNaNs)
2022	FuncAttrs.addAttribute(A: "no-nans-fp-math", V: "true");
2023	if (LangOpts.ApproxFunc)
2024	FuncAttrs.addAttribute(A: "approx-func-fp-math", V: "true");
2025	if (LangOpts.AllowFPReassoc && LangOpts.AllowRecip &&
2026	LangOpts.NoSignedZero && LangOpts.ApproxFunc &&
2027	(LangOpts.getDefaultFPContractMode() ==
2028	LangOptions::FPModeKind::FPM_Fast \|\|
2029	LangOpts.getDefaultFPContractMode() ==
2030	LangOptions::FPModeKind::FPM_FastHonorPragmas))
2031	FuncAttrs.addAttribute(A: "unsafe-fp-math", V: "true");
2032	if (CodeGenOpts.SoftFloat)
2033	FuncAttrs.addAttribute(A: "use-soft-float", V: "true");
2034	FuncAttrs.addAttribute(A: "stack-protector-buffer-size",
2035	V: llvm::utostr(X: CodeGenOpts.SSPBufferSize));
2036	if (LangOpts.NoSignedZero)
2037	FuncAttrs.addAttribute(A: "no-signed-zeros-fp-math", V: "true");
2038
2039	// TODO: Reciprocal estimate codegen options should apply to instructions?
2040	const std::vector<std::string> &Recips = CodeGenOpts.Reciprocals;
2041	if (!Recips.empty())
2042	FuncAttrs.addAttribute(A: "reciprocal-estimates", V: llvm::join(R: Recips, Separator: ","));
2043
2044	if (!CodeGenOpts.PreferVectorWidth.empty() &&
2045	CodeGenOpts.PreferVectorWidth != "none")
2046	FuncAttrs.addAttribute(A: "prefer-vector-width",
2047	V: CodeGenOpts.PreferVectorWidth);
2048
2049	if (CodeGenOpts.StackRealignment)
2050	FuncAttrs.addAttribute(A: "stackrealign");
2051	if (CodeGenOpts.Backchain)
2052	FuncAttrs.addAttribute(A: "backchain");
2053	if (CodeGenOpts.EnableSegmentedStacks)
2054	FuncAttrs.addAttribute(A: "split-stack");
2055
2056	if (CodeGenOpts.SpeculativeLoadHardening)
2057	FuncAttrs.addAttribute(llvm::Attribute::SpeculativeLoadHardening);
2058
2059	// Add zero-call-used-regs attribute.
2060	switch (CodeGenOpts.getZeroCallUsedRegs()) {
2061	case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::Skip:
2062	FuncAttrs.removeAttribute(A: "zero-call-used-regs");
2063	break;
2064	case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::UsedGPRArg:
2065	FuncAttrs.addAttribute(A: "zero-call-used-regs", V: "used-gpr-arg");
2066	break;
2067	case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::UsedGPR:
2068	FuncAttrs.addAttribute(A: "zero-call-used-regs", V: "used-gpr");
2069	break;
2070	case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::UsedArg:
2071	FuncAttrs.addAttribute(A: "zero-call-used-regs", V: "used-arg");
2072	break;
2073	case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::Used:
2074	FuncAttrs.addAttribute(A: "zero-call-used-regs", V: "used");
2075	break;
2076	case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::AllGPRArg:
2077	FuncAttrs.addAttribute(A: "zero-call-used-regs", V: "all-gpr-arg");
2078	break;
2079	case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::AllGPR:
2080	FuncAttrs.addAttribute(A: "zero-call-used-regs", V: "all-gpr");
2081	break;
2082	case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::AllArg:
2083	FuncAttrs.addAttribute(A: "zero-call-used-regs", V: "all-arg");
2084	break;
2085	case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::All:
2086	FuncAttrs.addAttribute(A: "zero-call-used-regs", V: "all");
2087	break;
2088	}
2089	}
2090
2091	if (LangOpts.assumeFunctionsAreConvergent()) {
2092	// Conservatively, mark all functions and calls in CUDA and OpenCL as
2093	// convergent (meaning, they may call an intrinsically convergent op, such
2094	// as __syncthreads() / barrier(), and so can't have certain optimizations
2095	// applied around them). LLVM will remove this attribute where it safely
2096	// can.
2097	FuncAttrs.addAttribute(llvm::Attribute::Convergent);
2098	}
2099
2100	// TODO: NoUnwind attribute should be added for other GPU modes HIP,
2101	// OpenMP offload. AFAIK, neither of them support exceptions in device code.
2102	if ((LangOpts.CUDA && LangOpts.CUDAIsDevice) \|\| LangOpts.OpenCL \|\|
2103	LangOpts.SYCLIsDevice) {
2104	FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
2105	}
2106
2107	if (CodeGenOpts.SaveRegParams && !AttrOnCallSite)
2108	FuncAttrs.addAttribute(A: "save-reg-params");
2109
2110	for (StringRef Attr : CodeGenOpts.DefaultFunctionAttrs) {
2111	StringRef Var, Value;
2112	std::tie(args&: Var, args&: Value) = Attr.split(Separator: `'='`);
2113	FuncAttrs.addAttribute(A: Var, V: Value);
2114	}
2115
2116	TargetInfo::BranchProtectionInfo BPI(LangOpts);
2117	TargetCodeGenInfo::initBranchProtectionFnAttributes(BPI, FuncAttrs);
2118	}
2119
2120	/// Merges `target-features` from \TargetOpts and \F, and sets the result in
2121	/// \FuncAttr
2122	/// features from \F are always kept*
2123	/// a feature from \TargetOpts is kept if itself and its opposite are absent*
2124	/// from \F
2125	static void
2126	overrideFunctionFeaturesWithTargetFeatures(llvm::AttrBuilder &FuncAttr,
2127	const llvm::Function &F,
2128	const TargetOptions &TargetOpts) {
2129	auto FFeatures = F.getFnAttribute(Kind: "target-features");
2130
2131	llvm::StringSet<> MergedNames;
2132	SmallVector<StringRef> MergedFeatures;
2133	MergedFeatures.reserve(N: TargetOpts.Features.size());
2134
2135	auto AddUnmergedFeatures = [&](auto &&FeatureRange) {
2136	for (StringRef Feature : FeatureRange) {
2137	if (Feature.empty())
2138	continue;
2139	assert(Feature[`0`] == `'+'` \|\| Feature[`0`] == `'-'`);
2140	StringRef Name = Feature.drop_front(N: `1`);
2141	bool Merged = !MergedNames.insert(key: Name).second;
2142	if (!Merged)
2143	MergedFeatures.push_back(Elt: Feature);
2144	}
2145	};
2146
2147	if (FFeatures.isValid())
2148	AddUnmergedFeatures (llvm::split(Str: FFeatures.getValueAsString(), Separator: `','`));
2149	AddUnmergedFeatures (TargetOpts.Features);
2150
2151	if (!MergedFeatures.empty()) {
2152	llvm::sort(C&: MergedFeatures);
2153	FuncAttr.addAttribute(A: "target-features", V: llvm::join(R&: MergedFeatures, Separator: ","));
2154	}
2155	}
2156
2157	void CodeGen::mergeDefaultFunctionDefinitionAttributes(
2158	llvm::Function &F, const CodeGenOptions &CodeGenOpts,
2159	const LangOptions &LangOpts, const TargetOptions &TargetOpts,
2160	bool WillInternalize) {
2161
2162	llvm::AttrBuilder FuncAttrs(F.getContext());
2163	// Here we only extract the options that are relevant compared to the version
2164	// from GetCPUAndFeaturesAttributes.
2165	if (!TargetOpts.CPU.empty())
2166	FuncAttrs.addAttribute(A: "target-cpu", V: TargetOpts.CPU);
2167	if (!TargetOpts.TuneCPU.empty())
2168	FuncAttrs.addAttribute(A: "tune-cpu", V: TargetOpts.TuneCPU);
2169
2170	::getTrivialDefaultFunctionAttributes(Name: F.getName(), HasOptnone: F.hasOptNone(),
2171	CodeGenOpts, LangOpts,
2172	/AttrOnCallSite=/false, FuncAttrs);
2173
2174	if (!WillInternalize && F.isInterposable()) {
2175	// Do not promote "dynamic" denormal-fp-math to this translation unit's
2176	// setting for weak functions that won't be internalized. The user has no
2177	// real control for how builtin bitcode is linked, so we shouldn't assume
2178	// later copies will use a consistent mode.
2179	F.addFnAttrs(Attrs: FuncAttrs);
2180	return;
2181	}
2182
2183	llvm::AttributeMask AttrsToRemove;
2184
2185	llvm::DenormalMode DenormModeToMerge = F.getDenormalModeRaw();
2186	llvm::DenormalMode DenormModeToMergeF32 = F.getDenormalModeF32Raw();
2187	llvm::DenormalMode Merged =
2188	CodeGenOpts.FPDenormalMode.mergeCalleeMode(Callee: DenormModeToMerge);
2189	llvm::DenormalMode MergedF32 = CodeGenOpts.FP32DenormalMode;
2190
2191	if (DenormModeToMergeF32.isValid()) {
2192	MergedF32 =
2193	CodeGenOpts.FP32DenormalMode.mergeCalleeMode(Callee: DenormModeToMergeF32);
2194	}
2195
2196	if (Merged == llvm::DenormalMode::getDefault()) {
2197	AttrsToRemove.addAttribute(A: "denormal-fp-math");
2198	} else if (Merged != DenormModeToMerge) {
2199	// Overwrite existing attribute
2200	FuncAttrs.addAttribute(A: "denormal-fp-math",
2201	V: CodeGenOpts.FPDenormalMode.str());
2202	}
2203
2204	if (MergedF32 == llvm::DenormalMode::getDefault()) {
2205	AttrsToRemove.addAttribute(A: "denormal-fp-math-f32");
2206	} else if (MergedF32 != DenormModeToMergeF32) {
2207	// Overwrite existing attribute
2208	FuncAttrs.addAttribute(A: "denormal-fp-math-f32",
2209	V: CodeGenOpts.FP32DenormalMode.str());
2210	}
2211
2212	F.removeFnAttrs(Attrs: AttrsToRemove);
2213	addDenormalModeAttrs(FPDenormalMode: Merged, FP32DenormalMode: MergedF32, FuncAttrs);
2214
2215	overrideFunctionFeaturesWithTargetFeatures(FuncAttr&: FuncAttrs, F, TargetOpts);
2216
2217	F.addFnAttrs(Attrs: FuncAttrs);
2218	}
2219
2220	void CodeGenModule::getTrivialDefaultFunctionAttributes(
2221	StringRef Name, bool HasOptnone, bool AttrOnCallSite,
2222	llvm::AttrBuilder &FuncAttrs) {
2223	::getTrivialDefaultFunctionAttributes(Name, HasOptnone, CodeGenOpts: getCodeGenOpts(),
2224	LangOpts: getLangOpts(), AttrOnCallSite,
2225	FuncAttrs);
2226	}
2227
2228	void CodeGenModule::getDefaultFunctionAttributes(StringRef Name,
2229	bool HasOptnone,
2230	bool AttrOnCallSite,
2231	llvm::AttrBuilder &FuncAttrs) {
2232	getTrivialDefaultFunctionAttributes(Name, HasOptnone, AttrOnCallSite,
2233	FuncAttrs);
2234
2235	if (!AttrOnCallSite)
2236	TargetCodeGenInfo::initPointerAuthFnAttributes(Opts: CodeGenOpts.PointerAuth,
2237	FuncAttrs);
2238
2239	// If we're just getting the default, get the default values for mergeable
2240	// attributes.
2241	if (!AttrOnCallSite)
2242	addMergableDefaultFunctionAttributes(CodeGenOpts, FuncAttrs);
2243	}
2244
2245	void CodeGenModule::addDefaultFunctionDefinitionAttributes(
2246	llvm::AttrBuilder &attrs) {
2247	getDefaultFunctionAttributes(/function name/ Name: "", /optnone/ HasOptnone: false,
2248	/for call/ AttrOnCallSite: false, FuncAttrs&: attrs);
2249	GetCPUAndFeaturesAttributes(GD: GlobalDecl (), AttrBuilder&: attrs);
2250	}
2251
2252	static void addNoBuiltinAttributes(llvm::AttrBuilder &FuncAttrs,
2253	const LangOptions &LangOpts,
2254	const NoBuiltinAttr NBA = nullptr*) {
2255	auto AddNoBuiltinAttr = [&FuncAttrs](StringRef BuiltinName) {
2256	SmallString<`32`> AttributeName;
2257	AttributeName += "no-builtin-";
2258	AttributeName += BuiltinName;
2259	FuncAttrs.addAttribute(A: AttributeName);
2260	};
2261
2262	// First, handle the language options passed through -fno-builtin.
2263	if (LangOpts.NoBuiltin) {
2264	// -fno-builtin disables them all.
2265	FuncAttrs.addAttribute(A: "no-builtins");
2266	return;
2267	}
2268
2269	// Then, add attributes for builtins specified through -fno-builtin-<name>.
2270	llvm::for_each(Range: LangOpts.NoBuiltinFuncs, F: AddNoBuiltinAttr);
2271
2272	// Now, let's check the __attribute__((no_builtin("...")) attribute added to
2273	// the source.
2274	if (!NBA)
2275	return;
2276
2277	// If there is a wildcard in the builtin names specified through the
2278	// attribute, disable them all.
2279	if (llvm::is_contained(NBA->builtinNames(), "*")) {
2280	FuncAttrs.addAttribute(A: "no-builtins");
2281	return;
2282	}
2283
2284	// And last, add the rest of the builtin names.
2285	llvm::for_each(NBA->builtinNames(), AddNoBuiltinAttr);
2286	}
2287
2288	static bool DetermineNoUndef(QualType QTy, CodeGenTypes &Types,
2289	const llvm::DataLayout &DL, const ABIArgInfo &AI,
2290	bool CheckCoerce = true) {
2291	llvm::Type *Ty = Types.ConvertTypeForMem(T: QTy);
2292	if (AI.getKind() == ABIArgInfo::Indirect \|\|
2293	AI.getKind() == ABIArgInfo::IndirectAliased)
2294	return true;
2295	if (AI.getKind() == ABIArgInfo::Extend && !AI.isNoExt())
2296	return true;
2297	if (!DL.typeSizeEqualsStoreSize(Ty))
2298	// TODO: This will result in a modest amount of values not marked noundef
2299	// when they could be. We care about values that invisibly* contain undef*
2300	// bits from the perspective of LLVM IR.
2301	return false;
2302	if (CheckCoerce && AI.canHaveCoerceToType()) {
2303	llvm::Type *CoerceTy = AI.getCoerceToType();
2304	if (llvm::TypeSize::isKnownGT(LHS: DL.getTypeSizeInBits(Ty: CoerceTy),
2305	RHS: DL.getTypeSizeInBits(Ty)))
2306	// If we're coercing to a type with a greater size than the canonical one,
2307	// we're introducing new undef bits.
2308	// Coercing to a type of smaller or equal size is ok, as we know that
2309	// there's no internal padding (typeSizeEqualsStoreSize).
2310	return false;
2311	}
2312	if (QTy ->isBitIntType())
2313	return true;
2314	if (QTy ->isReferenceType())
2315	return true;
2316	if (QTy ->isNullPtrType())
2317	return false;
2318	if (QTy ->isMemberPointerType())
2319	// TODO: Some member pointers are `noundef`, but it depends on the ABI. For
2320	// now, never mark them.
2321	return false;
2322	if (QTy ->isScalarType()) {
2323	if (const ComplexType *Complex = dyn_cast<ComplexType>(Val&: QTy))
2324	return DetermineNoUndef(QTy: Complex->getElementType(), Types, DL, AI, CheckCoerce: false);
2325	return true;
2326	}
2327	if (const VectorType *Vector = dyn_cast<VectorType>(Val&: QTy))
2328	return DetermineNoUndef(QTy: Vector->getElementType(), Types, DL, AI, CheckCoerce: false);
2329	if (const MatrixType *Matrix = dyn_cast<MatrixType>(Val&: QTy))
2330	return DetermineNoUndef(QTy: Matrix->getElementType(), Types, DL, AI, CheckCoerce: false);
2331	if (const ArrayType *Array = dyn_cast<ArrayType>(Val&: QTy))
2332	return DetermineNoUndef(QTy: Array->getElementType(), Types, DL, AI, CheckCoerce: false);
2333
2334	// TODO: Some structs may be `noundef`, in specific situations.
2335	return false;
2336	}
2337
2338	/// Check if the argument of a function has maybe_undef attribute.
2339	static bool IsArgumentMaybeUndef(const Decl *TargetDecl,
2340	unsigned NumRequiredArgs, unsigned ArgNo) {
2341	const auto *FD = dyn_cast_or_null<FunctionDecl>(Val: TargetDecl);
2342	if (!FD)
2343	return false;
2344
2345	// Assume variadic arguments do not have maybe_undef attribute.
2346	if (ArgNo >= NumRequiredArgs)
2347	return false;
2348
2349	// Check if argument has maybe_undef attribute.
2350	if (ArgNo < FD->getNumParams()) {
2351	const ParmVarDecl *Param = FD->getParamDecl(i: ArgNo);
2352	if (Param && Param->hasAttr<MaybeUndefAttr>())
2353	return true;
2354	}
2355
2356	return false;
2357	}
2358
2359	/// Test if it's legal to apply nofpclass for the given parameter type and it's
2360	/// lowered IR type.
2361	static bool canApplyNoFPClass(const ABIArgInfo &AI, QualType ParamType,
2362	bool IsReturn) {
2363	// Should only apply to FP types in the source, not ABI promoted.
2364	if (!ParamType ->hasFloatingRepresentation())
2365	return false;
2366
2367	// The promoted-to IR type also needs to support nofpclass.
2368	llvm::Type *IRTy = AI.getCoerceToType();
2369	if (llvm::AttributeFuncs::isNoFPClassCompatibleType(Ty: IRTy))
2370	return true;
2371
2372	if (llvm::StructType *ST = dyn_cast<llvm::StructType>(Val: IRTy)) {
2373	return !IsReturn && AI.getCanBeFlattened() &&
2374	llvm::all_of(Range: ST->elements(),
2375	P: llvm::AttributeFuncs::isNoFPClassCompatibleType);
2376	}
2377
2378	return false;
2379	}
2380
2381	/// Return the nofpclass mask that can be applied to floating-point parameters.
2382	static llvm::FPClassTest getNoFPClassTestMask(const LangOptions &LangOpts) {
2383	llvm::FPClassTest Mask = llvm::fcNone;
2384	if (LangOpts.NoHonorInfs)
2385	Mask \|= llvm::fcInf;
2386	if (LangOpts.NoHonorNaNs)
2387	Mask \|= llvm::fcNan;
2388	return Mask;
2389	}
2390
2391	void CodeGenModule::AdjustMemoryAttribute(StringRef Name,
2392	CGCalleeInfo CalleeInfo,
2393	llvm::AttributeList &Attrs) {
2394	if (Attrs.getMemoryEffects().getModRef() == llvm::ModRefInfo::NoModRef) {
2395	Attrs = Attrs.removeFnAttribute(getLLVMContext(), llvm::Attribute::Memory);
2396	llvm::Attribute MemoryAttr = llvm::Attribute::getWithMemoryEffects(
2397	Context&: getLLVMContext(), ME: llvm::MemoryEffects::writeOnly());
2398	Attrs = Attrs.addFnAttribute(C&: getLLVMContext(), Attr: MemoryAttr);
2399	}
2400	}
2401
2402	/// Construct the IR attribute list of a function or call.
2403	///
2404	/// When adding an attribute, please consider where it should be handled:
2405	///
2406	/// - getDefaultFunctionAttributes is for attributes that are essentially
2407	/// part of the global target configuration (but perhaps can be
2408	/// overridden on a per-function basis). Adding attributes there
2409	/// will cause them to also be set in frontends that build on Clang's
2410	/// target-configuration logic, as well as for code defined in library
2411	/// modules such as CUDA's libdevice.
2412	///
2413	/// - ConstructAttributeList builds on top of getDefaultFunctionAttributes
2414	/// and adds declaration-specific, convention-specific, and
2415	/// frontend-specific logic. The last is of particular importance:
2416	/// attributes that restrict how the frontend generates code must be
2417	/// added here rather than getDefaultFunctionAttributes.
2418	///
2419	void CodeGenModule::ConstructAttributeList(StringRef Name,
2420	const CGFunctionInfo &FI,
2421	CGCalleeInfo CalleeInfo,
2422	llvm::AttributeList &AttrList,
2423	unsigned &CallingConv,
2424	bool AttrOnCallSite, bool IsThunk) {
2425	llvm::AttrBuilder FuncAttrs(getLLVMContext());
2426	llvm::AttrBuilder RetAttrs(getLLVMContext());
2427
2428	// Collect function IR attributes from the CC lowering.
2429	// We'll collect the paramete and result attributes later.
2430	CallingConv = FI.getEffectiveCallingConvention();
2431	if (FI.isNoReturn())
2432	FuncAttrs.addAttribute(llvm::Attribute::NoReturn);
2433	if (FI.isCmseNSCall())
2434	FuncAttrs.addAttribute(A: "cmse_nonsecure_call");
2435
2436	// Collect function IR attributes from the callee prototype if we have one.
2437	AddAttributesFromFunctionProtoType(Ctx&: getContext(), FuncAttrs,
2438	FPT: CalleeInfo.getCalleeFunctionProtoType());
2439	const Decl *TargetDecl = CalleeInfo.getCalleeDecl().getDecl();
2440
2441	// Attach assumption attributes to the declaration. If this is a call
2442	// site, attach assumptions from the caller to the call as well.
2443	AddAttributesFromOMPAssumes(FuncAttrs, Callee: TargetDecl);
2444
2445	bool HasOptnone = false;
2446	// The NoBuiltinAttr attached to the target FunctionDecl.
2447	const NoBuiltinAttr NBA = nullptr*;
2448
2449	// Some ABIs may result in additional accesses to arguments that may
2450	// otherwise not be present.
2451	auto AddPotentialArgAccess = [&]() {
2452	llvm::Attribute A = FuncAttrs.getAttribute(llvm::Attribute::Memory);
2453	if (A.isValid())
2454	FuncAttrs.addMemoryAttr(ME: A.getMemoryEffects() \|
2455	llvm::MemoryEffects::argMemOnly());
2456	};
2457
2458	// Collect function IR attributes based on declaration-specific
2459	// information.
2460	// FIXME: handle sseregparm someday...
2461	if (TargetDecl) {
2462	if (TargetDecl->hasAttr<ReturnsTwiceAttr>())
2463	FuncAttrs.addAttribute(llvm::Attribute::ReturnsTwice);
2464	if (TargetDecl->hasAttr<NoThrowAttr>())
2465	FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
2466	if (TargetDecl->hasAttr<NoReturnAttr>())
2467	FuncAttrs.addAttribute(llvm::Attribute::NoReturn);
2468	if (TargetDecl->hasAttr<ColdAttr>())
2469	FuncAttrs.addAttribute(llvm::Attribute::Cold);
2470	if (TargetDecl->hasAttr<HotAttr>())
2471	FuncAttrs.addAttribute(llvm::Attribute::Hot);
2472	if (TargetDecl->hasAttr<NoDuplicateAttr>())
2473	FuncAttrs.addAttribute(llvm::Attribute::NoDuplicate);
2474	if (TargetDecl->hasAttr<ConvergentAttr>())
2475	FuncAttrs.addAttribute(llvm::Attribute::Convergent);
2476
2477	if (const FunctionDecl *Fn = dyn_cast<FunctionDecl>(Val: TargetDecl)) {
2478	AddAttributesFromFunctionProtoType(
2479	getContext(), FuncAttrs, Fn->getType()->getAs<FunctionProtoType>());
2480	if (AttrOnCallSite && Fn->isReplaceableGlobalAllocationFunction()) {
2481	// A sane operator new returns a non-aliasing pointer.
2482	auto Kind = Fn->getDeclName().getCXXOverloadedOperator();
2483	if (getCodeGenOpts().AssumeSaneOperatorNew &&
2484	(Kind == OO_New \|\| Kind == OO_Array_New))
2485	RetAttrs.addAttribute(llvm::Attribute::NoAlias);
2486	}
2487	const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(Val: Fn);
2488	const bool IsVirtualCall = MD && MD->isVirtual();
2489	// Don't use [[noreturn]], _Noreturn or [[no_builtin]] for a call to a
2490	// virtual function. These attributes are not inherited by overloads.
2491	if (!(AttrOnCallSite && IsVirtualCall)) {
2492	if (Fn->isNoReturn())
2493	FuncAttrs.addAttribute(llvm::Attribute::NoReturn);
2494	NBA = Fn->getAttr<NoBuiltinAttr>();
2495	}
2496	}
2497
2498	if (isa<FunctionDecl>(Val: TargetDecl) \|\| isa<VarDecl>(Val: TargetDecl)) {
2499	// Only place nomerge attribute on call sites, never functions. This
2500	// allows it to work on indirect virtual function calls.
2501	if (AttrOnCallSite && TargetDecl->hasAttr<NoMergeAttr>())
2502	FuncAttrs.addAttribute(llvm::Attribute::NoMerge);
2503	}
2504
2505	// 'const', 'pure' and 'noalias' attributed functions are also nounwind.
2506	if (TargetDecl->hasAttr<ConstAttr>()) {
2507	FuncAttrs.addMemoryAttr(ME: llvm::MemoryEffects::none());
2508	FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
2509	// gcc specifies that 'const' functions have greater restrictions than
2510	// 'pure' functions, so they also cannot have infinite loops.
2511	FuncAttrs.addAttribute(llvm::Attribute::WillReturn);
2512	} else if (TargetDecl->hasAttr<PureAttr>()) {
2513	FuncAttrs.addMemoryAttr(ME: llvm::MemoryEffects::readOnly());
2514	FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
2515	// gcc specifies that 'pure' functions cannot have infinite loops.
2516	FuncAttrs.addAttribute(llvm::Attribute::WillReturn);
2517	} else if (TargetDecl->hasAttr<NoAliasAttr>()) {
2518	FuncAttrs.addMemoryAttr(ME: llvm::MemoryEffects::inaccessibleOrArgMemOnly());
2519	FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
2520	}
2521	if (const auto *RA = TargetDecl->getAttr<RestrictAttr>();
2522	RA && RA->getDeallocator() == nullptr)
2523	RetAttrs.addAttribute(llvm::Attribute::NoAlias);
2524	if (TargetDecl->hasAttr<ReturnsNonNullAttr>() &&
2525	!CodeGenOpts.NullPointerIsValid)
2526	RetAttrs.addAttribute(llvm::Attribute::NonNull);
2527	if (TargetDecl->hasAttr<AnyX86NoCallerSavedRegistersAttr>())
2528	FuncAttrs.addAttribute(A: "no_caller_saved_registers");
2529	if (TargetDecl->hasAttr<AnyX86NoCfCheckAttr>())
2530	FuncAttrs.addAttribute(llvm::Attribute::NoCfCheck);
2531	if (TargetDecl->hasAttr<LeafAttr>())
2532	FuncAttrs.addAttribute(llvm::Attribute::NoCallback);
2533	if (TargetDecl->hasAttr<BPFFastCallAttr>())
2534	FuncAttrs.addAttribute(A: "bpf_fastcall");
2535
2536	HasOptnone = TargetDecl->hasAttr<OptimizeNoneAttr>();
2537	if (auto *AllocSize = TargetDecl->getAttr<AllocSizeAttr>()) {
2538	std::optional<unsigned> NumElemsParam;
2539	if (AllocSize->getNumElemsParam().isValid())
2540	NumElemsParam = AllocSize->getNumElemsParam().getLLVMIndex();
2541	FuncAttrs.addAllocSizeAttr(ElemSizeArg: AllocSize->getElemSizeParam().getLLVMIndex(),
2542	NumElemsArg: NumElemsParam);
2543	}
2544
2545	if (DeviceKernelAttr::isOpenCLSpelling(
2546	TargetDecl->getAttr<DeviceKernelAttr>()) &&
2547	CallingConv != CallingConv::CC_C &&
2548	CallingConv != CallingConv::CC_SpirFunction) {
2549	// Check CallingConv to avoid adding uniform-work-group-size attribute to
2550	// OpenCL Kernel Stub
2551	if (getLangOpts().OpenCLVersion <= `120`) {
2552	// OpenCL v1.2 Work groups are always uniform
2553	FuncAttrs.addAttribute(A: "uniform-work-group-size", V: "true");
2554	} else {
2555	// OpenCL v2.0 Work groups may be whether uniform or not.
2556	// '-cl-uniform-work-group-size' compile option gets a hint
2557	// to the compiler that the global work-size be a multiple of
2558	// the work-group size specified to clEnqueueNDRangeKernel
2559	// (i.e. work groups are uniform).
2560	FuncAttrs.addAttribute(
2561	A: "uniform-work-group-size",
2562	V: llvm::toStringRef(B: getLangOpts().OffloadUniformBlock));
2563	}
2564	}
2565
2566	if (TargetDecl->hasAttr<CUDAGlobalAttr>() &&
2567	getLangOpts().OffloadUniformBlock)
2568	FuncAttrs.addAttribute(A: "uniform-work-group-size", V: "true");
2569
2570	if (TargetDecl->hasAttr<ArmLocallyStreamingAttr>())
2571	FuncAttrs.addAttribute(A: "aarch64_pstate_sm_body");
2572	}
2573
2574	// Attach "no-builtins" attributes to:
2575	// call sites: both `nobuiltin` and "no-builtins" or "no-builtin-<name>".*
2576	// definitions: "no-builtins" or "no-builtin-<name>" only.*
2577	// The attributes can come from:
2578	// LangOpts: -ffreestanding, -fno-builtin, -fno-builtin-<name>*
2579	// FunctionDecl attributes: __attribute__((no_builtin(...)))*
2580	addNoBuiltinAttributes(FuncAttrs, getLangOpts(), NBA);
2581
2582	// Collect function IR attributes based on global settiings.
2583	getDefaultFunctionAttributes(Name, HasOptnone, AttrOnCallSite, FuncAttrs);
2584
2585	// Override some default IR attributes based on declaration-specific
2586	// information.
2587	if (TargetDecl) {
2588	if (TargetDecl->hasAttr<NoSpeculativeLoadHardeningAttr>())
2589	FuncAttrs.removeAttribute(llvm::Attribute::SpeculativeLoadHardening);
2590	if (TargetDecl->hasAttr<SpeculativeLoadHardeningAttr>())
2591	FuncAttrs.addAttribute(llvm::Attribute::SpeculativeLoadHardening);
2592	if (TargetDecl->hasAttr<NoSplitStackAttr>())
2593	FuncAttrs.removeAttribute(A: "split-stack");
2594	if (TargetDecl->hasAttr<ZeroCallUsedRegsAttr>()) {
2595	// A function "__attribute__((...))" overrides the command-line flag.
2596	auto Kind =
2597	TargetDecl->getAttr<ZeroCallUsedRegsAttr>()->getZeroCallUsedRegs();
2598	FuncAttrs.removeAttribute(A: "zero-call-used-regs");
2599	FuncAttrs.addAttribute(
2600	"zero-call-used-regs",
2601	ZeroCallUsedRegsAttr::ConvertZeroCallUsedRegsKindToStr(Kind));
2602	}
2603
2604	// Add NonLazyBind attribute to function declarations when -fno-plt
2605	// is used.
2606	// FIXME: what if we just haven't processed the function definition
2607	// yet, or if it's an external definition like C99 inline?
2608	if (CodeGenOpts.NoPLT) {
2609	if (auto *Fn = dyn_cast<FunctionDecl>(Val: TargetDecl)) {
2610	if (!Fn->isDefined() && !AttrOnCallSite) {
2611	FuncAttrs.addAttribute(llvm::Attribute::NonLazyBind);
2612	}
2613	}
2614	}
2615	// Remove 'convergent' if requested.
2616	if (TargetDecl->hasAttr<NoConvergentAttr>())
2617	FuncAttrs.removeAttribute(llvm::Attribute::Convergent);
2618	}
2619
2620	// Add "sample-profile-suffix-elision-policy" attribute for internal linkage
2621	// functions with -funique-internal-linkage-names.
2622	if (TargetDecl && CodeGenOpts.UniqueInternalLinkageNames) {
2623	if (const auto *FD = dyn_cast_or_null<FunctionDecl>(Val: TargetDecl)) {
2624	if (!FD->isExternallyVisible())
2625	FuncAttrs.addAttribute(A: "sample-profile-suffix-elision-policy",
2626	V: "selected");
2627	}
2628	}
2629
2630	// Collect non-call-site function IR attributes from declaration-specific
2631	// information.
2632	if (!AttrOnCallSite) {
2633	if (TargetDecl && TargetDecl->hasAttr<CmseNSEntryAttr>())
2634	FuncAttrs.addAttribute(A: "cmse_nonsecure_entry");
2635
2636	// Whether tail calls are enabled.
2637	auto shouldDisableTailCalls = [&] {
2638	// Should this be honored in getDefaultFunctionAttributes?
2639	if (CodeGenOpts.DisableTailCalls)
2640	return true;
2641
2642	if (!TargetDecl)
2643	return false;
2644
2645	if (TargetDecl->hasAttr<DisableTailCallsAttr>() \|\|
2646	TargetDecl->hasAttr<AnyX86InterruptAttr>())
2647	return true;
2648
2649	if (CodeGenOpts.NoEscapingBlockTailCalls) {
2650	if (const auto *BD = dyn_cast<BlockDecl>(Val: TargetDecl))
2651	if (!BD->doesNotEscape())
2652	return true;
2653	}
2654
2655	return false;
2656	};
2657	if (shouldDisableTailCalls ())
2658	FuncAttrs.addAttribute(A: "disable-tail-calls", V: "true");
2659
2660	// These functions require the returns_twice attribute for correct codegen,
2661	// but the attribute may not be added if -fno-builtin is specified. We
2662	// explicitly add that attribute here.
2663	static const llvm::StringSet<> ReturnsTwiceFn{
2664	"_setjmpex", "setjmp", "_setjmp", "vfork",
2665	"sigsetjmp", "__sigsetjmp", "savectx", "getcontext"};
2666	if (ReturnsTwiceFn.contains(Name))
2667	FuncAttrs.addAttribute(llvm::Attribute::ReturnsTwice);
2668
2669	// CPU/feature overrides. addDefaultFunctionDefinitionAttributes
2670	// handles these separately to set them based on the global defaults.
2671	GetCPUAndFeaturesAttributes(GD: CalleeInfo.getCalleeDecl(), AttrBuilder&: FuncAttrs);
2672	}
2673
2674	// Mark functions that are replaceable by the loader.
2675	if (CodeGenOpts.isLoaderReplaceableFunctionName(FuncName: Name))
2676	FuncAttrs.addAttribute(A: "loader-replaceable");
2677
2678	// Collect attributes from arguments and return values.
2679	ClangToLLVMArgMapping IRFunctionArgs(getContext(), FI);
2680
2681	QualType RetTy = FI.getReturnType();
2682	const ABIArgInfo &RetAI = FI.getReturnInfo();
2683	const llvm::DataLayout &DL = getDataLayout();
2684
2685	// Determine if the return type could be partially undef
2686	if (CodeGenOpts.EnableNoundefAttrs &&
2687	HasStrictReturn(Module: *this, RetTy, TargetDecl)) {
2688	if (!RetTy->isVoidType() && RetAI.getKind() != ABIArgInfo::Indirect &&
2689	DetermineNoUndef(RetTy, getTypes(), DL, RetAI))
2690	RetAttrs.addAttribute(llvm::Attribute::NoUndef);
2691	}
2692
2693	switch (RetAI.getKind()) {
2694	case ABIArgInfo::Extend:
2695	if (RetAI.isSignExt())
2696	RetAttrs.addAttribute(llvm::Attribute::SExt);
2697	else if (RetAI.isZeroExt())
2698	RetAttrs.addAttribute(llvm::Attribute::ZExt);
2699	else
2700	RetAttrs.addAttribute(llvm::Attribute::NoExt);
2701	[[fallthrough]];
2702	case ABIArgInfo::Direct:
2703	if (RetAI.getInReg())
2704	RetAttrs.addAttribute(llvm::Attribute::InReg);
2705
2706	if (canApplyNoFPClass(AI: RetAI, ParamType: RetTy, IsReturn: true))
2707	RetAttrs.addNoFPClassAttr(NoFPClassMask: getNoFPClassTestMask(LangOpts: getLangOpts()));
2708
2709	break;
2710	case ABIArgInfo::Ignore:
2711	break;
2712
2713	case ABIArgInfo::InAlloca:
2714	case ABIArgInfo::Indirect: {
2715	// inalloca and sret disable readnone and readonly
2716	AddPotentialArgAccess ();
2717	break;
2718	}
2719
2720	case ABIArgInfo::CoerceAndExpand:
2721	break;
2722
2723	case ABIArgInfo::Expand:
2724	case ABIArgInfo::IndirectAliased:
2725	llvm_unreachable("Invalid ABI kind for return argument");
2726	}
2727
2728	if (!IsThunk) {
2729	// FIXME: fix this properly, https://reviews.llvm.org/D100388
2730	if (const auto *RefTy = RetTy ->getAs<ReferenceType>()) {
2731	QualType PTy = RefTy->getPointeeType();
2732	if (!PTy ->isIncompleteType() && PTy ->isConstantSizeType())
2733	RetAttrs.addDereferenceableAttr(
2734	Bytes: getMinimumObjectSize(Ty: PTy).getQuantity());
2735	if (getTypes().getTargetAddressSpace(PTy) == `0` &&
2736	!CodeGenOpts.NullPointerIsValid)
2737	RetAttrs.addAttribute(llvm::Attribute::NonNull);
2738	if (PTy ->isObjectType()) {
2739	llvm::Align Alignment =
2740	getNaturalPointeeTypeAlignment(T: RetTy).getAsAlign();
2741	RetAttrs.addAlignmentAttr(Align: Alignment);
2742	}
2743	}
2744	}
2745
2746	bool hasUsedSRet = false;
2747	SmallVector<llvm::AttributeSet, `4`> ArgAttrs(IRFunctionArgs.totalIRArgs());
2748
2749	// Attach attributes to sret.
2750	if (IRFunctionArgs.hasSRetArg()) {
2751	llvm::AttrBuilder SRETAttrs(getLLVMContext());
2752	SRETAttrs.addStructRetAttr(Ty: getTypes().ConvertTypeForMem(T: RetTy));
2753	SRETAttrs.addAttribute(llvm::Attribute::Writable);
2754	SRETAttrs.addAttribute(llvm::Attribute::DeadOnUnwind);
2755	hasUsedSRet = true;
2756	if (RetAI.getInReg())
2757	SRETAttrs.addAttribute(llvm::Attribute::InReg);
2758	SRETAttrs.addAlignmentAttr(Align: RetAI.getIndirectAlign().getQuantity());
2759	ArgAttrs [IRFunctionArgs.getSRetArgNo()] =
2760	llvm::AttributeSet::get(C&: getLLVMContext(), B: SRETAttrs);
2761	}
2762
2763	// Attach attributes to inalloca argument.
2764	if (IRFunctionArgs.hasInallocaArg()) {
2765	llvm::AttrBuilder Attrs(getLLVMContext());
2766	Attrs.addInAllocaAttr(Ty: FI.getArgStruct());
2767	ArgAttrs [IRFunctionArgs.getInallocaArgNo()] =
2768	llvm::AttributeSet::get(C&: getLLVMContext(), B: Attrs);
2769	}
2770
2771	// Apply `nonnull`, `dereferenceable(N)` and `align N` to the `this` argument,
2772	// unless this is a thunk function.
2773	// FIXME: fix this properly, https://reviews.llvm.org/D100388
2774	if (FI.isInstanceMethod() && !IRFunctionArgs.hasInallocaArg() &&
2775	!FI.arg_begin()->type->isVoidPointerType() && !IsThunk) {
2776	auto IRArgs = IRFunctionArgs.getIRArgs(ArgNo: `0`);
2777
2778	assert(IRArgs.second == `1` && "Expected only a single `this` pointer.");
2779
2780	llvm::AttrBuilder Attrs(getLLVMContext());
2781
2782	QualType ThisTy = FI.arg_begin()->type.getTypePtr()->getPointeeType();
2783
2784	if (!CodeGenOpts.NullPointerIsValid &&
2785	getTypes().getTargetAddressSpace(T: FI.arg_begin()->type) == `0`) {
2786	Attrs.addAttribute(llvm::Attribute::NonNull);
2787	Attrs.addDereferenceableAttr(Bytes: getMinimumObjectSize(Ty: ThisTy).getQuantity());
2788	} else {
2789	// FIXME dereferenceable should be correct here, regardless of
2790	// NullPointerIsValid. However, dereferenceable currently does not always
2791	// respect NullPointerIsValid and may imply nonnull and break the program.
2792	// See https://reviews.llvm.org/D66618 for discussions.
2793	Attrs.addDereferenceableOrNullAttr(
2794	Bytes: getMinimumObjectSize(
2795	Ty: FI.arg_begin()->type.castAs<PointerType>()->getPointeeType())
2796	.getQuantity());
2797	}
2798
2799	llvm::Align Alignment =
2800	getNaturalTypeAlignment(T: ThisTy, /BaseInfo=/nullptr,
2801	/TBAAInfo=/nullptr, /forPointeeType=/true)
2802	.getAsAlign();
2803	Attrs.addAlignmentAttr(Align: Alignment);
2804
2805	ArgAttrs [IRArgs.first] = llvm::AttributeSet::get(C&: getLLVMContext(), B: Attrs);
2806	}
2807
2808	unsigned ArgNo = `0`;
2809	for (CGFunctionInfo::const_arg_iterator I = FI.arg_begin(), E = FI.arg_end();
2810	I != E; ++I, ++ArgNo) {
2811	QualType ParamType = I->type;
2812	const ABIArgInfo &AI = I->info;
2813	llvm::AttrBuilder Attrs(getLLVMContext());
2814
2815	// Add attribute for padding argument, if necessary.
2816	if (IRFunctionArgs.hasPaddingArg(ArgNo)) {
2817	if (AI.getPaddingInReg()) {
2818	ArgAttrs[IRFunctionArgs.getPaddingArgNo(ArgNo)] =
2819	llvm::AttributeSet::get(getLLVMContext(),
2820	llvm::AttrBuilder(getLLVMContext())
2821	.addAttribute(llvm::Attribute::InReg));
2822	}
2823	}
2824
2825	// Decide whether the argument we're handling could be partially undef
2826	if (CodeGenOpts.EnableNoundefAttrs &&
2827	DetermineNoUndef(QTy: ParamType, Types&: getTypes(), DL, AI)) {
2828	Attrs.addAttribute(llvm::Attribute::NoUndef);
2829	}
2830
2831	// 'restrict' -> 'noalias' is done in EmitFunctionProlog when we
2832	// have the corresponding parameter variable. It doesn't make
2833	// sense to do it here because parameters are so messed up.
2834	switch (AI.getKind()) {
2835	case ABIArgInfo::Extend:
2836	if (AI.isSignExt())
2837	Attrs.addAttribute(llvm::Attribute::SExt);
2838	else if (AI.isZeroExt())
2839	Attrs.addAttribute(llvm::Attribute::ZExt);
2840	else
2841	Attrs.addAttribute(llvm::Attribute::NoExt);
2842	[[fallthrough]];
2843	case ABIArgInfo::Direct:
2844	if (ArgNo == `0` && FI.isChainCall())
2845	Attrs.addAttribute(llvm::Attribute::Nest);
2846	else if (AI.getInReg())
2847	Attrs.addAttribute(llvm::Attribute::InReg);
2848	Attrs.addStackAlignmentAttr(Align: llvm::MaybeAlign (AI.getDirectAlign()));
2849
2850	if (canApplyNoFPClass(AI, ParamType, IsReturn: false))
2851	Attrs.addNoFPClassAttr(NoFPClassMask: getNoFPClassTestMask(LangOpts: getLangOpts()));
2852	break;
2853	case ABIArgInfo::Indirect: {
2854	if (AI.getInReg())
2855	Attrs.addAttribute(llvm::Attribute::InReg);
2856
2857	if (AI.getIndirectByVal())
2858	Attrs.addByValAttr(Ty: getTypes().ConvertTypeForMem(T: ParamType));
2859
2860	auto *Decl = ParamType ->getAsRecordDecl();
2861	if (CodeGenOpts.PassByValueIsNoAlias && Decl &&
2862	Decl->getArgPassingRestrictions() ==
2863	RecordArgPassingKind::CanPassInRegs)
2864	// When calling the function, the pointer passed in will be the only
2865	// reference to the underlying object. Mark it accordingly.
2866	Attrs.addAttribute(llvm::Attribute::NoAlias);
2867
2868	// TODO: We could add the byref attribute if not byval, but it would
2869	// require updating many testcases.
2870
2871	CharUnits Align = AI.getIndirectAlign();
2872
2873	// In a byval argument, it is important that the required
2874	// alignment of the type is honored, as LLVM might be creating a
2875	// new* stack object, and needs to know what alignment to give*
2876	// it. (Sometimes it can deduce a sensible alignment on its own,
2877	// but not if clang decides it must emit a packed struct, or the
2878	// user specifies increased alignment requirements.)
2879	//
2880	// This is different from indirect not* byval, where the object*
2881	// exists already, and the align attribute is purely
2882	// informative.
2883	assert(!Align.isZero());
2884
2885	// For now, only add this when we have a byval argument.
2886	// TODO: be less lazy about updating test cases.
2887	if (AI.getIndirectByVal())
2888	Attrs.addAlignmentAttr(Align: Align.getQuantity());
2889
2890	// byval disables readnone and readonly.
2891	AddPotentialArgAccess ();
2892	break;
2893	}
2894	case ABIArgInfo::IndirectAliased: {
2895	CharUnits Align = AI.getIndirectAlign();
2896	Attrs.addByRefAttr(Ty: getTypes().ConvertTypeForMem(T: ParamType));
2897	Attrs.addAlignmentAttr(Align: Align.getQuantity());
2898	break;
2899	}
2900	case ABIArgInfo::Ignore:
2901	case ABIArgInfo::Expand:
2902	case ABIArgInfo::CoerceAndExpand:
2903	break;
2904
2905	case ABIArgInfo::InAlloca:
2906	// inalloca disables readnone and readonly.
2907	AddPotentialArgAccess ();
2908	continue;
2909	}
2910
2911	if (const auto *RefTy = ParamType->getAs<ReferenceType>()) {
2912	QualType PTy = RefTy->getPointeeType();
2913	if (!PTy ->isIncompleteType() && PTy ->isConstantSizeType())
2914	Attrs.addDereferenceableAttr(Bytes: getMinimumObjectSize(Ty: PTy).getQuantity());
2915	if (getTypes().getTargetAddressSpace(PTy) == `0` &&
2916	!CodeGenOpts.NullPointerIsValid)
2917	Attrs.addAttribute(llvm::Attribute::NonNull);
2918	if (PTy ->isObjectType()) {
2919	llvm::Align Alignment =
2920	getNaturalPointeeTypeAlignment(T: ParamType).getAsAlign();
2921	Attrs.addAlignmentAttr(Align: Alignment);
2922	}
2923	}
2924
2925	// From OpenCL spec v3.0.10 section 6.3.5 Alignment of Types:
2926	// > For arguments to a __kernel function declared to be a pointer to a
2927	// > data type, the OpenCL compiler can assume that the pointee is always
2928	// > appropriately aligned as required by the data type.
2929	if (TargetDecl &&
2930	DeviceKernelAttr::isOpenCLSpelling(
2931	TargetDecl->getAttr<DeviceKernelAttr>()) &&
2932	ParamType->isPointerType()) {
2933	QualType PTy = ParamType ->getPointeeType();
2934	if (!PTy ->isIncompleteType() && PTy ->isConstantSizeType()) {
2935	llvm::Align Alignment =
2936	getNaturalPointeeTypeAlignment(T: ParamType).getAsAlign();
2937	Attrs.addAlignmentAttr(Align: Alignment);
2938	}
2939	}
2940
2941	switch (FI.getExtParameterInfo(argIndex: ArgNo).getABI()) {
2942	case ParameterABI::HLSLOut:
2943	case ParameterABI::HLSLInOut:
2944	Attrs.addAttribute(llvm::Attribute::NoAlias);
2945	break;
2946	case ParameterABI::Ordinary:
2947	break;
2948
2949	case ParameterABI::SwiftIndirectResult: {
2950	// Add 'sret' if we haven't already used it for something, but
2951	// only if the result is void.
2952	if (!hasUsedSRet && RetTy ->isVoidType()) {
2953	Attrs.addStructRetAttr(Ty: getTypes().ConvertTypeForMem(T: ParamType));
2954	hasUsedSRet = true;
2955	}
2956
2957	// Add 'noalias' in either case.
2958	Attrs.addAttribute(llvm::Attribute::NoAlias);
2959
2960	// Add 'dereferenceable' and 'alignment'.
2961	auto PTy = ParamType ->getPointeeType();
2962	if (!PTy->isIncompleteType() && PTy->isConstantSizeType()) {
2963	auto info = getContext().getTypeInfoInChars(PTy);
2964	Attrs.addDereferenceableAttr(Bytes: info.Width.getQuantity());
2965	Attrs.addAlignmentAttr(info.Align.getAsAlign());
2966	}
2967	break;
2968	}
2969
2970	case ParameterABI::SwiftErrorResult:
2971	Attrs.addAttribute(llvm::Attribute::SwiftError);
2972	break;
2973
2974	case ParameterABI::SwiftContext:
2975	Attrs.addAttribute(llvm::Attribute::SwiftSelf);
2976	break;
2977
2978	case ParameterABI::SwiftAsyncContext:
2979	Attrs.addAttribute(llvm::Attribute::SwiftAsync);
2980	break;
2981	}
2982
2983	if (FI.getExtParameterInfo(argIndex: ArgNo).isNoEscape())
2984	Attrs.addCapturesAttr(CI: llvm::CaptureInfo::none());
2985
2986	if (Attrs.hasAttributes()) {
2987	unsigned FirstIRArg, NumIRArgs;
2988	std::tie(args&: FirstIRArg, args&: NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
2989	for (unsigned i = `0`; i < NumIRArgs; i++)
2990	ArgAttrs [FirstIRArg + i] = ArgAttrs [FirstIRArg + i].addAttributes(
2991	C&: getLLVMContext(), AS: llvm::AttributeSet::get(C&: getLLVMContext(), B: Attrs));
2992	}
2993	}
2994	assert(ArgNo == FI.arg_size());
2995
2996	AttrList = llvm::AttributeList::get(
2997	C&: getLLVMContext(), FnAttrs: llvm::AttributeSet::get(C&: getLLVMContext(), B: FuncAttrs),
2998	RetAttrs: llvm::AttributeSet::get(C&: getLLVMContext(), B: RetAttrs), ArgAttrs);
2999	}
3000
3001	/// An argument came in as a promoted argument; demote it back to its
3002	/// declared type.
3003	static llvm::Value *emitArgumentDemotion(CodeGenFunction &CGF,
3004	const VarDecl *var,
3005	llvm::Value *value) {
3006	llvm::Type *varType = CGF.ConvertType(var->getType());
3007
3008	// This can happen with promotions that actually don't change the
3009	// underlying type, like the enum promotions.
3010	if (value->getType() == varType)
3011	return value;
3012
3013	assert((varType->isIntegerTy() \|\| varType->isFloatingPointTy()) &&
3014	"unexpected promotion type");
3015
3016	if (isa<llvm::IntegerType>(Val: varType))
3017	return CGF.Builder.CreateTrunc(V: value, DestTy: varType, Name: "arg.unpromote");
3018
3019	return CGF.Builder.CreateFPCast(V: value, DestTy: varType, Name: "arg.unpromote");
3020	}
3021
3022	/// Returns the attribute (either parameter attribute, or function
3023	/// attribute), which declares argument ArgNo to be non-null.
3024	static const NonNullAttr getNonNullAttr(const* Decl FD, const* ParmVarDecl *PVD,
3025	QualType ArgType, unsigned ArgNo) {
3026	// FIXME: __attribute__((nonnull)) can also be applied to:
3027	// - references to pointers, where the pointee is known to be
3028	// nonnull (apparently a Clang extension)
3029	// - transparent unions containing pointers
3030	// In the former case, LLVM IR cannot represent the constraint. In
3031	// the latter case, we have no guarantee that the transparent union
3032	// is in fact passed as a pointer.
3033	if (!ArgType ->isAnyPointerType() && !ArgType ->isBlockPointerType())
3034	return nullptr;
3035	// First, check attribute on parameter itself.
3036	if (PVD) {
3037	if (auto ParmNNAttr = PVD->getAttr<NonNullAttr>())
3038	return ParmNNAttr;
3039	}
3040	// Check function attributes.
3041	if (!FD)
3042	return nullptr;
3043	for (const auto *NNAttr : FD->specific_attrs<NonNullAttr>()) {
3044	if (NNAttr->isNonNull(ArgNo))
3045	return NNAttr;
3046	}
3047	return nullptr;
3048	}
3049
3050	namespace {
3051	struct CopyBackSwiftError final : EHScopeStack::Cleanup {
3052	Address Temp;
3053	Address Arg;
3054	CopyBackSwiftError(Address temp, Address arg) : Temp (temp), Arg (arg) {}
3055	void Emit(CodeGenFunction &CGF, Flags flags) override {
3056	llvm::Value *errorValue = CGF.Builder.CreateLoad(Addr: Temp);
3057	CGF.Builder.CreateStore(Val: errorValue, Addr: Arg);
3058	}
3059	};
3060	} // namespace
3061
3062	void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
3063	llvm::Function *Fn,
3064	const FunctionArgList &Args) {
3065	if (CurCodeDecl && CurCodeDecl->hasAttr<NakedAttr>())
3066	// Naked functions don't have prologues.
3067	return;
3068
3069	// If this is an implicit-return-zero function, go ahead and
3070	// initialize the return value. TODO: it might be nice to have
3071	// a more general mechanism for this that didn't require synthesized
3072	// return statements.
3073	if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(Val: CurCodeDecl)) {
3074	if (FD->hasImplicitReturnZero()) {
3075	QualType RetTy = FD->getReturnType().getUnqualifiedType();
3076	llvm::Type *LLVMTy = CGM.getTypes().ConvertType(T: RetTy);
3077	llvm::Constant *Zero = llvm::Constant::getNullValue(Ty: LLVMTy);
3078	Builder.CreateStore(Val: Zero, Addr: ReturnValue);
3079	}
3080	}
3081
3082	// FIXME: We no longer need the types from FunctionArgList; lift up and
3083	// simplify.
3084
3085	ClangToLLVMArgMapping IRFunctionArgs(CGM.getContext(), FI);
3086	assert(Fn->arg_size() == IRFunctionArgs.totalIRArgs());
3087
3088	// If we're using inalloca, all the memory arguments are GEPs off of the last
3089	// parameter, which is a pointer to the complete memory area.
3090	Address ArgStruct = Address::invalid();
3091	if (IRFunctionArgs.hasInallocaArg())
3092	ArgStruct = Address (Fn->getArg(i: IRFunctionArgs.getInallocaArgNo()),
3093	FI.getArgStruct(), FI.getArgStructAlignment());
3094
3095	// Name the struct return parameter.
3096	if (IRFunctionArgs.hasSRetArg()) {
3097	auto AI = Fn->getArg(i: IRFunctionArgs.getSRetArgNo());
3098	AI->setName("agg.result");
3099	AI->addAttr(llvm::Attribute::NoAlias);
3100	}
3101
3102	// Track if we received the parameter as a pointer (indirect, byval, or
3103	// inalloca). If already have a pointer, EmitParmDecl doesn't need to copy it
3104	// into a local alloca for us.
3105	SmallVector<ParamValue, `16`> ArgVals;
3106	ArgVals.reserve(N: Args.size());
3107
3108	// Create a pointer value for every parameter declaration. This usually
3109	// entails copying one or more LLVM IR arguments into an alloca. Don't push
3110	// any cleanups or do anything that might unwind. We do that separately, so
3111	// we can push the cleanups in the correct order for the ABI.
3112	assert(FI.arg_size() == Args.size() &&
3113	"Mismatch between function signature & arguments.");
3114	unsigned ArgNo = `0`;
3115	CGFunctionInfo::const_arg_iterator info_it = FI.arg_begin();
3116	for (FunctionArgList::const_iterator i = Args.begin(), e = Args.end(); i != e;
3117	++i, ++info_it, ++ArgNo) {
3118	const VarDecl Arg = i;
3119	const ABIArgInfo &ArgI = info_it->info;
3120
3121	bool isPromoted =
3122	isa<ParmVarDecl>(Val: Arg) && cast<ParmVarDecl>(Val: Arg)->isKNRPromoted();
3123	// We are converting from ABIArgInfo type to VarDecl type directly, unless
3124	// the parameter is promoted. In this case we convert to
3125	// CGFunctionInfo::ArgInfo type with subsequent argument demotion.
3126	QualType Ty = isPromoted ? info_it->type : Arg->getType();
3127	assert(hasScalarEvaluationKind(Ty) ==
3128	hasScalarEvaluationKind(Arg->getType()));
3129
3130	unsigned FirstIRArg, NumIRArgs;
3131	std::tie(args&: FirstIRArg, args&: NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
3132
3133	switch (ArgI.getKind()) {
3134	case ABIArgInfo::InAlloca: {
3135	assert(NumIRArgs == `0`);
3136	auto FieldIndex = ArgI.getInAllocaFieldIndex();
3137	Address V =
3138	Builder.CreateStructGEP(ArgStruct, FieldIndex, Arg->getName());
3139	if (ArgI.getInAllocaIndirect())
3140	V = Address (Builder.CreateLoad(Addr: V), ConvertTypeForMem(T: Ty),
3141	getContext().getTypeAlignInChars(T: Ty));
3142	ArgVals.push_back(Elt: ParamValue::forIndirect(addr: V));
3143	break;
3144	}
3145
3146	case ABIArgInfo::Indirect:
3147	case ABIArgInfo::IndirectAliased: {
3148	assert(NumIRArgs == `1`);
3149	Address ParamAddr = makeNaturalAddressForPointer(
3150	Ptr: Fn->getArg(i: FirstIRArg), T: Ty, Alignment: ArgI.getIndirectAlign(), ForPointeeType: false, BaseInfo: nullptr,
3151	TBAAInfo: nullptr, IsKnownNonNull: KnownNonNull);
3152
3153	if (!hasScalarEvaluationKind(T: Ty)) {
3154	// Aggregates and complex variables are accessed by reference. All we
3155	// need to do is realign the value, if requested. Also, if the address
3156	// may be aliased, copy it to ensure that the parameter variable is
3157	// mutable and has a unique adress, as C requires.
3158	if (ArgI.getIndirectRealign() \|\| ArgI.isIndirectAliased()) {
3159	RawAddress AlignedTemp = CreateMemTemp(T: Ty, Name: "coerce");
3160
3161	// Copy from the incoming argument pointer to the temporary with the
3162	// appropriate alignment.
3163	//
3164	// FIXME: We should have a common utility for generating an aggregate
3165	// copy.
3166	CharUnits Size = getContext().getTypeSizeInChars(T: Ty);
3167	Builder.CreateMemCpy(
3168	Dst: AlignedTemp.getPointer(), DstAlign: AlignedTemp.getAlignment().getAsAlign(),
3169	Src: ParamAddr.emitRawPointer(CGF&: *this),
3170	SrcAlign: ParamAddr.getAlignment().getAsAlign(),
3171	Size: llvm::ConstantInt::get(Ty: IntPtrTy, V: Size.getQuantity()));
3172	ParamAddr = AlignedTemp;
3173	}
3174	ArgVals.push_back(Elt: ParamValue::forIndirect(addr: ParamAddr));
3175	} else {
3176	// Load scalar value from indirect argument.
3177	llvm::Value *V =
3178	EmitLoadOfScalar(ParamAddr, false, Ty, Arg->getBeginLoc());
3179
3180	if (isPromoted)
3181	V = emitArgumentDemotion(CGF&: *this, var: Arg, value: V);
3182	ArgVals.push_back(Elt: ParamValue::forDirect(value: V));
3183	}
3184	break;
3185	}
3186
3187	case ABIArgInfo::Extend:
3188	case ABIArgInfo::Direct: {
3189	auto AI = Fn->getArg(i: FirstIRArg);
3190	llvm::Type *LTy = ConvertType(Arg->getType());
3191
3192	// Prepare parameter attributes. So far, only attributes for pointer
3193	// parameters are prepared. See
3194	// http://llvm.org/docs/LangRef.html#paramattrs.
3195	if (ArgI.getDirectOffset() == `0` && LTy->isPointerTy() &&
3196	ArgI.getCoerceToType()->isPointerTy()) {
3197	assert(NumIRArgs == `1`);
3198
3199	if (const ParmVarDecl *PVD = dyn_cast<ParmVarDecl>(Val: Arg)) {
3200	// Set `nonnull` attribute if any.
3201	if (getNonNullAttr(CurCodeDecl, PVD, PVD->getType(),
3202	PVD->getFunctionScopeIndex()) &&
3203	!CGM.getCodeGenOpts().NullPointerIsValid)
3204	AI->addAttr(llvm::Attribute::NonNull);
3205
3206	QualType OTy = PVD->getOriginalType();
3207	if (const auto *ArrTy = getContext().getAsConstantArrayType(T: OTy)) {
3208	// A C99 array parameter declaration with the static keyword also
3209	// indicates dereferenceability, and if the size is constant we can
3210	// use the dereferenceable attribute (which requires the size in
3211	// bytes).
3212	if (ArrTy->getSizeModifier() == ArraySizeModifier::Static) {
3213	QualType ETy = ArrTy->getElementType();
3214	llvm::Align Alignment =
3215	CGM.getNaturalTypeAlignment(T: ETy).getAsAlign();
3216	AI->addAttrs(B&: llvm::AttrBuilder (getLLVMContext())
3217	.addAlignmentAttr(Align: Alignment));
3218	uint64_t ArrSize = ArrTy->getZExtSize();
3219	if (!ETy ->isIncompleteType() && ETy ->isConstantSizeType() &&
3220	ArrSize) {
3221	llvm::AttrBuilder Attrs(getLLVMContext());
3222	Attrs.addDereferenceableAttr(
3223	Bytes: getContext().getTypeSizeInChars(T: ETy).getQuantity() *
3224	ArrSize);
3225	AI->addAttrs(B&: Attrs);
3226	} else if (getContext().getTargetInfo().getNullPointerValue(
3227	AddrSpace: ETy.getAddressSpace()) == `0` &&
3228	!CGM.getCodeGenOpts().NullPointerIsValid) {
3229	AI->addAttr(llvm::Attribute::NonNull);
3230	}
3231	}
3232	} else if (const auto *ArrTy =
3233	getContext().getAsVariableArrayType(T: OTy)) {
3234	// For C99 VLAs with the static keyword, we don't know the size so
3235	// we can't use the dereferenceable attribute, but in addrspace(0)
3236	// we know that it must be nonnull.
3237	if (ArrTy->getSizeModifier() == ArraySizeModifier::Static) {
3238	QualType ETy = ArrTy->getElementType();
3239	llvm::Align Alignment =
3240	CGM.getNaturalTypeAlignment(T: ETy).getAsAlign();
3241	AI->addAttrs(B&: llvm::AttrBuilder (getLLVMContext())
3242	.addAlignmentAttr(Align: Alignment));
3243	if (!getTypes().getTargetAddressSpace(ETy) &&
3244	!CGM.getCodeGenOpts().NullPointerIsValid)
3245	AI->addAttr(llvm::Attribute::NonNull);
3246	}
3247	}
3248
3249	// Set `align` attribute if any.
3250	const auto *AVAttr = PVD->getAttr<AlignValueAttr>();
3251	if (!AVAttr)
3252	if (const auto *TOTy = OTy->getAs<TypedefType>())
3253	AVAttr = TOTy->getDecl()->getAttr<AlignValueAttr>();
3254	if (AVAttr && !SanOpts.has(K: SanitizerKind::Alignment)) {
3255	// If alignment-assumption sanitizer is enabled, we do not* add*
3256	// alignment attribute here, but emit normal alignment assumption,
3257	// so the UBSAN check could function.
3258	llvm::ConstantInt *AlignmentCI =
3259	cast<llvm::ConstantInt>(EmitScalarExpr(E: AVAttr->getAlignment()));
3260	uint64_t AlignmentInt =
3261	AlignmentCI->getLimitedValue(Limit: llvm::Value::MaximumAlignment);
3262	if (AI->getParamAlign().valueOrOne() < AlignmentInt) {
3263	AI->removeAttr(llvm::Attribute::AttrKind::Alignment);
3264	AI->addAttrs(B&: llvm::AttrBuilder (getLLVMContext())
3265	.addAlignmentAttr(Align: llvm::Align (AlignmentInt)));
3266	}
3267	}
3268	}
3269
3270	// Set 'noalias' if an argument type has the `restrict` qualifier.
3271	if (Arg->getType().isRestrictQualified())
3272	AI->addAttr(llvm::Attribute::NoAlias);
3273	}
3274
3275	// Prepare the argument value. If we have the trivial case, handle it
3276	// with no muss and fuss.
3277	if (!isa<llvm::StructType>(Val: ArgI.getCoerceToType()) &&
3278	ArgI.getCoerceToType() == ConvertType(T: Ty) &&
3279	ArgI.getDirectOffset() == `0`) {
3280	assert(NumIRArgs == `1`);
3281
3282	// LLVM expects swifterror parameters to be used in very restricted
3283	// ways. Copy the value into a less-restricted temporary.
3284	llvm::Value *V = AI;
3285	if (FI.getExtParameterInfo(argIndex: ArgNo).getABI() ==
3286	ParameterABI::SwiftErrorResult) {
3287	QualType pointeeTy = Ty ->getPointeeType();
3288	assert(pointeeTy ->isPointerType());
3289	RawAddress temp =
3290	CreateMemTemp(pointeeTy, getPointerAlign(), "swifterror.temp");
3291	Address arg = makeNaturalAddressForPointer(
3292	Ptr: V, T: pointeeTy, Alignment: getContext().getTypeAlignInChars(T: pointeeTy));
3293	llvm::Value *incomingErrorValue = Builder.CreateLoad(Addr: arg);
3294	Builder.CreateStore(Val: incomingErrorValue, Addr: temp);
3295	V = temp.getPointer();
3296
3297	// Push a cleanup to copy the value back at the end of the function.
3298	// The convention does not guarantee that the value will be written
3299	// back if the function exits with an unwind exception.
3300	EHStack.pushCleanup<CopyBackSwiftError>(Kind: NormalCleanup, A: temp, A: arg);
3301	}
3302
3303	// Ensure the argument is the correct type.
3304	if (V->getType() != ArgI.getCoerceToType())
3305	V = Builder.CreateBitCast(V, DestTy: ArgI.getCoerceToType());
3306
3307	if (isPromoted)
3308	V = emitArgumentDemotion(CGF&: *this, var: Arg, value: V);
3309
3310	// Because of merging of function types from multiple decls it is
3311	// possible for the type of an argument to not match the corresponding
3312	// type in the function type. Since we are codegening the callee
3313	// in here, add a cast to the argument type.
3314	llvm::Type *LTy = ConvertType(Arg->getType());
3315	if (V->getType() != LTy)
3316	V = Builder.CreateBitCast(V, DestTy: LTy);
3317
3318	ArgVals.push_back(Elt: ParamValue::forDirect(value: V));
3319	break;
3320	}
3321
3322	// VLST arguments are coerced to VLATs at the function boundary for
3323	// ABI consistency. If this is a VLST that was coerced to
3324	// a VLAT at the function boundary and the types match up, use
3325	// llvm.vector.extract to convert back to the original VLST.
3326	if (auto *VecTyTo = dyn_cast<llvm::FixedVectorType>(ConvertType(Ty))) {
3327	llvm::Value *ArgVal = Fn->getArg(i: FirstIRArg);
3328	if (auto *VecTyFrom =
3329	dyn_cast<llvm::ScalableVectorType>(Val: ArgVal->getType())) {
3330	auto [Coerced, Extracted] = CoerceScalableToFixed(
3331	*this, VecTyTo, VecTyFrom, ArgVal, Arg->getName());
3332	if (Extracted) {
3333	assert(NumIRArgs == `1`);
3334	ArgVals.push_back(Elt: ParamValue::forDirect(value: Coerced));
3335	break;
3336	}
3337	}
3338	}
3339
3340	// Struct of fixed-length vectors and struct of array of fixed-length
3341	// vector in VLS calling convention are coerced to vector tuple
3342	// type(represented as TargetExtType) and scalable vector type
3343	// respectively, they're no longer handled as struct.
3344	if (ArgI.isDirect() && isa<llvm::StructType>(Val: ConvertType(T: Ty)) &&
3345	(isa<llvm::TargetExtType>(Val: ArgI.getCoerceToType()) \|\|
3346	isa<llvm::ScalableVectorType>(Val: ArgI.getCoerceToType()))) {
3347	ArgVals.push_back(Elt: ParamValue::forDirect(value: AI));
3348	break;
3349	}
3350
3351	llvm::StructType *STy =
3352	dyn_cast<llvm::StructType>(Val: ArgI.getCoerceToType());
3353	Address Alloca =
3354	CreateMemTemp(Ty, getContext().getDeclAlign(Arg), Arg->getName());
3355
3356	// Pointer to store into.
3357	Address Ptr = emitAddressAtOffset(CGF&: *this, addr: Alloca, info: ArgI);
3358
3359	// Fast-isel and the optimizer generally like scalar values better than
3360	// FCAs, so we flatten them if this is safe to do for this argument.
3361	if (ArgI.isDirect() && ArgI.getCanBeFlattened() && STy &&
3362	STy->getNumElements() > `1`) {
3363	llvm::TypeSize StructSize = CGM.getDataLayout().getTypeAllocSize(Ty: STy);
3364	llvm::TypeSize PtrElementSize =
3365	CGM.getDataLayout().getTypeAllocSize(Ty: Ptr.getElementType());
3366	if (StructSize.isScalable()) {
3367	assert(STy->containsHomogeneousScalableVectorTypes() &&
3368	"ABI only supports structure with homogeneous scalable vector "
3369	"type");
3370	assert(StructSize == PtrElementSize &&
3371	"Only allow non-fractional movement of structure with"
3372	"homogeneous scalable vector type");
3373	assert(STy->getNumElements() == NumIRArgs);
3374
3375	llvm::Value *LoadedStructValue = llvm::PoisonValue::get(T: STy);
3376	for (unsigned i = `0`, e = STy->getNumElements(); i != e; ++i) {
3377	auto *AI = Fn->getArg(i: FirstIRArg + i);
3378	AI->setName(Arg->getName() + ".coerce" + Twine(i));
3379	LoadedStructValue =
3380	Builder.CreateInsertValue(Agg: LoadedStructValue, Val: AI, Idxs: i);
3381	}
3382
3383	Builder.CreateStore(Val: LoadedStructValue, Addr: Ptr);
3384	} else {
3385	uint64_t SrcSize = StructSize.getFixedValue();
3386	uint64_t DstSize = PtrElementSize.getFixedValue();
3387
3388	Address AddrToStoreInto = Address::invalid();
3389	if (SrcSize <= DstSize) {
3390	AddrToStoreInto = Ptr.withElementType(ElemTy: STy);
3391	} else {
3392	AddrToStoreInto =
3393	CreateTempAlloca(Ty: STy, align: Alloca.getAlignment(), Name: "coerce");
3394	}
3395
3396	assert(STy->getNumElements() == NumIRArgs);
3397	for (unsigned i = `0`, e = STy->getNumElements(); i != e; ++i) {
3398	auto AI = Fn->getArg(i: FirstIRArg + i);
3399	AI->setName(Arg->getName() + ".coerce" + Twine(i));
3400	Address EltPtr = Builder.CreateStructGEP(Addr: AddrToStoreInto, Index: i);
3401	Builder.CreateStore(Val: AI, Addr: EltPtr);
3402	}
3403
3404	if (SrcSize > DstSize) {
3405	Builder.CreateMemCpy(Dest: Ptr, Src: AddrToStoreInto, Size: DstSize);
3406	}
3407	}
3408	} else {
3409	// Simple case, just do a coerced store of the argument into the alloca.
3410	assert(NumIRArgs == `1`);
3411	auto AI = Fn->getArg(i: FirstIRArg);
3412	AI->setName(Arg->getName() + ".coerce");
3413	CreateCoercedStore(
3414	Src: AI, Dst: Ptr,
3415	DstSize: llvm::TypeSize::getFixed(
3416	ExactSize: getContext().getTypeSizeInChars(T: Ty).getQuantity() -
3417	ArgI.getDirectOffset()),
3418	/DstIsVolatile=/false);
3419	}
3420
3421	// Match to what EmitParmDecl is expecting for this type.
3422	if (CodeGenFunction::hasScalarEvaluationKind(T: Ty)) {
3423	llvm::Value *V =
3424	EmitLoadOfScalar(Alloca, false, Ty, Arg->getBeginLoc());
3425	if (isPromoted)
3426	V = emitArgumentDemotion(CGF&: *this, var: Arg, value: V);
3427	ArgVals.push_back(Elt: ParamValue::forDirect(value: V));
3428	} else {
3429	ArgVals.push_back(Elt: ParamValue::forIndirect(addr: Alloca));
3430	}
3431	break;
3432	}
3433
3434	case ABIArgInfo::CoerceAndExpand: {
3435	// Reconstruct into a temporary.
3436	Address alloca = CreateMemTemp(T: Ty, Align: getContext().getDeclAlign(Arg));
3437	ArgVals.push_back(Elt: ParamValue::forIndirect(addr: alloca));
3438
3439	auto coercionType = ArgI.getCoerceAndExpandType();
3440	auto unpaddedCoercionType = ArgI.getUnpaddedCoerceAndExpandType();
3441	auto *unpaddedStruct = dyn_cast<llvm::StructType>(Val: unpaddedCoercionType);
3442
3443	alloca = alloca.withElementType(ElemTy: coercionType);
3444
3445	unsigned argIndex = FirstIRArg;
3446	unsigned unpaddedIndex = `0`;
3447	for (unsigned i = `0`, e = coercionType->getNumElements(); i != e; ++i) {
3448	llvm::Type *eltType = coercionType->getElementType(N: i);
3449	if (ABIArgInfo::isPaddingForCoerceAndExpand(eltType))
3450	continue;
3451
3452	auto eltAddr = Builder.CreateStructGEP(Addr: alloca, Index: i);
3453	llvm::Value *elt = Fn->getArg(i: argIndex++);
3454
3455	auto paramType = unpaddedStruct
3456	? unpaddedStruct->getElementType(N: unpaddedIndex++)
3457	: unpaddedCoercionType;
3458
3459	if (auto *VecTyTo = dyn_cast<llvm::FixedVectorType>(Val: eltType)) {
3460	if (auto *VecTyFrom = dyn_cast<llvm::ScalableVectorType>(Val: paramType)) {
3461	bool Extracted;
3462	std::tie(args&: elt, args&: Extracted) = CoerceScalableToFixed(
3463	CGF&: *this, ToTy: VecTyTo, FromTy: VecTyFrom, V: elt, Name: elt->getName());
3464	assert(Extracted && "Unexpected scalable to fixed vector coercion");
3465	}
3466	}
3467	Builder.CreateStore(Val: elt, Addr: eltAddr);
3468	}
3469	assert(argIndex == FirstIRArg + NumIRArgs);
3470	break;
3471	}
3472
3473	case ABIArgInfo::Expand: {
3474	// If this structure was expanded into multiple arguments then
3475	// we need to create a temporary and reconstruct it from the
3476	// arguments.
3477	Address Alloca = CreateMemTemp(T: Ty, Align: getContext().getDeclAlign(Arg));
3478	LValue LV = MakeAddrLValue(Addr: Alloca, T: Ty);
3479	ArgVals.push_back(Elt: ParamValue::forIndirect(addr: Alloca));
3480
3481	auto FnArgIter = Fn->arg_begin() + FirstIRArg;
3482	ExpandTypeFromArgs(Ty, LV, AI&: FnArgIter);
3483	assert(FnArgIter == Fn->arg_begin() + FirstIRArg + NumIRArgs);
3484	for (unsigned i = `0`, e = NumIRArgs; i != e; ++i) {
3485	auto AI = Fn->getArg(i: FirstIRArg + i);
3486	AI->setName(Arg->getName() + "." + Twine(i));
3487	}
3488	break;
3489	}
3490
3491	case ABIArgInfo::Ignore:
3492	assert(NumIRArgs == `0`);
3493	// Initialize the local variable appropriately.
3494	if (!hasScalarEvaluationKind(T: Ty)) {
3495	ArgVals.push_back(Elt: ParamValue::forIndirect(addr: CreateMemTemp(T: Ty)));
3496	} else {
3497	llvm::Value *U = llvm::UndefValue::get(T: ConvertType(Arg->getType()));
3498	ArgVals.push_back(Elt: ParamValue::forDirect(value: U));
3499	}
3500	break;
3501	}
3502	}
3503
3504	if (getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee()) {
3505	for (int I = Args.size() - `1`; I >= `0`; --I)
3506	EmitParmDecl(D: *Args [I], Arg: ArgVals [I], ArgNo: I + `1`);
3507	} else {
3508	for (unsigned I = `0`, E = Args.size(); I != E; ++I)
3509	EmitParmDecl(D: *Args [I], Arg: ArgVals [I], ArgNo: I + `1`);
3510	}
3511	}
3512
3513	static void eraseUnusedBitCasts(llvm::Instruction *insn) {
3514	while (insn->use_empty()) {
3515	llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(Val: insn);
3516	if (!bitcast)
3517	return;
3518
3519	// This is "safe" because we would have used a ConstantExpr otherwise.
3520	insn = cast<llvm::Instruction>(Val: bitcast->getOperand(i_nocapture: `0`));
3521	bitcast->eraseFromParent();
3522	}
3523	}
3524
3525	/// Try to emit a fused autorelease of a return result.
3526	static llvm::Value *tryEmitFusedAutoreleaseOfResult(CodeGenFunction &CGF,
3527	llvm::Value *result) {
3528	// We must be immediately followed the cast.
3529	llvm::BasicBlock *BB = CGF.Builder.GetInsertBlock();
3530	if (BB->empty())
3531	return nullptr;
3532	if (&BB->back() != result)
3533	return nullptr;
3534
3535	llvm::Type *resultType = result->getType();
3536
3537	// result is in a BasicBlock and is therefore an Instruction.
3538	llvm::Instruction *generator = cast<llvm::Instruction>(Val: result);
3539
3540	SmallVector<llvm::Instruction *, `4`> InstsToKill;
3541
3542	// Look for:
3543	// %generator = bitcast %type1* %generator2 to %type2*
3544	while (llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(Val: generator)) {
3545	// We would have emitted this as a constant if the operand weren't
3546	// an Instruction.
3547	generator = cast<llvm::Instruction>(Val: bitcast->getOperand(i_nocapture: `0`));
3548
3549	// Require the generator to be immediately followed by the cast.
3550	if (generator->getNextNode() != bitcast)
3551	return nullptr;
3552
3553	InstsToKill.push_back(Elt: bitcast);
3554	}
3555
3556	// Look for:
3557	// %generator = call i8 @objc_retain(i8* %originalResult)*
3558	// or
3559	// %generator = call i8 @objc_retainAutoreleasedReturnValue(i8* %originalResult)*
3560	llvm::CallInst *call = dyn_cast<llvm::CallInst>(Val: generator);
3561	if (!call)
3562	return nullptr;
3563
3564	bool doRetainAutorelease;
3565
3566	if (call->getCalledOperand() == CGF.CGM.getObjCEntrypoints().objc_retain) {
3567	doRetainAutorelease = true;
3568	} else if (call->getCalledOperand() ==
3569	CGF.CGM.getObjCEntrypoints().objc_retainAutoreleasedReturnValue) {
3570	doRetainAutorelease = false;
3571
3572	// If we emitted an assembly marker for this call (and the
3573	// ARCEntrypoints field should have been set if so), go looking
3574	// for that call. If we can't find it, we can't do this
3575	// optimization. But it should always be the immediately previous
3576	// instruction, unless we needed bitcasts around the call.
3577	if (CGF.CGM.getObjCEntrypoints().retainAutoreleasedReturnValueMarker) {
3578	llvm::Instruction *prev = call->getPrevNode();
3579	assert(prev);
3580	if (isa<llvm::BitCastInst>(Val: prev)) {
3581	prev = prev->getPrevNode();
3582	assert(prev);
3583	}
3584	assert(isa<llvm::CallInst>(prev));
3585	assert(cast<llvm::CallInst>(prev)->getCalledOperand() ==
3586	CGF.CGM.getObjCEntrypoints().retainAutoreleasedReturnValueMarker);
3587	InstsToKill.push_back(Elt: prev);
3588	}
3589	} else {
3590	return nullptr;
3591	}
3592
3593	result = call->getArgOperand(i: `0`);
3594	InstsToKill.push_back(Elt: call);
3595
3596	// Keep killing bitcasts, for sanity. Note that we no longer care
3597	// about precise ordering as long as there's exactly one use.
3598	while (llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(Val: result)) {
3599	if (!bitcast->hasOneUse())
3600	break;
3601	InstsToKill.push_back(Elt: bitcast);
3602	result = bitcast->getOperand(i_nocapture: `0`);
3603	}
3604
3605	// Delete all the unnecessary instructions, from latest to earliest.
3606	for (auto *I : InstsToKill)
3607	I->eraseFromParent();
3608
3609	// Do the fused retain/autorelease if we were asked to.
3610	if (doRetainAutorelease)
3611	result = CGF.EmitARCRetainAutoreleaseReturnValue(value: result);
3612
3613	// Cast back to the result type.
3614	return CGF.Builder.CreateBitCast(V: result, DestTy: resultType);
3615	}
3616
3617	/// If this is a +1 of the value of an immutable 'self', remove it.
3618	static llvm::Value *tryRemoveRetainOfSelf(CodeGenFunction &CGF,
3619	llvm::Value *result) {
3620	// This is only applicable to a method with an immutable 'self'.
3621	const ObjCMethodDecl *method =
3622	dyn_cast_or_null<ObjCMethodDecl>(Val: CGF.CurCodeDecl);
3623	if (!method)
3624	return nullptr;
3625	const VarDecl *self = method->getSelfDecl();
3626	if (!self->getType().isConstQualified())
3627	return nullptr;
3628
3629	// Look for a retain call. Note: stripPointerCasts looks through returned arg
3630	// functions, which would cause us to miss the retain.
3631	llvm::CallInst *retainCall = dyn_cast<llvm::CallInst>(Val: result);
3632	if (!retainCall \|\| retainCall->getCalledOperand() !=
3633	CGF.CGM.getObjCEntrypoints().objc_retain)
3634	return nullptr;
3635
3636	// Look for an ordinary load of 'self'.
3637	llvm::Value *retainedValue = retainCall->getArgOperand(i: `0`);
3638	llvm::LoadInst *load =
3639	dyn_cast<llvm::LoadInst>(Val: retainedValue->stripPointerCasts());
3640	if (!load \|\| load->isAtomic() \|\| load->isVolatile() \|\|
3641	load->getPointerOperand() != CGF.GetAddrOfLocalVar(VD: self).getBasePointer())
3642	return nullptr;
3643
3644	// Okay! Burn it all down. This relies for correctness on the
3645	// assumption that the retain is emitted as part of the return and
3646	// that thereafter everything is used "linearly".
3647	llvm::Type *resultType = result->getType();
3648	eraseUnusedBitCasts(insn: cast<llvm::Instruction>(Val: result));
3649	assert(retainCall->use_empty());
3650	retainCall->eraseFromParent();
3651	eraseUnusedBitCasts(insn: cast<llvm::Instruction>(Val: retainedValue));
3652
3653	return CGF.Builder.CreateBitCast(V: load, DestTy: resultType);
3654	}
3655
3656	/// Emit an ARC autorelease of the result of a function.
3657	///
3658	/// \return the value to actually return from the function
3659	static llvm::Value *emitAutoreleaseOfResult(CodeGenFunction &CGF,
3660	llvm::Value *result) {
3661	// If we're returning 'self', kill the initial retain. This is a
3662	// heuristic attempt to "encourage correctness" in the really unfortunate
3663	// case where we have a return of self during a dealloc and we desperately
3664	// need to avoid the possible autorelease.
3665	if (llvm::Value *self = tryRemoveRetainOfSelf(CGF, result))
3666	return self;
3667
3668	// At -O0, try to emit a fused retain/autorelease.
3669	if (CGF.shouldUseFusedARCCalls())
3670	if (llvm::Value *fused = tryEmitFusedAutoreleaseOfResult(CGF, result))
3671	return fused;
3672
3673	return CGF.EmitARCAutoreleaseReturnValue(value: result);
3674	}
3675
3676	/// Heuristically search for a dominating store to the return-value slot.
3677	static llvm::StoreInst *findDominatingStoreToReturnValue(CodeGenFunction &CGF) {
3678	llvm::Value *ReturnValuePtr = CGF.ReturnValue.getBasePointer();
3679
3680	// Check if a User is a store which pointerOperand is the ReturnValue.
3681	// We are looking for stores to the ReturnValue, not for stores of the
3682	// ReturnValue to some other location.
3683	auto GetStoreIfValid = [&CGF,
3684	ReturnValuePtr](llvm::User U) -> llvm::StoreInst {
3685	auto *SI = dyn_cast<llvm::StoreInst>(Val: U);
3686	if (!SI \|\| SI->getPointerOperand() != ReturnValuePtr \|\|
3687	SI->getValueOperand()->getType() != CGF.ReturnValue.getElementType())
3688	return nullptr;
3689	// These aren't actually possible for non-coerced returns, and we
3690	// only care about non-coerced returns on this code path.
3691	// All memory instructions inside __try block are volatile.
3692	assert(!SI->isAtomic() &&
3693	(!SI->isVolatile() \|\| CGF.currentFunctionUsesSEHTry()));
3694	return SI;
3695	};
3696	// If there are multiple uses of the return-value slot, just check
3697	// for something immediately preceding the IP. Sometimes this can
3698	// happen with how we generate implicit-returns; it can also happen
3699	// with noreturn cleanups.
3700	if (!ReturnValuePtr->hasOneUse()) {
3701	llvm::BasicBlock *IP = CGF.Builder.GetInsertBlock();
3702	if (IP->empty())
3703	return nullptr;
3704
3705	// Look at directly preceding instruction, skipping bitcasts, lifetime
3706	// markers, and fake uses and their operands.
3707	const llvm::Instruction LoadIntoFakeUse = nullptr*;
3708	for (llvm::Instruction &I : llvm::reverse(C&: *IP)) {
3709	// Ignore instructions that are just loads for fake uses; the load should
3710	// immediately precede the fake use, so we only need to remember the
3711	// operand for the last fake use seen.
3712	if (LoadIntoFakeUse == &I)
3713	continue;
3714	if (isa<llvm::BitCastInst>(Val: &I))
3715	continue;
3716	if (auto *II = dyn_cast<llvm::IntrinsicInst>(Val: &I)) {
3717	if (II->getIntrinsicID() == llvm::Intrinsic::lifetime_end)
3718	continue;
3719
3720	if (II->getIntrinsicID() == llvm::Intrinsic::fake_use) {
3721	LoadIntoFakeUse = dyn_cast<llvm::Instruction>(Val: II->getArgOperand(i: `0`));
3722	continue;
3723	}
3724	}
3725	return GetStoreIfValid (&I);
3726	}
3727	return nullptr;
3728	}
3729
3730	llvm::StoreInst *store = GetStoreIfValid (ReturnValuePtr->user_back());
3731	if (!store)
3732	return nullptr;
3733
3734	// Now do a first-and-dirty dominance check: just walk up the
3735	// single-predecessors chain from the current insertion point.
3736	llvm::BasicBlock *StoreBB = store->getParent();
3737	llvm::BasicBlock *IP = CGF.Builder.GetInsertBlock();
3738	llvm::SmallPtrSet<llvm::BasicBlock *, `4`> SeenBBs;
3739	while (IP != StoreBB) {
3740	if (!SeenBBs.insert(Ptr: IP).second \|\| !(IP = IP->getSinglePredecessor()))
3741	return nullptr;
3742	}
3743
3744	// Okay, the store's basic block dominates the insertion point; we
3745	// can do our thing.
3746	return store;
3747	}
3748
3749	// Helper functions for EmitCMSEClearRecord
3750
3751	// Set the bits corresponding to a field having width `BitWidth` and located at
3752	// offset `BitOffset` (from the least significant bit) within a storage unit of
3753	// `Bits.size()` bytes. Each element of `Bits` corresponds to one target byte.
3754	// Use little-endian layout, i.e.`Bits[0]` is the LSB.
3755	static void setBitRange(SmallVectorImpl<uint64_t> &Bits, int BitOffset,
3756	int BitWidth, int CharWidth) {
3757	assert(CharWidth <= `64`);
3758	assert(static_cast<unsigned>(BitWidth) <= Bits.size() * CharWidth);
3759
3760	int Pos = `0`;
3761	if (BitOffset >= CharWidth) {
3762	Pos += BitOffset / CharWidth;
3763	BitOffset = BitOffset % CharWidth;
3764	}
3765
3766	const uint64_t Used = (uint64_t(`1`) << CharWidth) - `1`;
3767	if (BitOffset + BitWidth >= CharWidth) {
3768	Bits [Pos++] \|= (Used << BitOffset) & Used;
3769	BitWidth -= CharWidth - BitOffset;
3770	BitOffset = `0`;
3771	}
3772
3773	while (BitWidth >= CharWidth) {
3774	Bits [Pos++] = Used;
3775	BitWidth -= CharWidth;
3776	}
3777
3778	if (BitWidth > `0`)
3779	Bits [Pos++] \|= (Used >> (CharWidth - BitWidth)) << BitOffset;
3780	}
3781
3782	// Set the bits corresponding to a field having width `BitWidth` and located at
3783	// offset `BitOffset` (from the least significant bit) within a storage unit of
3784	// `StorageSize` bytes, located at `StorageOffset` in `Bits`. Each element of
3785	// `Bits` corresponds to one target byte. Use target endian layout.
3786	static void setBitRange(SmallVectorImpl<uint64_t> &Bits, int StorageOffset,
3787	int StorageSize, int BitOffset, int BitWidth,
3788	int CharWidth, bool BigEndian) {
3789
3790	SmallVector<uint64_t, `8`> TmpBits(StorageSize);
3791	setBitRange(Bits&: TmpBits, BitOffset, BitWidth, CharWidth);
3792
3793	if (BigEndian)
3794	std::reverse(first: TmpBits.begin(), last: TmpBits.end());
3795
3796	for (uint64_t V : TmpBits)
3797	Bits [StorageOffset++] \|= V;
3798	}
3799
3800	static void setUsedBits(CodeGenModule &, QualType, int,
3801	SmallVectorImpl<uint64_t> &);
3802
3803	// Set the bits in `Bits`, which correspond to the value representations of
3804	// the actual members of the record type `RTy`. Note that this function does
3805	// not handle base classes, virtual tables, etc, since they cannot happen in
3806	// CMSE function arguments or return. The bit mask corresponds to the target
3807	// memory layout, i.e. it's endian dependent.
3808	static void setUsedBits(CodeGenModule &CGM, const RecordType RTy, int* Offset,
3809	SmallVectorImpl<uint64_t> &Bits) {
3810	ASTContext &Context = CGM.getContext();
3811	int CharWidth = Context.getCharWidth();
3812	const RecordDecl *RD = RTy->getDecl()->getDefinition();
3813	const ASTRecordLayout &ASTLayout = Context.getASTRecordLayout(D: RD);
3814	const CGRecordLayout &Layout = CGM.getTypes().getCGRecordLayout(RD);
3815
3816	int Idx = `0`;
3817	for (auto I = RD->field_begin(), E = RD->field_end(); I != E; ++I, ++Idx) {
3818	const FieldDecl F = I;
3819
3820	if (F->isUnnamedBitField() \|\| F->isZeroLengthBitField() \|\|
3821	F->getType()->isIncompleteArrayType())
3822	continue;
3823
3824	if (F->isBitField()) {
3825	const CGBitFieldInfo &BFI = Layout.getBitFieldInfo(FD: F);
3826	setBitRange(Bits, StorageOffset: Offset + BFI.StorageOffset.getQuantity(),
3827	StorageSize: BFI.StorageSize / CharWidth, BitOffset: BFI.Offset, BitWidth: BFI.Size, CharWidth,
3828	BigEndian: CGM.getDataLayout().isBigEndian());
3829	continue;
3830	}
3831
3832	setUsedBits(CGM, F->getType(),
3833	Offset + ASTLayout.getFieldOffset(FieldNo: Idx) / CharWidth, Bits);
3834	}
3835	}
3836
3837	// Set the bits in `Bits`, which correspond to the value representations of
3838	// the elements of an array type `ATy`.
3839	static void setUsedBits(CodeGenModule &CGM, const ConstantArrayType *ATy,
3840	int Offset, SmallVectorImpl<uint64_t> &Bits) {
3841	const ASTContext &Context = CGM.getContext();
3842
3843	QualType ETy = Context.getBaseElementType(ATy);
3844	int Size = Context.getTypeSizeInChars(T: ETy).getQuantity();
3845	SmallVector<uint64_t, `4`> TmpBits(Size);
3846	setUsedBits(CGM, ETy, `0`, TmpBits);
3847
3848	for (int I = `0`, N = Context.getConstantArrayElementCount(CA: ATy); I < N; ++I) {
3849	auto Src = TmpBits.begin();
3850	auto Dst = Bits.begin() + Offset + I * Size;
3851	for (int J = `0`; J < Size; ++J)
3852	Dst++ \|= Src++;
3853	}
3854	}
3855
3856	// Set the bits in `Bits`, which correspond to the value representations of
3857	// the type `QTy`.
3858	static void setUsedBits(CodeGenModule &CGM, QualType QTy, int Offset,
3859	SmallVectorImpl<uint64_t> &Bits) {
3860	if (const auto *RTy = QTy ->getAs<RecordType>())
3861	return setUsedBits(CGM, RTy, Offset, Bits);
3862
3863	ASTContext &Context = CGM.getContext();
3864	if (const auto *ATy = Context.getAsConstantArrayType(T: QTy))
3865	return setUsedBits(CGM, ATy, Offset, Bits);
3866
3867	int Size = Context.getTypeSizeInChars(T: QTy).getQuantity();
3868	if (Size <= `0`)
3869	return;
3870
3871	std::fill_n(first: Bits.begin() + Offset, n: Size,
3872	value: (uint64_t(`1`) << Context.getCharWidth()) - `1`);
3873	}
3874
3875	static uint64_t buildMultiCharMask(const SmallVectorImpl<uint64_t> &Bits,
3876	int Pos, int Size, int CharWidth,
3877	bool BigEndian) {
3878	assert(Size > `0`);
3879	uint64_t Mask = `0`;
3880	if (BigEndian) {
3881	for (auto P = Bits.begin() + Pos, E = Bits.begin() + Pos + Size; P != E;
3882	++P)
3883	Mask = (Mask << CharWidth) \| *P;
3884	} else {
3885	auto P = Bits.begin() + Pos + Size, End = Bits.begin() + Pos;
3886	do
3887	Mask = (Mask << CharWidth) \| *--P;
3888	while (P != End);
3889	}
3890	return Mask;
3891	}
3892
3893	// Emit code to clear the bits in a record, which aren't a part of any user
3894	// declared member, when the record is a function return.
3895	llvm::Value CodeGenFunction::EmitCMSEClearRecord(llvm::Value Src,
3896	llvm::IntegerType *ITy,
3897	QualType QTy) {
3898	assert(Src->getType() == ITy);
3899	assert(ITy->getScalarSizeInBits() <= `64`);
3900
3901	const llvm::DataLayout &DataLayout = CGM.getDataLayout();
3902	int Size = DataLayout.getTypeStoreSize(Ty: ITy);
3903	SmallVector<uint64_t, `4`> Bits(Size);
3904	setUsedBits(CGM, RTy: QTy ->castAs<RecordType>(), Offset: `0`, Bits);
3905
3906	int CharWidth = CGM.getContext().getCharWidth();
3907	uint64_t Mask =
3908	buildMultiCharMask(Bits, Pos: `0`, Size, CharWidth, BigEndian: DataLayout.isBigEndian());
3909
3910	return Builder.CreateAnd(LHS: Src, RHS: Mask, Name: "cmse.clear");
3911	}
3912
3913	// Emit code to clear the bits in a record, which aren't a part of any user
3914	// declared member, when the record is a function argument.
3915	llvm::Value CodeGenFunction::EmitCMSEClearRecord(llvm::Value Src,
3916	llvm::ArrayType *ATy,
3917	QualType QTy) {
3918	const llvm::DataLayout &DataLayout = CGM.getDataLayout();
3919	int Size = DataLayout.getTypeStoreSize(Ty: ATy);
3920	SmallVector<uint64_t, `16`> Bits(Size);
3921	setUsedBits(CGM, RTy: QTy ->castAs<RecordType>(), Offset: `0`, Bits);
3922
3923	// Clear each element of the LLVM array.
3924	int CharWidth = CGM.getContext().getCharWidth();
3925	int CharsPerElt =
3926	ATy->getArrayElementType()->getScalarSizeInBits() / CharWidth;
3927	int MaskIndex = `0`;
3928	llvm::Value *R = llvm::PoisonValue::get(T: ATy);
3929	for (int I = `0`, N = ATy->getArrayNumElements(); I != N; ++I) {
3930	uint64_t Mask = buildMultiCharMask(Bits, Pos: MaskIndex, Size: CharsPerElt, CharWidth,
3931	BigEndian: DataLayout.isBigEndian());
3932	MaskIndex += CharsPerElt;
3933	llvm::Value *T0 = Builder.CreateExtractValue(Agg: Src, Idxs: I);
3934	llvm::Value *T1 = Builder.CreateAnd(LHS: T0, RHS: Mask, Name: "cmse.clear");
3935	R = Builder.CreateInsertValue(Agg: R, Val: T1, Idxs: I);
3936	}
3937
3938	return R;
3939	}
3940
3941	void CodeGenFunction::EmitFunctionEpilog(
3942	const CGFunctionInfo &FI, bool EmitRetDbgLoc, SourceLocation EndLoc,
3943	uint64_t RetKeyInstructionsSourceAtom) {
3944	if (FI.isNoReturn()) {
3945	// Noreturn functions don't return.
3946	EmitUnreachable(Loc: EndLoc);
3947	return;
3948	}
3949
3950	if (CurCodeDecl && CurCodeDecl->hasAttr<NakedAttr>()) {
3951	// Naked functions don't have epilogues.
3952	Builder.CreateUnreachable();
3953	return;
3954	}
3955
3956	// Functions with no result always return void.
3957	if (!ReturnValue.isValid()) {
3958	auto *I = Builder.CreateRetVoid();
3959	if (RetKeyInstructionsSourceAtom)
3960	addInstToSpecificSourceAtom(KeyInstruction: I, Backup: nullptr, Atom: RetKeyInstructionsSourceAtom);
3961	else
3962	addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
3963	return;
3964	}
3965
3966	llvm::DebugLoc RetDbgLoc;
3967	llvm::Value RV = nullptr*;
3968	QualType RetTy = FI.getReturnType();
3969	const ABIArgInfo &RetAI = FI.getReturnInfo();
3970
3971	switch (RetAI.getKind()) {
3972	case ABIArgInfo::InAlloca:
3973	// Aggregates get evaluated directly into the destination. Sometimes we
3974	// need to return the sret value in a register, though.
3975	assert(hasAggregateEvaluationKind(RetTy));
3976	if (RetAI.getInAllocaSRet()) {
3977	llvm::Function::arg_iterator EI = CurFn->arg_end();
3978	--EI;
3979	llvm::Value ArgStruct = &EI;
3980	llvm::Value *SRet = Builder.CreateStructGEP(
3981	Ty: FI.getArgStruct(), Ptr: ArgStruct, Idx: RetAI.getInAllocaFieldIndex());
3982	llvm::Type *Ty =
3983	cast<llvm::GetElementPtrInst>(Val: SRet)->getResultElementType();
3984	RV = Builder.CreateAlignedLoad(Ty, SRet, getPointerAlign(), "sret");
3985	}
3986	break;
3987
3988	case ABIArgInfo::Indirect: {
3989	auto AI = CurFn->arg_begin();
3990	if (RetAI.isSRetAfterThis())
3991	++AI;
3992	switch (getEvaluationKind(T: RetTy)) {
3993	case TEK_Complex: {
3994	ComplexPairTy RT =
3995	EmitLoadOfComplex(src: MakeAddrLValue(Addr: ReturnValue, T: RetTy), loc: EndLoc);
3996	EmitStoreOfComplex(V: RT, dest: MakeNaturalAlignAddrLValue(V: &*AI, T: RetTy),
3997	/isInit/ true);
3998	break;
3999	}
4000	case TEK_Aggregate:
4001	// Do nothing; aggregates get evaluated directly into the destination.
4002	break;
4003	case TEK_Scalar: {
4004	LValueBaseInfo BaseInfo;
4005	TBAAAccessInfo TBAAInfo;
4006	CharUnits Alignment =
4007	CGM.getNaturalTypeAlignment(T: RetTy, BaseInfo: &BaseInfo, TBAAInfo: &TBAAInfo);
4008	Address ArgAddr(&*AI, ConvertType(T: RetTy), Alignment);
4009	LValue ArgVal =
4010	LValue::MakeAddr(Addr: ArgAddr, type: RetTy, Context&: getContext(), BaseInfo, TBAAInfo);
4011	EmitStoreOfScalar(
4012	value: EmitLoadOfScalar(lvalue: MakeAddrLValue(Addr: ReturnValue, T: RetTy), Loc: EndLoc), lvalue: ArgVal,
4013	/isInit/ true);
4014	break;
4015	}
4016	}
4017	break;
4018	}
4019
4020	case ABIArgInfo::Extend:
4021	case ABIArgInfo::Direct:
4022	if (RetAI.getCoerceToType() == ConvertType(T: RetTy) &&
4023	RetAI.getDirectOffset() == `0`) {
4024	// The internal return value temp always will have pointer-to-return-type
4025	// type, just do a load.
4026
4027	// If there is a dominating store to ReturnValue, we can elide
4028	// the load, zap the store, and usually zap the alloca.
4029	if (llvm::StoreInst SI = findDominatingStoreToReturnValue(CGF&: this)) {
4030	// Reuse the debug location from the store unless there is
4031	// cleanup code to be emitted between the store and return
4032	// instruction.
4033	if (EmitRetDbgLoc && !AutoreleaseResult)
4034	RetDbgLoc = SI->getDebugLoc();
4035	// Get the stored value and nuke the now-dead store.
4036	RV = SI->getValueOperand();
4037	SI->eraseFromParent();
4038
4039	// Otherwise, we have to do a simple load.
4040	} else {
4041	RV = Builder.CreateLoad(Addr: ReturnValue);
4042	}
4043	} else {
4044	// If the value is offset in memory, apply the offset now.
4045	Address V = emitAddressAtOffset(CGF&: *this, addr: ReturnValue, info: RetAI);
4046
4047	RV = CreateCoercedLoad(Src: V, Ty: RetAI.getCoerceToType(), CGF&: *this);
4048	}
4049
4050	// In ARC, end functions that return a retainable type with a call
4051	// to objc_autoreleaseReturnValue.
4052	if (AutoreleaseResult) {
4053	#ifndef NDEBUG
4054	// Type::isObjCRetainabletype has to be called on a QualType that hasn't
4055	// been stripped of the typedefs, so we cannot use RetTy here. Get the
4056	// original return type of FunctionDecl, CurCodeDecl, and BlockDecl from
4057	// CurCodeDecl or BlockInfo.
4058	QualType RT;
4059
4060	if (auto *FD = dyn_cast<FunctionDecl>(Val: CurCodeDecl))
4061	RT = FD->getReturnType();
4062	else if (auto *MD = dyn_cast<ObjCMethodDecl>(Val: CurCodeDecl))
4063	RT = MD->getReturnType();
4064	else if (isa<BlockDecl>(Val: CurCodeDecl))
4065	RT = BlockInfo->BlockExpression->getFunctionType()->getReturnType();
4066	else
4067	llvm_unreachable("Unexpected function/method type");
4068
4069	assert(getLangOpts().ObjCAutoRefCount && !FI.isReturnsRetained() &&
4070	RT ->isObjCRetainableType());
4071	#endif
4072	RV = emitAutoreleaseOfResult(CGF&: *this, result: RV);
4073	}
4074
4075	break;
4076
4077	case ABIArgInfo::Ignore:
4078	break;
4079
4080	case ABIArgInfo::CoerceAndExpand: {
4081	auto coercionType = RetAI.getCoerceAndExpandType();
4082	auto unpaddedCoercionType = RetAI.getUnpaddedCoerceAndExpandType();
4083	auto *unpaddedStruct = dyn_cast<llvm::StructType>(Val: unpaddedCoercionType);
4084
4085	// Load all of the coerced elements out into results.
4086	llvm::SmallVector<llvm::Value *, `4`> results;
4087	Address addr = ReturnValue.withElementType(ElemTy: coercionType);
4088	unsigned unpaddedIndex = `0`;
4089	for (unsigned i = `0`, e = coercionType->getNumElements(); i != e; ++i) {
4090	auto coercedEltType = coercionType->getElementType(N: i);
4091	if (ABIArgInfo::isPaddingForCoerceAndExpand(eltType: coercedEltType))
4092	continue;
4093
4094	auto eltAddr = Builder.CreateStructGEP(Addr: addr, Index: i);
4095	llvm::Value *elt = CreateCoercedLoad(
4096	Src: eltAddr,
4097	Ty: unpaddedStruct ? unpaddedStruct->getElementType(N: unpaddedIndex++)
4098	: unpaddedCoercionType,
4099	CGF&: *this);
4100	results.push_back(Elt: elt);
4101	}
4102
4103	// If we have one result, it's the single direct result type.
4104	if (results.size() == `1`) {
4105	RV = results [`0`];
4106
4107	// Otherwise, we need to make a first-class aggregate.
4108	} else {
4109	// Construct a return type that lacks padding elements.
4110	llvm::Type *returnType = RetAI.getUnpaddedCoerceAndExpandType();
4111
4112	RV = llvm::PoisonValue::get(T: returnType);
4113	for (unsigned i = `0`, e = results.size(); i != e; ++i) {
4114	RV = Builder.CreateInsertValue(Agg: RV, Val: results [i], Idxs: i);
4115	}
4116	}
4117	break;
4118	}
4119	case ABIArgInfo::Expand:
4120	case ABIArgInfo::IndirectAliased:
4121	llvm_unreachable("Invalid ABI kind for return argument");
4122	}
4123
4124	llvm::Instruction *Ret;
4125	if (RV) {
4126	if (CurFuncDecl && CurFuncDecl->hasAttr<CmseNSEntryAttr>()) {
4127	// For certain return types, clear padding bits, as they may reveal
4128	// sensitive information.
4129	// Small struct/union types are passed as integers.
4130	auto *ITy = dyn_cast<llvm::IntegerType>(Val: RV->getType());
4131	if (ITy != nullptr && isa<RecordType>(Val: RetTy.getCanonicalType()))
4132	RV = EmitCMSEClearRecord(Src: RV, ITy, QTy: RetTy);
4133	}
4134	EmitReturnValueCheck(RV);
4135	Ret = Builder.CreateRet(V: RV);
4136	} else {
4137	Ret = Builder.CreateRetVoid();
4138	}
4139
4140	if (RetDbgLoc)
4141	Ret->setDebugLoc(std::move(RetDbgLoc));
4142
4143	llvm::Value Backup = RV ? Ret->getOperand(i: `0`) : nullptr*;
4144	if (RetKeyInstructionsSourceAtom)
4145	addInstToSpecificSourceAtom(KeyInstruction: Ret, Backup, Atom: RetKeyInstructionsSourceAtom);
4146	else
4147	addInstToNewSourceAtom(KeyInstruction: Ret, Backup);
4148	}
4149
4150	void CodeGenFunction::EmitReturnValueCheck(llvm::Value *RV) {
4151	// A current decl may not be available when emitting vtable thunks.
4152	if (!CurCodeDecl)
4153	return;
4154
4155	// If the return block isn't reachable, neither is this check, so don't emit
4156	// it.
4157	if (ReturnBlock.isValid() && ReturnBlock.getBlock()->use_empty())
4158	return;
4159
4160	ReturnsNonNullAttr RetNNAttr = nullptr*;
4161	if (SanOpts.has(SanitizerKind::ReturnsNonnullAttribute))
4162	RetNNAttr = CurCodeDecl->getAttr<ReturnsNonNullAttr>();
4163
4164	if (!RetNNAttr && !requiresReturnValueNullabilityCheck())
4165	return;
4166
4167	// Prefer the returns_nonnull attribute if it's present.
4168	SourceLocation AttrLoc;
4169	SanitizerKind::SanitizerOrdinal CheckKind;
4170	SanitizerHandler Handler;
4171	if (RetNNAttr) {
4172	assert(!requiresReturnValueNullabilityCheck() &&
4173	"Cannot check nullability and the nonnull attribute");
4174	AttrLoc = RetNNAttr->getLocation();
4175	CheckKind = SanitizerKind::SO_ReturnsNonnullAttribute;
4176	Handler = SanitizerHandler::NonnullReturn;
4177	} else {
4178	if (auto *DD = dyn_cast<DeclaratorDecl>(Val: CurCodeDecl))
4179	if (auto *TSI = DD->getTypeSourceInfo())
4180	if (auto FTL = TSI->getTypeLoc().getAsAdjusted<FunctionTypeLoc>())
4181	AttrLoc = FTL.getReturnLoc().findNullabilityLoc();
4182	CheckKind = SanitizerKind::SO_NullabilityReturn;
4183	Handler = SanitizerHandler::NullabilityReturn;
4184	}
4185
4186	SanitizerDebugLocation SanScope(this, {CheckKind}, Handler);
4187
4188	// Make sure the "return" source location is valid. If we're checking a
4189	// nullability annotation, make sure the preconditions for the check are met.
4190	llvm::BasicBlock *Check = createBasicBlock(name: "nullcheck");
4191	llvm::BasicBlock *NoCheck = createBasicBlock(name: "no.nullcheck");
4192	llvm::Value *SLocPtr = Builder.CreateLoad(Addr: ReturnLocation, Name: "return.sloc.load");
4193	llvm::Value *CanNullCheck = Builder.CreateIsNotNull(Arg: SLocPtr);
4194	if (requiresReturnValueNullabilityCheck())
4195	CanNullCheck =
4196	Builder.CreateAnd(LHS: CanNullCheck, RHS: RetValNullabilityPrecondition);
4197	Builder.CreateCondBr(Cond: CanNullCheck, True: Check, False: NoCheck);
4198	EmitBlock(BB: Check);
4199
4200	// Now do the null check.
4201	llvm::Value *Cond = Builder.CreateIsNotNull(Arg: RV);
4202	llvm::Constant *StaticData[] = {EmitCheckSourceLocation(Loc: AttrLoc)};
4203	llvm::Value *DynamicData[] = {SLocPtr};
4204	EmitCheck(Checked: std::make_pair(x&: Cond, y&: CheckKind), Check: Handler, StaticArgs: StaticData, DynamicArgs: DynamicData);
4205
4206	EmitBlock(BB: NoCheck);
4207
4208	#ifndef NDEBUG
4209	// The return location should not be used after the check has been emitted.
4210	ReturnLocation = Address::invalid();
4211	#endif
4212	}
4213
4214	static bool isInAllocaArgument(CGCXXABI &ABI, QualType type) {
4215	const CXXRecordDecl *RD = type ->getAsCXXRecordDecl();
4216	return RD && ABI.getRecordArgABI(RD) == CGCXXABI::RAA_DirectInMemory;
4217	}
4218
4219	static AggValueSlot createPlaceholderSlot(CodeGenFunction &CGF, QualType Ty) {
4220	// FIXME: Generate IR in one pass, rather than going back and fixing up these
4221	// placeholders.
4222	llvm::Type *IRTy = CGF.ConvertTypeForMem(T: Ty);
4223	llvm::Type *IRPtrTy = llvm::PointerType::getUnqual(C&: CGF.getLLVMContext());
4224	llvm::Value *Placeholder = llvm::PoisonValue::get(T: IRPtrTy);
4225
4226	// FIXME: When we generate this IR in one pass, we shouldn't need
4227	// this win32-specific alignment hack.
4228	CharUnits Align = CharUnits::fromQuantity(Quantity: `4`);
4229	Placeholder = CGF.Builder.CreateAlignedLoad(Ty: IRPtrTy, Addr: Placeholder, Align);
4230
4231	return AggValueSlot::forAddr(
4232	addr: Address (Placeholder, IRTy, Align), quals: Ty.getQualifiers(),
4233	isDestructed: AggValueSlot::IsNotDestructed, needsGC: AggValueSlot::DoesNotNeedGCBarriers,
4234	isAliased: AggValueSlot::IsNotAliased, mayOverlap: AggValueSlot::DoesNotOverlap);
4235	}
4236
4237	void CodeGenFunction::EmitDelegateCallArg(CallArgList &args,
4238	const VarDecl *param,
4239	SourceLocation loc) {
4240	// StartFunction converted the ABI-lowered parameter(s) into a
4241	// local alloca. We need to turn that into an r-value suitable
4242	// for EmitCall.
4243	Address local = GetAddrOfLocalVar(VD: param);
4244
4245	QualType type = param->getType();
4246
4247	// GetAddrOfLocalVar returns a pointer-to-pointer for references,
4248	// but the argument needs to be the original pointer.
4249	if (type ->isReferenceType()) {
4250	args.add(rvalue: RValue::get(V: Builder.CreateLoad(Addr: local)), type);
4251
4252	// In ARC, move out of consumed arguments so that the release cleanup
4253	// entered by StartFunction doesn't cause an over-release. This isn't
4254	// optimal -O0 code generation, but it should get cleaned up when
4255	// optimization is enabled. This also assumes that delegate calls are
4256	// performed exactly once for a set of arguments, but that should be safe.
4257	} else if (getLangOpts().ObjCAutoRefCount &&
4258	param->hasAttr<NSConsumedAttr>() && type->isObjCRetainableType()) {
4259	llvm::Value *ptr = Builder.CreateLoad(Addr: local);
4260	auto null =
4261	llvm::ConstantPointerNull::get(T: cast<llvm::PointerType>(Val: ptr->getType()));
4262	Builder.CreateStore(Val: null, Addr: local);
4263	args.add(rvalue: RValue::get(V: ptr), type);
4264
4265	// For the most part, we just need to load the alloca, except that
4266	// aggregate r-values are actually pointers to temporaries.
4267	} else {
4268	args.add(rvalue: convertTempToRValue(addr: local, type, Loc: loc), type);
4269	}
4270
4271	// Deactivate the cleanup for the callee-destructed param that was pushed.
4272	if (type ->isRecordType() && !CurFuncIsThunk &&
4273	type ->castAs<RecordType>()->getDecl()->isParamDestroyedInCallee() &&
4274	param->needsDestruction(Ctx: getContext())) {
4275	EHScopeStack::stable_iterator cleanup =
4276	CalleeDestructedParamCleanups.lookup(Val: cast<ParmVarDecl>(Val: param));
4277	assert(cleanup.isValid() &&
4278	"cleanup for callee-destructed param not recorded");
4279	// This unreachable is a temporary marker which will be removed later.
4280	llvm::Instruction *isActive = Builder.CreateUnreachable();
4281	args.addArgCleanupDeactivation(Cleanup: cleanup, IsActiveIP: isActive);
4282	}
4283	}
4284
4285	static bool isProvablyNull(llvm::Value *addr) {
4286	return llvm::isa_and_nonnull<llvm::ConstantPointerNull>(Val: addr);
4287	}
4288
4289	static bool isProvablyNonNull(Address Addr, CodeGenFunction &CGF) {
4290	return llvm::isKnownNonZero(V: Addr.getBasePointer(), Q: CGF.CGM.getDataLayout());
4291	}
4292
4293	/// Emit the actual writing-back of a writeback.
4294	static void emitWriteback(CodeGenFunction &CGF,
4295	const CallArgList::Writeback &writeback) {
4296	const LValue &srcLV = writeback.Source;
4297	Address srcAddr = srcLV.getAddress();
4298	assert(!isProvablyNull(srcAddr.getBasePointer()) &&
4299	"shouldn't have writeback for provably null argument");
4300
4301	if (writeback.WritebackExpr) {
4302	CGF.EmitIgnoredExpr(E: writeback.WritebackExpr);
4303
4304	if (writeback.LifetimeSz)
4305	CGF.EmitLifetimeEnd(Size: writeback.LifetimeSz,
4306	Addr: writeback.Temporary.getBasePointer());
4307	return;
4308	}
4309
4310	llvm::BasicBlock contBB = nullptr*;
4311
4312	// If the argument wasn't provably non-null, we need to null check
4313	// before doing the store.
4314	bool provablyNonNull = isProvablyNonNull(Addr: srcAddr, CGF);
4315
4316	if (!provablyNonNull) {
4317	llvm::BasicBlock *writebackBB = CGF.createBasicBlock(name: "icr.writeback");
4318	contBB = CGF.createBasicBlock(name: "icr.done");
4319
4320	llvm::Value *isNull = CGF.Builder.CreateIsNull(Addr: srcAddr, Name: "icr.isnull");
4321	CGF.Builder.CreateCondBr(Cond: isNull, True: contBB, False: writebackBB);
4322	CGF.EmitBlock(BB: writebackBB);
4323	}
4324
4325	// Load the value to writeback.
4326	llvm::Value *value = CGF.Builder.CreateLoad(Addr: writeback.Temporary);
4327
4328	// Cast it back, in case we're writing an id to a Foo or something.*
4329	value = CGF.Builder.CreateBitCast(V: value, DestTy: srcAddr.getElementType(),
4330	Name: "icr.writeback-cast");
4331
4332	// Perform the writeback.
4333
4334	// If we have a "to use" value, it's something we need to emit a use
4335	// of. This has to be carefully threaded in: if it's done after the
4336	// release it's potentially undefined behavior (and the optimizer
4337	// will ignore it), and if it happens before the retain then the
4338	// optimizer could move the release there.
4339	if (writeback.ToUse) {
4340	assert(srcLV.getObjCLifetime() == Qualifiers::OCL_Strong);
4341
4342	// Retain the new value. No need to block-copy here: the block's
4343	// being passed up the stack.
4344	value = CGF.EmitARCRetainNonBlock(value);
4345
4346	// Emit the intrinsic use here.
4347	CGF.EmitARCIntrinsicUse(values: writeback.ToUse);
4348
4349	// Load the old value (primitively).
4350	llvm::Value *oldValue = CGF.EmitLoadOfScalar(lvalue: srcLV, Loc: SourceLocation ());
4351
4352	// Put the new value in place (primitively).
4353	CGF.EmitStoreOfScalar(value, lvalue: srcLV, /init/ isInit: false);
4354
4355	// Release the old value.
4356	CGF.EmitARCRelease(value: oldValue, precise: srcLV.isARCPreciseLifetime());
4357
4358	// Otherwise, we can just do a normal lvalue store.
4359	} else {
4360	CGF.EmitStoreThroughLValue(Src: RValue::get(V: value), Dst: srcLV);
4361	}
4362
4363	// Jump to the continuation block.
4364	if (!provablyNonNull)
4365	CGF.EmitBlock(BB: contBB);
4366	}
4367
4368	static void deactivateArgCleanupsBeforeCall(CodeGenFunction &CGF,
4369	const CallArgList &CallArgs) {
4370	ArrayRef<CallArgList::CallArgCleanup> Cleanups =
4371	CallArgs.getCleanupsToDeactivate();
4372	// Iterate in reverse to increase the likelihood of popping the cleanup.
4373	for (const auto &I : llvm::reverse(C&: Cleanups)) {
4374	CGF.DeactivateCleanupBlock(Cleanup: I.Cleanup, DominatingIP: I.IsActiveIP);
4375	I.IsActiveIP->eraseFromParent();
4376	}
4377	}
4378
4379	static const Expr maybeGetUnaryAddrOfOperand(const* Expr *E) {
4380	if (const UnaryOperator *uop = dyn_cast<UnaryOperator>(Val: E->IgnoreParens()))
4381	if (uop->getOpcode() == UO_AddrOf)
4382	return uop->getSubExpr();
4383	return nullptr;
4384	}
4385
4386	/// Emit an argument that's being passed call-by-writeback. That is,
4387	/// we are passing the address of an __autoreleased temporary; it
4388	/// might be copy-initialized with the current value of the given
4389	/// address, but it will definitely be copied out of after the call.
4390	static void emitWritebackArg(CodeGenFunction &CGF, CallArgList &args,
4391	const ObjCIndirectCopyRestoreExpr *CRE) {
4392	LValue srcLV;
4393
4394	// Make an optimistic effort to emit the address as an l-value.
4395	// This can fail if the argument expression is more complicated.
4396	if (const Expr *lvExpr = maybeGetUnaryAddrOfOperand(E: CRE->getSubExpr())) {
4397	srcLV = CGF.EmitLValue(E: lvExpr);
4398
4399	// Otherwise, just emit it as a scalar.
4400	} else {
4401	Address srcAddr = CGF.EmitPointerWithAlignment(Addr: CRE->getSubExpr());
4402
4403	QualType srcAddrType =
4404	CRE->getSubExpr()->getType()->castAs<PointerType>()->getPointeeType();
4405	srcLV = CGF.MakeAddrLValue(Addr: srcAddr, T: srcAddrType);
4406	}
4407	Address srcAddr = srcLV.getAddress();
4408
4409	// The dest and src types don't necessarily match in LLVM terms
4410	// because of the crazy ObjC compatibility rules.
4411
4412	llvm::PointerType *destType =
4413	cast<llvm::PointerType>(CGF.ConvertType(CRE->getType()));
4414	llvm::Type *destElemType =
4415	CGF.ConvertTypeForMem(T: CRE->getType()->getPointeeType());
4416
4417	// If the address is a constant null, just pass the appropriate null.
4418	if (isProvablyNull(addr: srcAddr.getBasePointer())) {
4419	args.add(rvalue: RValue::get(V: llvm::ConstantPointerNull::get(T: destType)),
4420	type: CRE->getType());
4421	return;
4422	}
4423
4424	// Create the temporary.
4425	Address temp =
4426	CGF.CreateTempAlloca(destElemType, CGF.getPointerAlign(), "icr.temp");
4427	// Loading an l-value can introduce a cleanup if the l-value is __weak,
4428	// and that cleanup will be conditional if we can't prove that the l-value
4429	// isn't null, so we need to register a dominating point so that the cleanups
4430	// system will make valid IR.
4431	CodeGenFunction::ConditionalEvaluation condEval(CGF);
4432
4433	// Zero-initialize it if we're not doing a copy-initialization.
4434	bool shouldCopy = CRE->shouldCopy();
4435	if (!shouldCopy) {
4436	llvm::Value *null =
4437	llvm::ConstantPointerNull::get(T: cast<llvm::PointerType>(Val: destElemType));
4438	CGF.Builder.CreateStore(Val: null, Addr: temp);
4439	}
4440
4441	llvm::BasicBlock contBB = nullptr*;
4442	llvm::BasicBlock originBB = nullptr*;
4443
4444	// If the address is not* known to be non-null, we need to switch.*
4445	llvm::Value *finalArgument;
4446
4447	bool provablyNonNull = isProvablyNonNull(Addr: srcAddr, CGF);
4448
4449	if (provablyNonNull) {
4450	finalArgument = temp.emitRawPointer(CGF);
4451	} else {
4452	llvm::Value *isNull = CGF.Builder.CreateIsNull(Addr: srcAddr, Name: "icr.isnull");
4453
4454	finalArgument = CGF.Builder.CreateSelect(
4455	C: isNull, True: llvm::ConstantPointerNull::get(T: destType),
4456	False: temp.emitRawPointer(CGF), Name: "icr.argument");
4457
4458	// If we need to copy, then the load has to be conditional, which
4459	// means we need control flow.
4460	if (shouldCopy) {
4461	originBB = CGF.Builder.GetInsertBlock();
4462	contBB = CGF.createBasicBlock(name: "icr.cont");
4463	llvm::BasicBlock *copyBB = CGF.createBasicBlock(name: "icr.copy");
4464	CGF.Builder.CreateCondBr(Cond: isNull, True: contBB, False: copyBB);
4465	CGF.EmitBlock(BB: copyBB);
4466	condEval.begin(CGF);
4467	}
4468	}
4469
4470	llvm::Value valueToUse = nullptr*;
4471
4472	// Perform a copy if necessary.
4473	if (shouldCopy) {
4474	RValue srcRV = CGF.EmitLoadOfLValue(V: srcLV, Loc: SourceLocation ());
4475	assert(srcRV.isScalar());
4476
4477	llvm::Value *src = srcRV.getScalarVal();
4478	src = CGF.Builder.CreateBitCast(V: src, DestTy: destElemType, Name: "icr.cast");
4479
4480	// Use an ordinary store, not a store-to-lvalue.
4481	CGF.Builder.CreateStore(Val: src, Addr: temp);
4482
4483	// If optimization is enabled, and the value was held in a
4484	// __strong variable, we need to tell the optimizer that this
4485	// value has to stay alive until we're doing the store back.
4486	// This is because the temporary is effectively unretained,
4487	// and so otherwise we can violate the high-level semantics.
4488	if (CGF.CGM.getCodeGenOpts().OptimizationLevel != `0` &&
4489	srcLV.getObjCLifetime() == Qualifiers::OCL_Strong) {
4490	valueToUse = src;
4491	}
4492	}
4493
4494	// Finish the control flow if we needed it.
4495	if (shouldCopy && !provablyNonNull) {
4496	llvm::BasicBlock *copyBB = CGF.Builder.GetInsertBlock();
4497	CGF.EmitBlock(BB: contBB);
4498
4499	// Make a phi for the value to intrinsically use.
4500	if (valueToUse) {
4501	llvm::PHINode *phiToUse =
4502	CGF.Builder.CreatePHI(Ty: valueToUse->getType(), NumReservedValues: `2`, Name: "icr.to-use");
4503	phiToUse->addIncoming(V: valueToUse, BB: copyBB);
4504	phiToUse->addIncoming(V: llvm::PoisonValue::get(T: valueToUse->getType()),
4505	BB: originBB);
4506	valueToUse = phiToUse;
4507	}
4508
4509	condEval.end(CGF);
4510	}
4511
4512	args.addWriteback(srcLV, temporary: temp, toUse: valueToUse);
4513	args.add(rvalue: RValue::get(V: finalArgument), type: CRE->getType());
4514	}
4515
4516	void CallArgList::allocateArgumentMemory(CodeGenFunction &CGF) {
4517	assert(!StackBase);
4518
4519	// Save the stack.
4520	StackBase = CGF.Builder.CreateStackSave(Name: "inalloca.save");
4521	}
4522
4523	void CallArgList::freeArgumentMemory(CodeGenFunction &CGF) const {
4524	if (StackBase) {
4525	// Restore the stack after the call.
4526	CGF.Builder.CreateStackRestore(Ptr: StackBase);
4527	}
4528	}
4529
4530	void CodeGenFunction::EmitNonNullArgCheck(RValue RV, QualType ArgType,
4531	SourceLocation ArgLoc,
4532	AbstractCallee AC, unsigned ParmNum) {
4533	if (!AC.getDecl() \|\| !(SanOpts.has(K: SanitizerKind::NonnullAttribute) \|\|
4534	SanOpts.has(K: SanitizerKind::NullabilityArg)))
4535	return;
4536
4537	// The param decl may be missing in a variadic function.
4538	auto PVD = ParmNum < AC.getNumParams() ? AC.getParamDecl(I: ParmNum) : nullptr;
4539	unsigned ArgNo = PVD ? PVD->getFunctionScopeIndex() : ParmNum;
4540
4541	// Prefer the nonnull attribute if it's present.
4542	const NonNullAttr NNAttr = nullptr*;
4543	if (SanOpts.has(SanitizerKind::NonnullAttribute))
4544	NNAttr = getNonNullAttr(AC.getDecl(), PVD, ArgType, ArgNo);
4545
4546	bool CanCheckNullability = false;
4547	if (SanOpts.has(K: SanitizerKind::NullabilityArg) && !NNAttr && PVD &&
4548	!PVD->getType()->isRecordType()) {
4549	auto Nullability = PVD->getType()->getNullability();
4550	CanCheckNullability = Nullability &&
4551	*Nullability == NullabilityKind::NonNull &&
4552	PVD->getTypeSourceInfo();
4553	}
4554
4555	if (!NNAttr && !CanCheckNullability)
4556	return;
4557
4558	SourceLocation AttrLoc;
4559	SanitizerKind::SanitizerOrdinal CheckKind;
4560	SanitizerHandler Handler;
4561	if (NNAttr) {
4562	AttrLoc = NNAttr->getLocation();
4563	CheckKind = SanitizerKind::SO_NonnullAttribute;
4564	Handler = SanitizerHandler::NonnullArg;
4565	} else {
4566	AttrLoc = PVD->getTypeSourceInfo()->getTypeLoc().findNullabilityLoc();
4567	CheckKind = SanitizerKind::SO_NullabilityArg;
4568	Handler = SanitizerHandler::NullabilityArg;
4569	}
4570
4571	SanitizerDebugLocation SanScope(this, {CheckKind}, Handler);
4572	llvm::Value *Cond = EmitNonNullRValueCheck(RV, T: ArgType);
4573	llvm::Constant *StaticData[] = {
4574	EmitCheckSourceLocation(Loc: ArgLoc),
4575	EmitCheckSourceLocation(Loc: AttrLoc),
4576	llvm::ConstantInt::get(Ty: Int32Ty, V: ArgNo + `1`),
4577	};
4578	EmitCheck(Checked: std::make_pair(x&: Cond, y&: CheckKind), Check: Handler, StaticArgs: StaticData, DynamicArgs: {});
4579	}
4580
4581	void CodeGenFunction::EmitNonNullArgCheck(Address Addr, QualType ArgType,
4582	SourceLocation ArgLoc,
4583	AbstractCallee AC, unsigned ParmNum) {
4584	if (!AC.getDecl() \|\| !(SanOpts.has(K: SanitizerKind::NonnullAttribute) \|\|
4585	SanOpts.has(K: SanitizerKind::NullabilityArg)))
4586	return;
4587
4588	EmitNonNullArgCheck(RV: RValue::get(Addr, CGF&: *this), ArgType, ArgLoc, AC, ParmNum);
4589	}
4590
4591	// Check if the call is going to use the inalloca convention. This needs to
4592	// agree with CGFunctionInfo::usesInAlloca. The CGFunctionInfo is arranged
4593	// later, so we can't check it directly.
4594	static bool hasInAllocaArgs(CodeGenModule &CGM, CallingConv ExplicitCC,
4595	ArrayRef<QualType> ArgTypes) {
4596	// The Swift calling conventions don't go through the target-specific
4597	// argument classification, they never use inalloca.
4598	// TODO: Consider limiting inalloca use to only calling conventions supported
4599	// by MSVC.
4600	if (ExplicitCC == CC_Swift \|\| ExplicitCC == CC_SwiftAsync)
4601	return false;
4602	if (!CGM.getTarget().getCXXABI().isMicrosoft())
4603	return false;
4604	return llvm::any_of(Range&: ArgTypes, P: [&](QualType Ty) {
4605	return isInAllocaArgument(ABI&: CGM.getCXXABI(), type: Ty);
4606	});
4607	}
4608
4609	#ifndef NDEBUG
4610	// Determine whether the given argument is an Objective-C method
4611	// that may have type parameters in its signature.
4612	static bool isObjCMethodWithTypeParams(const ObjCMethodDecl *method) {
4613	const DeclContext *dc = method->getDeclContext();
4614	if (const ObjCInterfaceDecl *classDecl = dyn_cast<ObjCInterfaceDecl>(Val: dc)) {
4615	return classDecl->getTypeParamListAsWritten();
4616	}
4617
4618	if (const ObjCCategoryDecl *catDecl = dyn_cast<ObjCCategoryDecl>(Val: dc)) {
4619	return catDecl->getTypeParamList();
4620	}
4621
4622	return false;
4623	}
4624	#endif
4625
4626	/// EmitCallArgs - Emit call arguments for a function.
4627	void CodeGenFunction::EmitCallArgs(
4628	CallArgList &Args, PrototypeWrapper Prototype,
4629	llvm::iterator_range<CallExpr::const_arg_iterator> ArgRange,
4630	AbstractCallee AC, unsigned ParamsToSkip, EvaluationOrder Order) {
4631	SmallVector<QualType, `16`> ArgTypes;
4632
4633	assert((ParamsToSkip == `0` \|\| Prototype.P) &&
4634	"Can't skip parameters if type info is not provided");
4635
4636	// This variable only captures explicitly* written conventions, not those*
4637	// applied by default via command line flags or target defaults, such as
4638	// thiscall, aapcs, stdcall via -mrtd, etc. Computing that correctly would
4639	// require knowing if this is a C++ instance method or being able to see
4640	// unprototyped FunctionTypes.
4641	CallingConv ExplicitCC = CC_C;
4642
4643	// First, if a prototype was provided, use those argument types.
4644	bool IsVariadic = false;
4645	if (Prototype.P) {
4646	const auto MD = dyn_cast<const* ObjCMethodDecl *>(Val&: Prototype.P);
4647	if (MD) {
4648	IsVariadic = MD->isVariadic();
4649	ExplicitCC = getCallingConventionForDecl(
4650	D: MD, IsTargetDefaultMSABI: CGM.getTarget().getTriple().isOSWindows());
4651	ArgTypes.assign(in_start: MD->param_type_begin() + ParamsToSkip,
4652	in_end: MD->param_type_end());
4653	} else {
4654	const auto FPT = cast<const* FunctionProtoType *>(Val&: Prototype.P);
4655	IsVariadic = FPT->isVariadic();
4656	ExplicitCC = FPT->getExtInfo().getCC();
4657	ArgTypes.assign(in_start: FPT->param_type_begin() + ParamsToSkip,
4658	in_end: FPT->param_type_end());
4659	}
4660
4661	#ifndef NDEBUG
4662	// Check that the prototyped types match the argument expression types.
4663	bool isGenericMethod = MD && isObjCMethodWithTypeParams(method: MD);
4664	CallExpr::const_arg_iterator Arg = ArgRange.begin();
4665	for (QualType Ty : ArgTypes) {
4666	assert(Arg != ArgRange.end() && "Running over edge of argument list!");
4667	assert(
4668	(isGenericMethod \|\| Ty ->isVariablyModifiedType() \|\|
4669	Ty.getNonReferenceType()->isObjCRetainableType() \|\|
4670	getContext()
4671	.getCanonicalType(Ty.getNonReferenceType())
4672	.getTypePtr() ==
4673	getContext().getCanonicalType((*Arg)->getType()).getTypePtr()) &&
4674	"type mismatch in call argument!");
4675	++Arg;
4676	}
4677
4678	// Either we've emitted all the call args, or we have a call to variadic
4679	// function.
4680	assert((Arg == ArgRange.end() \|\| IsVariadic) &&
4681	"Extra arguments in non-variadic function!");
4682	#endif
4683	}
4684
4685	// If we still have any arguments, emit them using the type of the argument.
4686	for (auto *A : llvm::drop_begin(ArgRange, ArgTypes.size()))
4687	ArgTypes.push_back(IsVariadic ? getVarArgType(A) : A->getType());
4688	assert((int)ArgTypes.size() == (ArgRange.end() - ArgRange.begin()));
4689
4690	// We must evaluate arguments from right to left in the MS C++ ABI,
4691	// because arguments are destroyed left to right in the callee. As a special
4692	// case, there are certain language constructs that require left-to-right
4693	// evaluation, and in those cases we consider the evaluation order requirement
4694	// to trump the "destruction order is reverse construction order" guarantee.
4695	bool LeftToRight =
4696	CGM.getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee()
4697	? Order == EvaluationOrder::ForceLeftToRight
4698	: Order != EvaluationOrder::ForceRightToLeft;
4699
4700	auto MaybeEmitImplicitObjectSize = [&](unsigned I, const Expr *Arg,
4701	RValue EmittedArg) {
4702	if (!AC.hasFunctionDecl() \|\| I >= AC.getNumParams())
4703	return;
4704	auto *PS = AC.getParamDecl(I)->getAttr<PassObjectSizeAttr>();
4705	if (PS == nullptr)
4706	return;
4707
4708	const auto &Context = getContext();
4709	auto SizeTy = Context.getSizeType();
4710	auto T = Builder.getIntNTy(N: Context.getTypeSize(T: SizeTy));
4711	assert(EmittedArg.getScalarVal() && "We emitted nothing for the arg?");
4712	llvm::Value *V = evaluateOrEmitBuiltinObjectSize(
4713	E: Arg, Type: PS->getType(), ResType: T, EmittedE: EmittedArg.getScalarVal(), IsDynamic: PS->isDynamic());
4714	Args.add(rvalue: RValue::get(V), type: SizeTy);
4715	// If we're emitting args in reverse, be sure to do so with
4716	// pass_object_size, as well.
4717	if (!LeftToRight)
4718	std::swap(a&: Args.back(), b&: *(&Args.back() - `1`));
4719	};
4720
4721	// Insert a stack save if we're going to need any inalloca args.
4722	if (hasInAllocaArgs(CGM, ExplicitCC, ArgTypes)) {
4723	assert(getTarget().getTriple().getArch() == llvm::Triple::x86 &&
4724	"inalloca only supported on x86");
4725	Args.allocateArgumentMemory(CGF&: *this);
4726	}
4727
4728	// Evaluate each argument in the appropriate order.
4729	size_t CallArgsStart = Args.size();
4730	for (unsigned I = `0`, E = ArgTypes.size(); I != E; ++I) {
4731	unsigned Idx = LeftToRight ? I : E - I - `1`;
4732	CallExpr::const_arg_iterator Arg = ArgRange.begin() + Idx;
4733	unsigned InitialArgSize = Args.size();
4734	// If Arg is an ObjCIndirectCopyRestoreExpr, check that either the types of*
4735	// the argument and parameter match or the objc method is parameterized.
4736	assert((!isa<ObjCIndirectCopyRestoreExpr>(*Arg) \|\|
4737	getContext().hasSameUnqualifiedType((*Arg)->getType(),
4738	ArgTypes[Idx]) \|\|
4739	(isa<ObjCMethodDecl>(AC.getDecl()) &&
4740	isObjCMethodWithTypeParams(cast<ObjCMethodDecl>(AC.getDecl())))) &&
4741	"Argument and parameter types don't match");
4742	EmitCallArg(args&: Args, E: *Arg, ArgType: ArgTypes [Idx]);
4743	// In particular, we depend on it being the last arg in Args, and the
4744	// objectsize bits depend on there only being one arg if !LeftToRight.
4745	assert(InitialArgSize + `1` == Args.size() &&
4746	"The code below depends on only adding one arg per EmitCallArg");
4747	(void)InitialArgSize;
4748	// Since pointer argument are never emitted as LValue, it is safe to emit
4749	// non-null argument check for r-value only.
4750	if (!Args.back().hasLValue()) {
4751	RValue RVArg = Args.back().getKnownRValue();
4752	EmitNonNullArgCheck(RVArg, ArgTypes [Idx], (*Arg)->getExprLoc(), AC,
4753	ParamsToSkip + Idx);
4754	// @llvm.objectsize should never have side-effects and shouldn't need
4755	// destruction/cleanups, so we can safely "emit" it after its arg,
4756	// regardless of right-to-leftness
4757	MaybeEmitImplicitObjectSize(Idx, *Arg, RVArg);
4758	}
4759	}
4760
4761	if (!LeftToRight) {
4762	// Un-reverse the arguments we just evaluated so they match up with the LLVM
4763	// IR function.
4764	std::reverse(first: Args.begin() + CallArgsStart, last: Args.end());
4765
4766	// Reverse the writebacks to match the MSVC ABI.
4767	Args.reverseWritebacks();
4768	}
4769	}
4770
4771	namespace {
4772
4773	struct DestroyUnpassedArg final : EHScopeStack::Cleanup {
4774	DestroyUnpassedArg(Address Addr, QualType Ty) : Addr (Addr), Ty(Ty) {}
4775
4776	Address Addr;
4777	QualType Ty;
4778
4779	void Emit(CodeGenFunction &CGF, Flags flags) override {
4780	QualType::DestructionKind DtorKind = Ty.isDestructedType();
4781	if (DtorKind == QualType::DK_cxx_destructor) {
4782	const CXXDestructorDecl *Dtor = Ty->getAsCXXRecordDecl()->getDestructor();
4783	assert(!Dtor->isTrivial());
4784	CGF.EmitCXXDestructorCall(Dtor, Dtor_Complete, /for vbase/ false,
4785	/Delegating=/false, Addr, Ty);
4786	} else {
4787	CGF.callCStructDestructor(CGF.MakeAddrLValue(Addr, Ty));
4788	}
4789	}
4790	};
4791
4792	struct DisableDebugLocationUpdates {
4793	CodeGenFunction &CGF;
4794	bool disabledDebugInfo;
4795	DisableDebugLocationUpdates(CodeGenFunction &CGF, const Expr *E) : CGF(CGF) {
4796	if ((disabledDebugInfo = isa<CXXDefaultArgExpr>(Val: E) && CGF.getDebugInfo()))
4797	CGF.disableDebugInfo();
4798	}
4799	~DisableDebugLocationUpdates() {
4800	if (disabledDebugInfo)
4801	CGF.enableDebugInfo();
4802	}
4803	};
4804
4805	} // end anonymous namespace
4806
4807	RValue CallArg::getRValue(CodeGenFunction &CGF) const {
4808	if (!HasLV)
4809	return RV;
4810	LValue Copy = CGF.MakeAddrLValue(CGF.CreateMemTemp(Ty), Ty);
4811	CGF.EmitAggregateCopy(Copy, LV, Ty, AggValueSlot::DoesNotOverlap,
4812	LV.isVolatile());
4813	IsUsed = true;
4814	return RValue::getAggregate(addr: Copy.getAddress());
4815	}
4816
4817	void CallArg::copyInto(CodeGenFunction &CGF, Address Addr) const {
4818	LValue Dst = CGF.MakeAddrLValue(Addr, Ty);
4819	if (!HasLV && RV.isScalar())
4820	CGF.EmitStoreOfScalar(RV.getScalarVal(), Dst, /isInit=/true);
4821	else if (!HasLV && RV.isComplex())
4822	CGF.EmitStoreOfComplex(RV.getComplexVal(), Dst, /init=/true);
4823	else {
4824	auto Addr = HasLV ? LV.getAddress() : RV.getAggregateAddress();
4825	LValue SrcLV = CGF.MakeAddrLValue(Addr, Ty);
4826	// We assume that call args are never copied into subobjects.
4827	CGF.EmitAggregateCopy(Dst, SrcLV, Ty, AggValueSlot::DoesNotOverlap,
4828	HasLV ? LV.isVolatileQualified()
4829	: RV.isVolatileQualified());
4830	}
4831	IsUsed = true;
4832	}
4833
4834	void CodeGenFunction::EmitWritebacks(const CallArgList &args) {
4835	for (const auto &I : args.writebacks())
4836	emitWriteback(CGF&: *this, writeback: I);
4837	}
4838
4839	void CodeGenFunction::EmitCallArg(CallArgList &args, const Expr *E,
4840	QualType type) {
4841	DisableDebugLocationUpdates Dis(*this, E);
4842	if (const ObjCIndirectCopyRestoreExpr *CRE =
4843	dyn_cast<ObjCIndirectCopyRestoreExpr>(Val: E)) {
4844	assert(getLangOpts().ObjCAutoRefCount);
4845	return emitWritebackArg(CGF&: *this, args, CRE);
4846	}
4847
4848	// Add writeback for HLSLOutParamExpr.
4849	// Needs to be before the assert below because HLSLOutArgExpr is an LValue
4850	// and is not a reference.
4851	if (const HLSLOutArgExpr *OE = dyn_cast<HLSLOutArgExpr>(Val: E)) {
4852	EmitHLSLOutArgExpr(E: OE, Args&: args, Ty: type);
4853	return;
4854	}
4855
4856	assert(type ->isReferenceType() == E->isGLValue() &&
4857	"reference binding to unmaterialized r-value!");
4858
4859	if (E->isGLValue()) {
4860	assert(E->getObjectKind() == OK_Ordinary);
4861	return args.add(rvalue: EmitReferenceBindingToExpr(E), type);
4862	}
4863
4864	bool HasAggregateEvalKind = hasAggregateEvaluationKind(T: type);
4865
4866	// In the Microsoft C++ ABI, aggregate arguments are destructed by the callee.
4867	// However, we still have to push an EH-only cleanup in case we unwind before
4868	// we make it to the call.
4869	if (type ->isRecordType() &&
4870	type ->castAs<RecordType>()->getDecl()->isParamDestroyedInCallee()) {
4871	// If we're using inalloca, use the argument memory. Otherwise, use a
4872	// temporary.
4873	AggValueSlot Slot = args.isUsingInAlloca()
4874	? createPlaceholderSlot(CGF&: *this, Ty: type)
4875	: CreateAggTemp(T: type, Name: "agg.tmp");
4876
4877	bool DestroyedInCallee = true, NeedsCleanup = true;
4878	if (const auto *RD = type ->getAsCXXRecordDecl())
4879	DestroyedInCallee = RD->hasNonTrivialDestructor();
4880	else
4881	NeedsCleanup = type.isDestructedType();
4882
4883	if (DestroyedInCallee)
4884	Slot.setExternallyDestructed();
4885
4886	EmitAggExpr(E, AS: Slot);
4887	RValue RV = Slot.asRValue();
4888	args.add(rvalue: RV, type);
4889
4890	if (DestroyedInCallee && NeedsCleanup) {
4891	// Create a no-op GEP between the placeholder and the cleanup so we can
4892	// RAUW it successfully. It also serves as a marker of the first
4893	// instruction where the cleanup is active.
4894	pushFullExprCleanup<DestroyUnpassedArg>(kind: NormalAndEHCleanup,
4895	A: Slot.getAddress(), A: type);
4896	// This unreachable is a temporary marker which will be removed later.
4897	llvm::Instruction *IsActive =
4898	Builder.CreateFlagLoad(Addr: llvm::Constant::getNullValue(Ty: Int8PtrTy));
4899	args.addArgCleanupDeactivation(Cleanup: EHStack.stable_begin(), IsActiveIP: IsActive);
4900	}
4901	return;
4902	}
4903
4904	if (HasAggregateEvalKind && isa<ImplicitCastExpr>(Val: E) &&
4905	cast<CastExpr>(Val: E)->getCastKind() == CK_LValueToRValue &&
4906	!type ->isArrayParameterType() && !type.isNonTrivialToPrimitiveCopy()) {
4907	LValue L = EmitLValue(E: cast<CastExpr>(Val: E)->getSubExpr());
4908	assert(L.isSimple());
4909	args.addUncopiedAggregate(LV: L, type);
4910	return;
4911	}
4912
4913	args.add(rvalue: EmitAnyExprToTemp(E), type);
4914	}
4915
4916	QualType CodeGenFunction::getVarArgType(const Expr *Arg) {
4917	// System headers on Windows define NULL to 0 instead of 0LL on Win64. MSVC
4918	// implicitly widens null pointer constants that are arguments to varargs
4919	// functions to pointer-sized ints.
4920	if (!getTarget().getTriple().isOSWindows())
4921	return Arg->getType();
4922
4923	if (Arg->getType()->isIntegerType() &&
4924	getContext().getTypeSize(T: Arg->getType()) <
4925	getContext().getTargetInfo().getPointerWidth(AddrSpace: LangAS::Default) &&
4926	Arg->isNullPointerConstant(Ctx&: getContext(),
4927	NPC: Expr::NPC_ValueDependentIsNotNull)) {
4928	return getContext().getIntPtrType();
4929	}
4930
4931	return Arg->getType();
4932	}
4933
4934	// In ObjC ARC mode with no ObjC ARC exception safety, tell the ARC
4935	// optimizer it can aggressively ignore unwind edges.
4936	void CodeGenFunction::AddObjCARCExceptionMetadata(llvm::Instruction *Inst) {
4937	if (CGM.getCodeGenOpts().OptimizationLevel != `0` &&
4938	!CGM.getCodeGenOpts().ObjCAutoRefCountExceptions)
4939	Inst->setMetadata(Kind: "clang.arc.no_objc_arc_exceptions",
4940	Node: CGM.getNoObjCARCExceptionsMetadata());
4941	}
4942
4943	/// Emits a call to the given no-arguments nounwind runtime function.
4944	llvm::CallInst *
4945	CodeGenFunction::EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
4946	const llvm::Twine &name) {
4947	return EmitNounwindRuntimeCall(callee, args: ArrayRef<llvm::Value *>(), name);
4948	}
4949
4950	/// Emits a call to the given nounwind runtime function.
4951	llvm::CallInst *
4952	CodeGenFunction::EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
4953	ArrayRef<Address> args,
4954	const llvm::Twine &name) {
4955	SmallVector<llvm::Value *, `3`> values;
4956	for (auto arg : args)
4957	values.push_back(Elt: arg.emitRawPointer(CGF&: *this));
4958	return EmitNounwindRuntimeCall(callee, args: values, name);
4959	}
4960
4961	llvm::CallInst *
4962	CodeGenFunction::EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
4963	ArrayRef<llvm::Value *> args,
4964	const llvm::Twine &name) {
4965	llvm::CallInst *call = EmitRuntimeCall(callee, args, name);
4966	call->setDoesNotThrow();
4967	return call;
4968	}
4969
4970	/// Emits a simple call (never an invoke) to the given no-arguments
4971	/// runtime function.
4972	llvm::CallInst *CodeGenFunction::EmitRuntimeCall(llvm::FunctionCallee callee,
4973	const llvm::Twine &name) {
4974	return EmitRuntimeCall(callee, args: {}, name);
4975	}
4976
4977	// Calls which may throw must have operand bundles indicating which funclet
4978	// they are nested within.
4979	SmallVector<llvm::OperandBundleDef, `1`>
4980	CodeGenFunction::getBundlesForFunclet(llvm::Value *Callee) {
4981	// There is no need for a funclet operand bundle if we aren't inside a
4982	// funclet.
4983	if (!CurrentFuncletPad)
4984	return (SmallVector<llvm::OperandBundleDef, `1`>());
4985
4986	// Skip intrinsics which cannot throw (as long as they don't lower into
4987	// regular function calls in the course of IR transformations).
4988	if (auto *CalleeFn = dyn_cast<llvm::Function>(Val: Callee->stripPointerCasts())) {
4989	if (CalleeFn->isIntrinsic() && CalleeFn->doesNotThrow()) {
4990	auto IID = CalleeFn->getIntrinsicID();
4991	if (!llvm::IntrinsicInst::mayLowerToFunctionCall(IID))
4992	return (SmallVector<llvm::OperandBundleDef, `1`>());
4993	}
4994	}
4995
4996	SmallVector<llvm::OperandBundleDef, `1`> BundleList;
4997	BundleList.emplace_back(Args: "funclet", Args&: CurrentFuncletPad);
4998	return BundleList;
4999	}
5000
5001	/// Emits a simple call (never an invoke) to the given runtime function.
5002	llvm::CallInst *CodeGenFunction::EmitRuntimeCall(llvm::FunctionCallee callee,
5003	ArrayRef<llvm::Value *> args,
5004	const llvm::Twine &name) {
5005	llvm::CallInst *call = Builder.CreateCall(
5006	Callee: callee, Args: args, OpBundles: getBundlesForFunclet(Callee: callee.getCallee()), Name: name);
5007	call->setCallingConv(getRuntimeCC());
5008
5009	if (CGM.shouldEmitConvergenceTokens() && call->isConvergent())
5010	return cast<llvm::CallInst>(Val: addConvergenceControlToken(Input: call));
5011	return call;
5012	}
5013
5014	/// Emits a call or invoke to the given noreturn runtime function.
5015	void CodeGenFunction::EmitNoreturnRuntimeCallOrInvoke(
5016	llvm::FunctionCallee callee, ArrayRef<llvm::Value *> args) {
5017	SmallVector<llvm::OperandBundleDef, `1`> BundleList =
5018	getBundlesForFunclet(Callee: callee.getCallee());
5019
5020	if (getInvokeDest()) {
5021	llvm::InvokeInst *invoke = Builder.CreateInvoke(
5022	Callee: callee, NormalDest: getUnreachableBlock(), UnwindDest: getInvokeDest(), Args: args, OpBundles: BundleList);
5023	invoke->setDoesNotReturn();
5024	invoke->setCallingConv(getRuntimeCC());
5025	} else {
5026	llvm::CallInst *call = Builder.CreateCall(Callee: callee, Args: args, OpBundles: BundleList);
5027	call->setDoesNotReturn();
5028	call->setCallingConv(getRuntimeCC());
5029	Builder.CreateUnreachable();
5030	}
5031	}
5032
5033	/// Emits a call or invoke instruction to the given nullary runtime function.
5034	llvm::CallBase *
5035	CodeGenFunction::EmitRuntimeCallOrInvoke(llvm::FunctionCallee callee,
5036	const Twine &name) {
5037	return EmitRuntimeCallOrInvoke(callee, args: {}, name);
5038	}
5039
5040	/// Emits a call or invoke instruction to the given runtime function.
5041	llvm::CallBase *
5042	CodeGenFunction::EmitRuntimeCallOrInvoke(llvm::FunctionCallee callee,
5043	ArrayRef<llvm::Value *> args,
5044	const Twine &name) {
5045	llvm::CallBase *call = EmitCallOrInvoke(Callee: callee, Args: args, Name: name);
5046	call->setCallingConv(getRuntimeCC());
5047	return call;
5048	}
5049
5050	/// Emits a call or invoke instruction to the given function, depending
5051	/// on the current state of the EH stack.
5052	llvm::CallBase *CodeGenFunction::EmitCallOrInvoke(llvm::FunctionCallee Callee,
5053	ArrayRef<llvm::Value *> Args,
5054	const Twine &Name) {
5055	llvm::BasicBlock *InvokeDest = getInvokeDest();
5056	SmallVector<llvm::OperandBundleDef, `1`> BundleList =
5057	getBundlesForFunclet(Callee: Callee.getCallee());
5058
5059	llvm::CallBase *Inst;
5060	if (!InvokeDest)
5061	Inst = Builder.CreateCall(Callee, Args, OpBundles: BundleList, Name);
5062	else {
5063	llvm::BasicBlock *ContBB = createBasicBlock(name: "invoke.cont");
5064	Inst = Builder.CreateInvoke(Callee, NormalDest: ContBB, UnwindDest: InvokeDest, Args, OpBundles: BundleList,
5065	Name);
5066	EmitBlock(BB: ContBB);
5067	}
5068
5069	// In ObjC ARC mode with no ObjC ARC exception safety, tell the ARC
5070	// optimizer it can aggressively ignore unwind edges.
5071	if (CGM.getLangOpts().ObjCAutoRefCount)
5072	AddObjCARCExceptionMetadata(Inst);
5073
5074	return Inst;
5075	}
5076
5077	void CodeGenFunction::deferPlaceholderReplacement(llvm::Instruction *Old,
5078	llvm::Value *New) {
5079	DeferredReplacements.push_back(
5080	Elt: std::make_pair(x: llvm::WeakTrackingVH (Old), y&: New));
5081	}
5082
5083	namespace {
5084
5085	/// Specify given \p NewAlign as the alignment of return value attribute. If
5086	/// such attribute already exists, re-set it to the maximal one of two options.
5087	[[nodiscard]] llvm::AttributeList
5088	maybeRaiseRetAlignmentAttribute(llvm::LLVMContext &Ctx,
5089	const llvm::AttributeList &Attrs,
5090	llvm::Align NewAlign) {
5091	llvm::Align CurAlign = Attrs.getRetAlignment().valueOrOne();
5092	if (CurAlign >= NewAlign)
5093	return Attrs;
5094	llvm::Attribute AlignAttr = llvm::Attribute::getWithAlignment(Context&: Ctx, Alignment: NewAlign);
5095	return Attrs.removeRetAttribute(Ctx, llvm::Attribute::AttrKind::Alignment)
5096	.addRetAttribute(Ctx, AlignAttr);
5097	}
5098
5099	template <typename AlignedAttrTy> class AbstractAssumeAlignedAttrEmitter {
5100	protected:
5101	CodeGenFunction &CGF;
5102
5103	/// We do nothing if this is, or becomes, nullptr.
5104	const AlignedAttrTy AA = nullptr*;
5105
5106	llvm::Value Alignment = nullptr; // May or may not be a constant.*
5107	llvm::ConstantInt OffsetCI = nullptr; // Constant, hopefully zero.*
5108
5109	AbstractAssumeAlignedAttrEmitter(CodeGenFunction &CGF_, const Decl *FuncDecl)
5110	: CGF(CGF_) {
5111	if (!FuncDecl)
5112	return;
5113	AA = FuncDecl->getAttr<AlignedAttrTy>();
5114	}
5115
5116	public:
5117	/// If we can, materialize the alignment as an attribute on return value.
5118	[[nodiscard]] llvm::AttributeList
5119	TryEmitAsCallSiteAttribute(const llvm::AttributeList &Attrs) {
5120	if (!AA \|\| OffsetCI \|\| CGF.SanOpts.has(K: SanitizerKind::Alignment))
5121	return Attrs;
5122	const auto *AlignmentCI = dyn_cast<llvm::ConstantInt>(Val: Alignment);
5123	if (!AlignmentCI)
5124	return Attrs;
5125	// We may legitimately have non-power-of-2 alignment here.
5126	// If so, this is UB land, emit it via `@llvm.assume` instead.
5127	if (!AlignmentCI->getValue().isPowerOf2())
5128	return Attrs;
5129	llvm::AttributeList NewAttrs = maybeRaiseRetAlignmentAttribute(
5130	Ctx&: CGF.getLLVMContext(), Attrs,
5131	NewAlign: llvm::Align (
5132	AlignmentCI->getLimitedValue(Limit: llvm::Value::MaximumAlignment)));
5133	AA = nullptr; // We're done. Disallow doing anything else.
5134	return NewAttrs;
5135	}
5136
5137	/// Emit alignment assumption.
5138	/// This is a general fallback that we take if either there is an offset,
5139	/// or the alignment is variable or we are sanitizing for alignment.
5140	void EmitAsAnAssumption(SourceLocation Loc, QualType RetTy, RValue &Ret) {
5141	if (!AA)
5142	return;
5143	CGF.emitAlignmentAssumption(Ret.getScalarVal(), RetTy, Loc,
5144	AA->getLocation(), Alignment, OffsetCI);
5145	AA = nullptr; // We're done. Disallow doing anything else.
5146	}
5147	};
5148
5149	/// Helper data structure to emit `AssumeAlignedAttr`.
5150	class AssumeAlignedAttrEmitter final
5151	: public AbstractAssumeAlignedAttrEmitter<AssumeAlignedAttr> {
5152	public:
5153	AssumeAlignedAttrEmitter(CodeGenFunction &CGF_, const Decl *FuncDecl)
5154	: AbstractAssumeAlignedAttrEmitter(CGF_, FuncDecl) {
5155	if (!AA)
5156	return;
5157	// It is guaranteed that the alignment/offset are constants.
5158	Alignment = cast<llvm::ConstantInt>(CGF.EmitScalarExpr(AA->getAlignment()));
5159	if (Expr *Offset = AA->getOffset()) {
5160	OffsetCI = cast<llvm::ConstantInt>(CGF.EmitScalarExpr(Offset));
5161	if (OffsetCI->isNullValue()) // Canonicalize zero offset to no offset.
5162	OffsetCI = nullptr;
5163	}
5164	}
5165	};
5166
5167	/// Helper data structure to emit `AllocAlignAttr`.
5168	class AllocAlignAttrEmitter final
5169	: public AbstractAssumeAlignedAttrEmitter<AllocAlignAttr> {
5170	public:
5171	AllocAlignAttrEmitter(CodeGenFunction &CGF_, const Decl *FuncDecl,
5172	const CallArgList &CallArgs)
5173	: AbstractAssumeAlignedAttrEmitter(CGF_, FuncDecl) {
5174	if (!AA)
5175	return;
5176	// Alignment may or may not be a constant, and that is okay.
5177	Alignment = CallArgs[AA->getParamIndex().getLLVMIndex()]
5178	.getRValue(CGF)
5179	.getScalarVal();
5180	}
5181	};
5182
5183	} // namespace
5184
5185	static unsigned getMaxVectorWidth(const llvm::Type *Ty) {
5186	if (auto *VT = dyn_cast<llvm::VectorType>(Val: Ty))
5187	return VT->getPrimitiveSizeInBits().getKnownMinValue();
5188	if (auto *AT = dyn_cast<llvm::ArrayType>(Val: Ty))
5189	return getMaxVectorWidth(Ty: AT->getElementType());
5190
5191	unsigned MaxVectorWidth = `0`;
5192	if (auto *ST = dyn_cast<llvm::StructType>(Val: Ty))
5193	for (auto *I : ST->elements())
5194	MaxVectorWidth = std::max(a: MaxVectorWidth, b: getMaxVectorWidth(Ty: I));
5195	return MaxVectorWidth;
5196	}
5197
5198	RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
5199	const CGCallee &Callee,
5200	ReturnValueSlot ReturnValue,
5201	const CallArgList &CallArgs,
5202	llvm::CallBase *callOrInvoke, bool* IsMustTail,
5203	SourceLocation Loc,
5204	bool IsVirtualFunctionPointerThunk) {
5205	// FIXME: We no longer need the types from CallArgs; lift up and simplify.
5206
5207	assert(Callee.isOrdinary() \|\| Callee.isVirtual());
5208
5209	// Handle struct-return functions by passing a pointer to the
5210	// location that we would like to return into.
5211	QualType RetTy = CallInfo.getReturnType();
5212	const ABIArgInfo &RetAI = CallInfo.getReturnInfo();
5213
5214	llvm::FunctionType *IRFuncTy = getTypes().GetFunctionType(FI: CallInfo);
5215
5216	const Decl *TargetDecl = Callee.getAbstractInfo().getCalleeDecl().getDecl();
5217	if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(Val: TargetDecl)) {
5218	// We can only guarantee that a function is called from the correct
5219	// context/function based on the appropriate target attributes,
5220	// so only check in the case where we have both always_inline and target
5221	// since otherwise we could be making a conditional call after a check for
5222	// the proper cpu features (and it won't cause code generation issues due to
5223	// function based code generation).
5224	if (TargetDecl->hasAttr<AlwaysInlineAttr>() &&
5225	(TargetDecl->hasAttr<TargetAttr>() \|\|
5226	(CurFuncDecl && CurFuncDecl->hasAttr<TargetAttr>())))
5227	checkTargetFeatures(Loc, TargetDecl: FD);
5228	}
5229
5230	// Some architectures (such as x86-64) have the ABI changed based on
5231	// attribute-target/features. Give them a chance to diagnose.
5232	const FunctionDecl *CallerDecl = dyn_cast_or_null<FunctionDecl>(Val: CurCodeDecl);
5233	const FunctionDecl *CalleeDecl = dyn_cast_or_null<FunctionDecl>(Val: TargetDecl);
5234	CGM.getTargetCodeGenInfo().checkFunctionCallABI(CGM, CallLoc: Loc, Caller: CallerDecl,
5235	Callee: CalleeDecl, Args: CallArgs, ReturnType: RetTy);
5236
5237	// 1. Set up the arguments.
5238
5239	// If we're using inalloca, insert the allocation after the stack save.
5240	// FIXME: Do this earlier rather than hacking it in here!
5241	RawAddress ArgMemory = RawAddress::invalid();
5242	if (llvm::StructType *ArgStruct = CallInfo.getArgStruct()) {
5243	const llvm::DataLayout &DL = CGM.getDataLayout();
5244	llvm::Instruction *IP = CallArgs.getStackBase();
5245	llvm::AllocaInst *AI;
5246	if (IP) {
5247	IP = IP->getNextNode();
5248	AI = new llvm::AllocaInst (ArgStruct, DL.getAllocaAddrSpace(), "argmem",
5249	IP->getIterator());
5250	} else {
5251	AI = CreateTempAlloca(Ty: ArgStruct, Name: "argmem");
5252	}
5253	auto Align = CallInfo.getArgStructAlignment();
5254	AI->setAlignment(Align.getAsAlign());
5255	AI->setUsedWithInAlloca(true);
5256	assert(AI->isUsedWithInAlloca() && !AI->isStaticAlloca());
5257	ArgMemory = RawAddress (AI, ArgStruct, Align);
5258	}
5259
5260	ClangToLLVMArgMapping IRFunctionArgs(CGM.getContext(), CallInfo);
5261	SmallVector<llvm::Value *, `16`> IRCallArgs(IRFunctionArgs.totalIRArgs());
5262
5263	// If the call returns a temporary with struct return, create a temporary
5264	// alloca to hold the result, unless one is given to us.
5265	Address SRetPtr = Address::invalid();
5266	llvm::Value UnusedReturnSizePtr = nullptr*;
5267	if (RetAI.isIndirect() \|\| RetAI.isInAlloca() \|\| RetAI.isCoerceAndExpand()) {
5268	// For virtual function pointer thunks and musttail calls, we must always
5269	// forward an incoming SRet pointer to the callee, because a local alloca
5270	// would be de-allocated before the call. These cases both guarantee that
5271	// there will be an incoming SRet argument of the correct type.
5272	if ((IsVirtualFunctionPointerThunk \|\| IsMustTail) && RetAI.isIndirect()) {
5273	SRetPtr = makeNaturalAddressForPointer(Ptr: CurFn->arg_begin() +
5274	IRFunctionArgs.getSRetArgNo(),
5275	T: RetTy, Alignment: CharUnits::fromQuantity(Quantity: `1`));
5276	} else if (!ReturnValue.isNull()) {
5277	SRetPtr = ReturnValue.getAddress();
5278	} else {
5279	SRetPtr = CreateMemTempWithoutCast(T: RetTy, Name: "tmp");
5280	if (HaveInsertPoint() && ReturnValue.isUnused()) {
5281	llvm::TypeSize size =
5282	CGM.getDataLayout().getTypeAllocSize(Ty: ConvertTypeForMem(T: RetTy));
5283	UnusedReturnSizePtr = EmitLifetimeStart(Size: size, Addr: SRetPtr.getBasePointer());
5284	}
5285	}
5286	if (IRFunctionArgs.hasSRetArg()) {
5287	// A mismatch between the allocated return value's AS and the target's
5288	// chosen IndirectAS can happen e.g. when passing the this pointer through
5289	// a chain involving stores to / loads from the DefaultAS; we address this
5290	// here, symmetrically with the handling we have for normal pointer args.
5291	if (SRetPtr.getAddressSpace() != RetAI.getIndirectAddrSpace()) {
5292	llvm::Value *V = SRetPtr.getBasePointer();
5293	LangAS SAS = getLangASFromTargetAS(TargetAS: SRetPtr.getAddressSpace());
5294	llvm::Type *Ty = llvm::PointerType::get(C&: getLLVMContext(),
5295	AddressSpace: RetAI.getIndirectAddrSpace());
5296
5297	SRetPtr = SRetPtr.withPointer(
5298	NewPointer: getTargetHooks().performAddrSpaceCast(CGF&: *this, V, SrcAddr: SAS, DestTy: Ty, IsNonNull: true),
5299	IsKnownNonNull: SRetPtr.isKnownNonNull());
5300	}
5301	IRCallArgs [IRFunctionArgs.getSRetArgNo()] =
5302	getAsNaturalPointerTo(Addr: SRetPtr, PointeeType: RetTy);
5303	} else if (RetAI.isInAlloca()) {
5304	Address Addr =
5305	Builder.CreateStructGEP(Addr: ArgMemory, Index: RetAI.getInAllocaFieldIndex());
5306	Builder.CreateStore(Val: getAsNaturalPointerTo(Addr: SRetPtr, PointeeType: RetTy), Addr);
5307	}
5308	}
5309
5310	RawAddress swiftErrorTemp = RawAddress::invalid();
5311	Address swiftErrorArg = Address::invalid();
5312
5313	// When passing arguments using temporary allocas, we need to add the
5314	// appropriate lifetime markers. This vector keeps track of all the lifetime
5315	// markers that need to be ended right after the call.
5316	SmallVector<CallLifetimeEnd, `2`> CallLifetimeEndAfterCall;
5317
5318	// Translate all of the arguments as necessary to match the IR lowering.
5319	assert(CallInfo.arg_size() == CallArgs.size() &&
5320	"Mismatch between function signature & arguments.");
5321	unsigned ArgNo = `0`;
5322	CGFunctionInfo::const_arg_iterator info_it = CallInfo.arg_begin();
5323	for (CallArgList::const_iterator I = CallArgs.begin(), E = CallArgs.end();
5324	I != E; ++I, ++info_it, ++ArgNo) {
5325	const ABIArgInfo &ArgInfo = info_it->info;
5326
5327	// Insert a padding argument to ensure proper alignment.
5328	if (IRFunctionArgs.hasPaddingArg(ArgNo))
5329	IRCallArgs [IRFunctionArgs.getPaddingArgNo(ArgNo)] =
5330	llvm::UndefValue::get(T: ArgInfo.getPaddingType());
5331
5332	unsigned FirstIRArg, NumIRArgs;
5333	std::tie(args&: FirstIRArg, args&: NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
5334
5335	bool ArgHasMaybeUndefAttr =
5336	IsArgumentMaybeUndef(TargetDecl, NumRequiredArgs: CallInfo.getNumRequiredArgs(), ArgNo);
5337
5338	switch (ArgInfo.getKind()) {
5339	case ABIArgInfo::InAlloca: {
5340	assert(NumIRArgs == `0`);
5341	assert(getTarget().getTriple().getArch() == llvm::Triple::x86);
5342	if (I->isAggregate()) {
5343	RawAddress Addr = I->hasLValue()
5344	? I->getKnownLValue().getAddress()
5345	: I->getKnownRValue().getAggregateAddress();
5346	llvm::Instruction *Placeholder =
5347	cast<llvm::Instruction>(Val: Addr.getPointer());
5348
5349	if (!ArgInfo.getInAllocaIndirect()) {
5350	// Replace the placeholder with the appropriate argument slot GEP.
5351	CGBuilderTy::InsertPoint IP = Builder.saveIP();
5352	Builder.SetInsertPoint(Placeholder);
5353	Addr = Builder.CreateStructGEP(Addr: ArgMemory,
5354	Index: ArgInfo.getInAllocaFieldIndex());
5355	Builder.restoreIP(IP);
5356	} else {
5357	// For indirect things such as overaligned structs, replace the
5358	// placeholder with a regular aggregate temporary alloca. Store the
5359	// address of this alloca into the struct.
5360	Addr = CreateMemTemp(info_it->type, "inalloca.indirect.tmp");
5361	Address ArgSlot = Builder.CreateStructGEP(
5362	Addr: ArgMemory, Index: ArgInfo.getInAllocaFieldIndex());
5363	Builder.CreateStore(Val: Addr.getPointer(), Addr: ArgSlot);
5364	}
5365	deferPlaceholderReplacement(Old: Placeholder, New: Addr.getPointer());
5366	} else if (ArgInfo.getInAllocaIndirect()) {
5367	// Make a temporary alloca and store the address of it into the argument
5368	// struct.
5369	RawAddress Addr = CreateMemTempWithoutCast(
5370	I->Ty, getContext().getTypeAlignInChars(I->Ty),
5371	"indirect-arg-temp");
5372	I->copyInto(CGF&: *this, Addr);
5373	Address ArgSlot =
5374	Builder.CreateStructGEP(Addr: ArgMemory, Index: ArgInfo.getInAllocaFieldIndex());
5375	Builder.CreateStore(Val: Addr.getPointer(), Addr: ArgSlot);
5376	} else {
5377	// Store the RValue into the argument struct.
5378	Address Addr =
5379	Builder.CreateStructGEP(Addr: ArgMemory, Index: ArgInfo.getInAllocaFieldIndex());
5380	Addr = Addr.withElementType(ElemTy: ConvertTypeForMem(T: I->Ty));
5381	I->copyInto(CGF&: *this, Addr);
5382	}
5383	break;
5384	}
5385
5386	case ABIArgInfo::Indirect:
5387	case ABIArgInfo::IndirectAliased: {
5388	assert(NumIRArgs == `1`);
5389	if (I->isAggregate()) {
5390	// We want to avoid creating an unnecessary temporary+copy here;
5391	// however, we need one in three cases:
5392	// 1. If the argument is not byval, and we are required to copy the
5393	// source. (This case doesn't occur on any common architecture.)
5394	// 2. If the argument is byval, RV is not sufficiently aligned, and
5395	// we cannot force it to be sufficiently aligned.
5396	// 3. If the argument is byval, but RV is not located in default
5397	// or alloca address space.
5398	Address Addr = I->hasLValue()
5399	? I->getKnownLValue().getAddress()
5400	: I->getKnownRValue().getAggregateAddress();
5401	CharUnits Align = ArgInfo.getIndirectAlign();
5402	const llvm::DataLayout *TD = &CGM.getDataLayout();
5403
5404	assert((FirstIRArg >= IRFuncTy->getNumParams() \|\|
5405	IRFuncTy->getParamType(FirstIRArg)->getPointerAddressSpace() ==
5406	TD->getAllocaAddrSpace()) &&
5407	"indirect argument must be in alloca address space");
5408
5409	bool NeedCopy = false;
5410	if (Addr.getAlignment() < Align &&
5411	llvm::getOrEnforceKnownAlignment(V: Addr.emitRawPointer(CGF&: *this),
5412	PrefAlign: Align.getAsAlign(),
5413	DL: *TD) < Align.getAsAlign()) {
5414	NeedCopy = true;
5415	} else if (I->hasLValue()) {
5416	auto LV = I->getKnownLValue();
5417
5418	bool isByValOrRef =
5419	ArgInfo.isIndirectAliased() \|\| ArgInfo.getIndirectByVal();
5420
5421	if (!isByValOrRef \|\|
5422	(LV.getAlignment() < getContext().getTypeAlignInChars(I->Ty))) {
5423	NeedCopy = true;
5424	}
5425
5426	if (isByValOrRef && Addr.getType()->getAddressSpace() !=
5427	ArgInfo.getIndirectAddrSpace()) {
5428	NeedCopy = true;
5429	}
5430	}
5431
5432	if (!NeedCopy) {
5433	// Skip the extra memcpy call.
5434	llvm::Value *V = getAsNaturalPointerTo(Addr, PointeeType: I->Ty);
5435	auto *T = llvm::PointerType::get(C&: CGM.getLLVMContext(),
5436	AddressSpace: ArgInfo.getIndirectAddrSpace());
5437
5438	// FIXME: This should not depend on the language address spaces, and
5439	// only the contextual values. If the address space mismatches, see if
5440	// we can look through a cast to a compatible address space value,
5441	// otherwise emit a copy.
5442	llvm::Value *Val = getTargetHooks().performAddrSpaceCast(
5443	*this, V, I->Ty.getAddressSpace(), T, true);
5444	if (ArgHasMaybeUndefAttr)
5445	Val = Builder.CreateFreeze(V: Val);
5446	IRCallArgs [FirstIRArg] = Val;
5447	break;
5448	}
5449	} else if (I->getType()->isArrayParameterType()) {
5450	// Don't produce a temporary for ArrayParameterType arguments.
5451	// ArrayParameterType arguments are only created from
5452	// HLSL_ArrayRValue casts and HLSLOutArgExpr expressions, both
5453	// of which create temporaries already. This allows us to just use the
5454	// scalar for the decayed array pointer as the argument directly.
5455	IRCallArgs [FirstIRArg] = I->getKnownRValue().getScalarVal();
5456	break;
5457	}
5458
5459	// For non-aggregate args and aggregate args meeting conditions above
5460	// we need to create an aligned temporary, and copy to it.
5461	RawAddress AI = CreateMemTempWithoutCast(
5462	I->Ty, ArgInfo.getIndirectAlign(), "byval-temp");
5463	llvm::Value *Val = getAsNaturalPointerTo(Addr: AI, PointeeType: I->Ty);
5464	if (ArgHasMaybeUndefAttr)
5465	Val = Builder.CreateFreeze(V: Val);
5466	IRCallArgs [FirstIRArg] = Val;
5467
5468	// Emit lifetime markers for the temporary alloca.
5469	llvm::TypeSize ByvalTempElementSize =
5470	CGM.getDataLayout().getTypeAllocSize(Ty: AI.getElementType());
5471	llvm::Value *LifetimeSize =
5472	EmitLifetimeStart(Size: ByvalTempElementSize, Addr: AI.getPointer());
5473
5474	// Add cleanup code to emit the end lifetime marker after the call.
5475	if (LifetimeSize) // In case we disabled lifetime markers.
5476	CallLifetimeEndAfterCall.emplace_back(Args&: AI, Args&: LifetimeSize);
5477
5478	// Generate the copy.
5479	I->copyInto(CGF&: *this, Addr: AI);
5480	break;
5481	}
5482
5483	case ABIArgInfo::Ignore:
5484	assert(NumIRArgs == `0`);
5485	break;
5486
5487	case ABIArgInfo::Extend:
5488	case ABIArgInfo::Direct: {
5489	if (!isa<llvm::StructType>(Val: ArgInfo.getCoerceToType()) &&
5490	ArgInfo.getCoerceToType() == ConvertType(info_it->type) &&
5491	ArgInfo.getDirectOffset() == `0`) {
5492	assert(NumIRArgs == `1`);
5493	llvm::Value *V;
5494	if (!I->isAggregate())
5495	V = I->getKnownRValue().getScalarVal();
5496	else
5497	V = Builder.CreateLoad(
5498	Addr: I->hasLValue() ? I->getKnownLValue().getAddress()
5499	: I->getKnownRValue().getAggregateAddress());
5500
5501	// Implement swifterror by copying into a new swifterror argument.
5502	// We'll write back in the normal path out of the call.
5503	if (CallInfo.getExtParameterInfo(argIndex: ArgNo).getABI() ==
5504	ParameterABI::SwiftErrorResult) {
5505	assert(!swiftErrorTemp.isValid() && "multiple swifterror args");
5506
5507	QualType pointeeTy = I->Ty->getPointeeType();
5508	swiftErrorArg = makeNaturalAddressForPointer(
5509	Ptr: V, T: pointeeTy, Alignment: getContext().getTypeAlignInChars(T: pointeeTy));
5510
5511	swiftErrorTemp =
5512	CreateMemTemp(pointeeTy, getPointerAlign(), "swifterror.temp");
5513	V = swiftErrorTemp.getPointer();
5514	cast<llvm::AllocaInst>(Val: V)->setSwiftError(true);
5515
5516	llvm::Value *errorValue = Builder.CreateLoad(Addr: swiftErrorArg);
5517	Builder.CreateStore(Val: errorValue, Addr: swiftErrorTemp);
5518	}
5519
5520	// We might have to widen integers, but we should never truncate.
5521	if (ArgInfo.getCoerceToType() != V->getType() &&
5522	V->getType()->isIntegerTy())
5523	V = Builder.CreateZExt(V, DestTy: ArgInfo.getCoerceToType());
5524
5525	// The only plausible mismatch here would be for pointer address spaces.
5526	// We assume that the target has a reasonable mapping for the DefaultAS
5527	// (it can be casted to from incoming specific ASes), and insert an AS
5528	// cast to address the mismatch.
5529	if (FirstIRArg < IRFuncTy->getNumParams() &&
5530	V->getType() != IRFuncTy->getParamType(i: FirstIRArg)) {
5531	assert(V->getType()->isPointerTy() && "Only pointers can mismatch!");
5532	auto ActualAS = I->Ty.getAddressSpace();
5533	V = getTargetHooks().performAddrSpaceCast(
5534	*this, V, ActualAS, IRFuncTy->getParamType(i: FirstIRArg));
5535	}
5536
5537	if (ArgHasMaybeUndefAttr)
5538	V = Builder.CreateFreeze(V);
5539	IRCallArgs [FirstIRArg] = V;
5540	break;
5541	}
5542
5543	llvm::StructType *STy =
5544	dyn_cast<llvm::StructType>(Val: ArgInfo.getCoerceToType());
5545
5546	// FIXME: Avoid the conversion through memory if possible.
5547	Address Src = Address::invalid();
5548	if (!I->isAggregate()) {
5549	Src = CreateMemTemp(I->Ty, "coerce");
5550	I->copyInto(CGF&: *this, Addr: Src);
5551	} else {
5552	Src = I->hasLValue() ? I->getKnownLValue().getAddress()
5553	: I->getKnownRValue().getAggregateAddress();
5554	}
5555
5556	// If the value is offset in memory, apply the offset now.
5557	Src = emitAddressAtOffset(CGF&: *this, addr: Src, info: ArgInfo);
5558
5559	// Fast-isel and the optimizer generally like scalar values better than
5560	// FCAs, so we flatten them if this is safe to do for this argument.
5561	if (STy && ArgInfo.isDirect() && ArgInfo.getCanBeFlattened()) {
5562	llvm::Type *SrcTy = Src.getElementType();
5563	llvm::TypeSize SrcTypeSize =
5564	CGM.getDataLayout().getTypeAllocSize(Ty: SrcTy);
5565	llvm::TypeSize DstTypeSize = CGM.getDataLayout().getTypeAllocSize(Ty: STy);
5566	if (SrcTypeSize.isScalable()) {
5567	assert(STy->containsHomogeneousScalableVectorTypes() &&
5568	"ABI only supports structure with homogeneous scalable vector "
5569	"type");
5570	assert(SrcTypeSize == DstTypeSize &&
5571	"Only allow non-fractional movement of structure with "
5572	"homogeneous scalable vector type");
5573	assert(NumIRArgs == STy->getNumElements());
5574
5575	llvm::Value *StoredStructValue =
5576	Builder.CreateLoad(Addr: Src, Name: Src.getName() + ".tuple");
5577	for (unsigned i = `0`, e = STy->getNumElements(); i != e; ++i) {
5578	llvm::Value *Extract = Builder.CreateExtractValue(
5579	Agg: StoredStructValue, Idxs: i, Name: Src.getName() + ".extract" + Twine(i));
5580	IRCallArgs [FirstIRArg + i] = Extract;
5581	}
5582	} else {
5583	uint64_t SrcSize = SrcTypeSize.getFixedValue();
5584	uint64_t DstSize = DstTypeSize.getFixedValue();
5585
5586	// If the source type is smaller than the destination type of the
5587	// coerce-to logic, copy the source value into a temp alloca the size
5588	// of the destination type to allow loading all of it. The bits past
5589	// the source value are left undef.
5590	if (SrcSize < DstSize) {
5591	Address TempAlloca = CreateTempAlloca(Ty: STy, align: Src.getAlignment(),
5592	Name: Src.getName() + ".coerce");
5593	Builder.CreateMemCpy(Dest: TempAlloca, Src, Size: SrcSize);
5594	Src = TempAlloca;
5595	} else {
5596	Src = Src.withElementType(ElemTy: STy);
5597	}
5598
5599	assert(NumIRArgs == STy->getNumElements());
5600	for (unsigned i = `0`, e = STy->getNumElements(); i != e; ++i) {
5601	Address EltPtr = Builder.CreateStructGEP(Addr: Src, Index: i);
5602	llvm::Value *LI = Builder.CreateLoad(Addr: EltPtr);
5603	if (ArgHasMaybeUndefAttr)
5604	LI = Builder.CreateFreeze(V: LI);
5605	IRCallArgs [FirstIRArg + i] = LI;
5606	}
5607	}
5608	} else {
5609	// In the simple case, just pass the coerced loaded value.
5610	assert(NumIRArgs == `1`);
5611	llvm::Value *Load =
5612	CreateCoercedLoad(Src, Ty: ArgInfo.getCoerceToType(), CGF&: *this);
5613
5614	if (CallInfo.isCmseNSCall()) {
5615	// For certain parameter types, clear padding bits, as they may reveal
5616	// sensitive information.
5617	// Small struct/union types are passed as integer arrays.
5618	auto *ATy = dyn_cast<llvm::ArrayType>(Val: Load->getType());
5619	if (ATy != nullptr && isa<RecordType>(I->Ty.getCanonicalType()))
5620	Load = EmitCMSEClearRecord(Load, ATy, I->Ty);
5621	}
5622
5623	if (ArgHasMaybeUndefAttr)
5624	Load = Builder.CreateFreeze(V: Load);
5625	IRCallArgs [FirstIRArg] = Load;
5626	}
5627
5628	break;
5629	}
5630
5631	case ABIArgInfo::CoerceAndExpand: {
5632	auto coercionType = ArgInfo.getCoerceAndExpandType();
5633	auto layout = CGM.getDataLayout().getStructLayout(Ty: coercionType);
5634	auto unpaddedCoercionType = ArgInfo.getUnpaddedCoerceAndExpandType();
5635	auto *unpaddedStruct = dyn_cast<llvm::StructType>(Val: unpaddedCoercionType);
5636
5637	llvm::Value tempSize = nullptr*;
5638	Address addr = Address::invalid();
5639	RawAddress AllocaAddr = RawAddress::invalid();
5640	if (I->isAggregate()) {
5641	addr = I->hasLValue() ? I->getKnownLValue().getAddress()
5642	: I->getKnownRValue().getAggregateAddress();
5643
5644	} else {
5645	RValue RV = I->getKnownRValue();
5646	assert(RV.isScalar()); // complex should always just be direct
5647
5648	llvm::Type *scalarType = RV.getScalarVal()->getType();
5649	auto scalarSize = CGM.getDataLayout().getTypeAllocSize(Ty: scalarType);
5650	auto scalarAlign = CGM.getDataLayout().getPrefTypeAlign(Ty: scalarType);
5651
5652	// Materialize to a temporary.
5653	addr = CreateTempAlloca(Ty: RV.getScalarVal()->getType(),
5654	align: CharUnits::fromQuantity(Quantity: std::max(
5655	a: layout->getAlignment(), b: scalarAlign)),
5656	Name: "tmp",
5657	/ArraySize=/nullptr, Alloca: &AllocaAddr);
5658	tempSize = EmitLifetimeStart(Size: scalarSize, Addr: AllocaAddr.getPointer());
5659
5660	Builder.CreateStore(Val: RV.getScalarVal(), Addr: addr);
5661	}
5662
5663	addr = addr.withElementType(ElemTy: coercionType);
5664
5665	unsigned IRArgPos = FirstIRArg;
5666	unsigned unpaddedIndex = `0`;
5667	for (unsigned i = `0`, e = coercionType->getNumElements(); i != e; ++i) {
5668	llvm::Type *eltType = coercionType->getElementType(N: i);
5669	if (ABIArgInfo::isPaddingForCoerceAndExpand(eltType))
5670	continue;
5671	Address eltAddr = Builder.CreateStructGEP(Addr: addr, Index: i);
5672	llvm::Value *elt = CreateCoercedLoad(
5673	Src: eltAddr,
5674	Ty: unpaddedStruct ? unpaddedStruct->getElementType(N: unpaddedIndex++)
5675	: unpaddedCoercionType,
5676	CGF&: *this);
5677	if (ArgHasMaybeUndefAttr)
5678	elt = Builder.CreateFreeze(V: elt);
5679	IRCallArgs [IRArgPos++] = elt;
5680	}
5681	assert(IRArgPos == FirstIRArg + NumIRArgs);
5682
5683	if (tempSize) {
5684	EmitLifetimeEnd(Size: tempSize, Addr: AllocaAddr.getPointer());
5685	}
5686
5687	break;
5688	}
5689
5690	case ABIArgInfo::Expand: {
5691	unsigned IRArgPos = FirstIRArg;
5692	ExpandTypeToArgs(Ty: I->Ty, Arg: *I, IRFuncTy, IRCallArgs, IRCallArgPos&: IRArgPos);
5693	assert(IRArgPos == FirstIRArg + NumIRArgs);
5694	break;
5695	}
5696	}
5697	}
5698
5699	const CGCallee &ConcreteCallee = Callee.prepareConcreteCallee(CGF&: *this);
5700	llvm::Value *CalleePtr = ConcreteCallee.getFunctionPointer();
5701
5702	// If we're using inalloca, set up that argument.
5703	if (ArgMemory.isValid()) {
5704	llvm::Value *Arg = ArgMemory.getPointer();
5705	assert(IRFunctionArgs.hasInallocaArg());
5706	IRCallArgs [IRFunctionArgs.getInallocaArgNo()] = Arg;
5707	}
5708
5709	// 2. Prepare the function pointer.
5710
5711	// If the callee is a bitcast of a non-variadic function to have a
5712	// variadic function pointer type, check to see if we can remove the
5713	// bitcast. This comes up with unprototyped functions.
5714	//
5715	// This makes the IR nicer, but more importantly it ensures that we
5716	// can inline the function at -O0 if it is marked always_inline.
5717	auto simplifyVariadicCallee = [](llvm::FunctionType *CalleeFT,
5718	llvm::Value Ptr) -> llvm::Function {
5719	if (!CalleeFT->isVarArg())
5720	return nullptr;
5721
5722	// Get underlying value if it's a bitcast
5723	if (llvm::ConstantExpr *CE = dyn_cast<llvm::ConstantExpr>(Val: Ptr)) {
5724	if (CE->getOpcode() == llvm::Instruction::BitCast)
5725	Ptr = CE->getOperand(i_nocapture: `0`);
5726	}
5727
5728	llvm::Function *OrigFn = dyn_cast<llvm::Function>(Val: Ptr);
5729	if (!OrigFn)
5730	return nullptr;
5731
5732	llvm::FunctionType *OrigFT = OrigFn->getFunctionType();
5733
5734	// If the original type is variadic, or if any of the component types
5735	// disagree, we cannot remove the cast.
5736	if (OrigFT->isVarArg() \|\|
5737	OrigFT->getNumParams() != CalleeFT->getNumParams() \|\|
5738	OrigFT->getReturnType() != CalleeFT->getReturnType())
5739	return nullptr;
5740
5741	for (unsigned i = `0`, e = OrigFT->getNumParams(); i != e; ++i)
5742	if (OrigFT->getParamType(i) != CalleeFT->getParamType(i))
5743	return nullptr;
5744
5745	return OrigFn;
5746	};
5747
5748	if (llvm::Function *OrigFn = simplifyVariadicCallee (IRFuncTy, CalleePtr)) {
5749	CalleePtr = OrigFn;
5750	IRFuncTy = OrigFn->getFunctionType();
5751	}
5752
5753	// 3. Perform the actual call.
5754
5755	// Deactivate any cleanups that we're supposed to do immediately before
5756	// the call.
5757	if (!CallArgs.getCleanupsToDeactivate().empty())
5758	deactivateArgCleanupsBeforeCall(CGF&: *this, CallArgs);
5759
5760	// Update the largest vector width if any arguments have vector types.
5761	for (unsigned i = `0`; i < IRCallArgs.size(); ++i)
5762	LargestVectorWidth = std::max(a: LargestVectorWidth,
5763	b: getMaxVectorWidth(Ty: IRCallArgs [i]->getType()));
5764
5765	// Compute the calling convention and attributes.
5766	unsigned CallingConv;
5767	llvm::AttributeList Attrs;
5768	CGM.ConstructAttributeList(Name: CalleePtr->getName(), FI: CallInfo,
5769	CalleeInfo: Callee.getAbstractInfo(), AttrList&: Attrs, CallingConv,
5770	/AttrOnCallSite=/true,
5771	/IsThunk=/false);
5772
5773	if (CallingConv == llvm::CallingConv::X86_VectorCall &&
5774	getTarget().getTriple().isWindowsArm64EC()) {
5775	CGM.Error(loc: Loc, error: "__vectorcall calling convention is not currently "
5776	"supported");
5777	}
5778
5779	if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(Val: CurFuncDecl)) {
5780	if (FD->hasAttr<StrictFPAttr>())
5781	// All calls within a strictfp function are marked strictfp
5782	Attrs = Attrs.addFnAttribute(getLLVMContext(), llvm::Attribute::StrictFP);
5783
5784	// If -ffast-math is enabled and the function is guarded by an
5785	// '__attribute__((optnone)) adjust the memory attribute so the BE emits the
5786	// library call instead of the intrinsic.
5787	if (FD->hasAttr<OptimizeNoneAttr>() && getLangOpts().FastMath)
5788	CGM.AdjustMemoryAttribute(Name: CalleePtr->getName(), CalleeInfo: Callee.getAbstractInfo(),
5789	Attrs);
5790	}
5791	// Add call-site nomerge attribute if exists.
5792	if (InNoMergeAttributedStmt)
5793	Attrs = Attrs.addFnAttribute(getLLVMContext(), llvm::Attribute::NoMerge);
5794
5795	// Add call-site noinline attribute if exists.
5796	if (InNoInlineAttributedStmt)
5797	Attrs = Attrs.addFnAttribute(getLLVMContext(), llvm::Attribute::NoInline);
5798
5799	// Add call-site always_inline attribute if exists.
5800	// Note: This corresponds to the [[clang::always_inline]] statement attribute.
5801	if (InAlwaysInlineAttributedStmt &&
5802	!CGM.getTargetCodeGenInfo().wouldInliningViolateFunctionCallABI(
5803	CallerDecl, CalleeDecl))
5804	Attrs =
5805	Attrs.addFnAttribute(getLLVMContext(), llvm::Attribute::AlwaysInline);
5806
5807	// Remove call-site convergent attribute if requested.
5808	if (InNoConvergentAttributedStmt)
5809	Attrs =
5810	Attrs.removeFnAttribute(getLLVMContext(), llvm::Attribute::Convergent);
5811
5812	// Apply some call-site-specific attributes.
5813	// TODO: work this into building the attribute set.
5814
5815	// Apply always_inline to all calls within flatten functions.
5816	// FIXME: should this really take priority over __try, below?
5817	if (CurCodeDecl && CurCodeDecl->hasAttr<FlattenAttr>() &&
5818	!InNoInlineAttributedStmt &&
5819	!(TargetDecl && TargetDecl->hasAttr<NoInlineAttr>()) &&
5820	!CGM.getTargetCodeGenInfo().wouldInliningViolateFunctionCallABI(
5821	CallerDecl, CalleeDecl)) {
5822	Attrs =
5823	Attrs.addFnAttribute(getLLVMContext(), llvm::Attribute::AlwaysInline);
5824	}
5825
5826	// Disable inlining inside SEH __try blocks.
5827	if (isSEHTryScope()) {
5828	Attrs = Attrs.addFnAttribute(getLLVMContext(), llvm::Attribute::NoInline);
5829	}
5830
5831	// Decide whether to use a call or an invoke.
5832	bool CannotThrow;
5833	if (currentFunctionUsesSEHTry()) {
5834	// SEH cares about asynchronous exceptions, so everything can "throw."
5835	CannotThrow = false;
5836	} else if (isCleanupPadScope() &&
5837	EHPersonality::get(CGF&: *this).isMSVCXXPersonality()) {
5838	// The MSVC++ personality will implicitly terminate the program if an
5839	// exception is thrown during a cleanup outside of a try/catch.
5840	// We don't need to model anything in IR to get this behavior.
5841	CannotThrow = true;
5842	} else {
5843	// Otherwise, nounwind call sites will never throw.
5844	CannotThrow = Attrs.hasFnAttr(llvm::Attribute::NoUnwind);
5845
5846	if (auto *FPtr = dyn_cast<llvm::Function>(Val: CalleePtr))
5847	if (FPtr->hasFnAttribute(llvm::Attribute::NoUnwind))
5848	CannotThrow = true;
5849	}
5850
5851	// If we made a temporary, be sure to clean up after ourselves. Note that we
5852	// can't depend on being inside of an ExprWithCleanups, so we need to manually
5853	// pop this cleanup later on. Being eager about this is OK, since this
5854	// temporary is 'invisible' outside of the callee.
5855	if (UnusedReturnSizePtr)
5856	pushFullExprCleanup<CallLifetimeEnd>(kind: NormalEHLifetimeMarker, A: SRetPtr,
5857	A: UnusedReturnSizePtr);
5858
5859	llvm::BasicBlock InvokeDest = CannotThrow ? nullptr* : getInvokeDest();
5860
5861	SmallVector<llvm::OperandBundleDef, `1`> BundleList =
5862	getBundlesForFunclet(Callee: CalleePtr);
5863
5864	if (SanOpts.has(K: SanitizerKind::KCFI) &&
5865	!isa_and_nonnull<FunctionDecl>(Val: TargetDecl))
5866	EmitKCFIOperandBundle(Callee: ConcreteCallee, Bundles&: BundleList);
5867
5868	// Add the pointer-authentication bundle.
5869	EmitPointerAuthOperandBundle(Info: ConcreteCallee.getPointerAuthInfo(), Bundles&: BundleList);
5870
5871	if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(CurFuncDecl))
5872	if (FD->hasAttr<StrictFPAttr>())
5873	// All calls within a strictfp function are marked strictfp
5874	Attrs = Attrs.addFnAttribute(getLLVMContext(), llvm::Attribute::StrictFP);
5875
5876	AssumeAlignedAttrEmitter AssumeAlignedAttrEmitter(*this, TargetDecl);
5877	Attrs = AssumeAlignedAttrEmitter.TryEmitAsCallSiteAttribute(Attrs);
5878
5879	AllocAlignAttrEmitter AllocAlignAttrEmitter(*this, TargetDecl, CallArgs);
5880	Attrs = AllocAlignAttrEmitter.TryEmitAsCallSiteAttribute(Attrs);
5881
5882	// Emit the actual call/invoke instruction.
5883	llvm::CallBase *CI;
5884	if (!InvokeDest) {
5885	CI = Builder.CreateCall(FTy: IRFuncTy, Callee: CalleePtr, Args: IRCallArgs, OpBundles: BundleList);
5886	} else {
5887	llvm::BasicBlock *Cont = createBasicBlock(name: "invoke.cont");
5888	CI = Builder.CreateInvoke(Ty: IRFuncTy, Callee: CalleePtr, NormalDest: Cont, UnwindDest: InvokeDest, Args: IRCallArgs,
5889	OpBundles: BundleList);
5890	EmitBlock(BB: Cont);
5891	}
5892	if (CI->getCalledFunction() && CI->getCalledFunction()->hasName() &&
5893	CI->getCalledFunction()->getName().starts_with(Prefix: "_Z4sqrt")) {
5894	SetSqrtFPAccuracy(CI);
5895	}
5896	if (callOrInvoke)
5897	*callOrInvoke = CI;
5898
5899	// If this is within a function that has the guard(nocf) attribute and is an
5900	// indirect call, add the "guard_nocf" attribute to this call to indicate that
5901	// Control Flow Guard checks should not be added, even if the call is inlined.
5902	if (const auto *FD = dyn_cast_or_null<FunctionDecl>(Val: CurFuncDecl)) {
5903	if (const auto *A = FD->getAttr<CFGuardAttr>()) {
5904	if (A->getGuard() == CFGuardAttr::GuardArg::nocf &&
5905	!CI->getCalledFunction())
5906	Attrs = Attrs.addFnAttribute(C&: getLLVMContext(), Kind: "guard_nocf");
5907	}
5908	}
5909
5910	// Apply the attributes and calling convention.
5911	CI->setAttributes(Attrs);
5912	CI->setCallingConv(static_cast<llvm::CallingConv::ID>(CallingConv));
5913
5914	// Apply various metadata.
5915
5916	if (!CI->getType()->isVoidTy())
5917	CI->setName("call");
5918
5919	if (CGM.shouldEmitConvergenceTokens() && CI->isConvergent())
5920	CI = addConvergenceControlToken(Input: CI);
5921
5922	// Update largest vector width from the return type.
5923	LargestVectorWidth =
5924	std::max(a: LargestVectorWidth, b: getMaxVectorWidth(Ty: CI->getType()));
5925
5926	// Insert instrumentation or attach profile metadata at indirect call sites.
5927	// For more details, see the comment before the definition of
5928	// IPVK_IndirectCallTarget in InstrProfData.inc.
5929	if (!CI->getCalledFunction())
5930	PGO ->valueProfile(Builder, ValueKind: llvm::IPVK_IndirectCallTarget, ValueSite: CI, ValuePtr: CalleePtr);
5931
5932	// In ObjC ARC mode with no ObjC ARC exception safety, tell the ARC
5933	// optimizer it can aggressively ignore unwind edges.
5934	if (CGM.getLangOpts().ObjCAutoRefCount)
5935	AddObjCARCExceptionMetadata(Inst: CI);
5936
5937	// Set tail call kind if necessary.
5938	if (llvm::CallInst *Call = dyn_cast<llvm::CallInst>(Val: CI)) {
5939	if (TargetDecl && TargetDecl->hasAttr<NotTailCalledAttr>())
5940	Call->setTailCallKind(llvm::CallInst::TCK_NoTail);
5941	else if (IsMustTail) {
5942	if (getTarget().getTriple().isPPC()) {
5943	if (getTarget().getTriple().isOSAIX())
5944	CGM.getDiags().Report(Loc, diag::err_aix_musttail_unsupported);
5945	else if (!getTarget().hasFeature(Feature: "pcrelative-memops")) {
5946	if (getTarget().hasFeature(Feature: "longcall"))
5947	CGM.getDiags().Report(Loc, diag::err_ppc_impossible_musttail) << `0`;
5948	else if (Call->isIndirectCall())
5949	CGM.getDiags().Report(Loc, diag::err_ppc_impossible_musttail) << `1`;
5950	else if (isa_and_nonnull<FunctionDecl>(Val: TargetDecl)) {
5951	if (!cast<FunctionDecl>(Val: TargetDecl)->isDefined())
5952	// The undefined callee may be a forward declaration. Without
5953	// knowning all symbols in the module, we won't know the symbol is
5954	// defined or not. Collect all these symbols for later diagnosing.
5955	CGM.addUndefinedGlobalForTailCall(
5956	Global: {cast<FunctionDecl>(Val: TargetDecl), Loc});
5957	else {
5958	llvm::GlobalValue::LinkageTypes Linkage = CGM.getFunctionLinkage(
5959	GD: GlobalDecl (cast<FunctionDecl>(Val: TargetDecl)));
5960	if (llvm::GlobalValue::isWeakForLinker(Linkage) \|\|
5961	llvm::GlobalValue::isDiscardableIfUnused(Linkage))
5962	CGM.getDiags().Report(Loc, diag::err_ppc_impossible_musttail)
5963	<< `2`;
5964	}
5965	}
5966	}
5967	}
5968	Call->setTailCallKind(llvm::CallInst::TCK_MustTail);
5969	}
5970	}
5971
5972	// Add metadata for calls to MSAllocator functions
5973	if (getDebugInfo() && TargetDecl && TargetDecl->hasAttr<MSAllocatorAttr>())
5974	getDebugInfo()->addHeapAllocSiteMetadata(CallSite: CI, AllocatedTy: RetTy ->getPointeeType(), Loc);
5975
5976	// Add metadata if calling an __attribute__((error(""))) or warning fn.
5977	if (TargetDecl && TargetDecl->hasAttr<ErrorAttr>()) {
5978	llvm::ConstantInt *Line =
5979	llvm::ConstantInt::get(Ty: Int64Ty, V: Loc.getRawEncoding());
5980	llvm::ConstantAsMetadata *MD = llvm::ConstantAsMetadata::get(C: Line);
5981	llvm::MDTuple *MDT = llvm::MDNode::get(Context&: getLLVMContext(), MDs: {MD});
5982	CI->setMetadata(Kind: "srcloc", Node: MDT);
5983	}
5984
5985	// 4. Finish the call.
5986
5987	// If the call doesn't return, finish the basic block and clear the
5988	// insertion point; this allows the rest of IRGen to discard
5989	// unreachable code.
5990	if (CI->doesNotReturn()) {
5991	if (UnusedReturnSizePtr)
5992	PopCleanupBlock();
5993
5994	// Strip away the noreturn attribute to better diagnose unreachable UB.
5995	if (SanOpts.has(K: SanitizerKind::Unreachable)) {
5996	// Also remove from function since CallBase::hasFnAttr additionally checks
5997	// attributes of the called function.
5998	if (auto *F = CI->getCalledFunction())
5999	F->removeFnAttr(llvm::Attribute::NoReturn);
6000	CI->removeFnAttr(llvm::Attribute::NoReturn);
6001
6002	// Avoid incompatibility with ASan which relies on the `noreturn`
6003	// attribute to insert handler calls.
6004	if (SanOpts.hasOneOf(K: SanitizerKind::Address \|
6005	SanitizerKind::KernelAddress)) {
6006	SanitizerScope SanScope(this);
6007	llvm::IRBuilder<>::InsertPointGuard IPGuard(Builder);
6008	Builder.SetInsertPoint(CI);
6009	auto FnType = llvm::FunctionType::get(Result: CGM.VoidTy, /isVarArg=/*false);
6010	llvm::FunctionCallee Fn =
6011	CGM.CreateRuntimeFunction(Ty: FnType, Name: "__asan_handle_no_return");
6012	EmitNounwindRuntimeCall(callee: Fn);
6013	}
6014	}
6015
6016	EmitUnreachable(Loc);
6017	Builder.ClearInsertionPoint();
6018
6019	// FIXME: For now, emit a dummy basic block because expr emitters in
6020	// generally are not ready to handle emitting expressions at unreachable
6021	// points.
6022	EnsureInsertPoint();
6023
6024	// Return a reasonable RValue.
6025	return GetUndefRValue(Ty: RetTy);
6026	}
6027
6028	// If this is a musttail call, return immediately. We do not branch to the
6029	// epilogue in this case.
6030	if (IsMustTail) {
6031	for (auto it = EHStack.find(sp: CurrentCleanupScopeDepth); it != EHStack.end();
6032	++it) {
6033	EHCleanupScope Cleanup = dyn_cast<EHCleanupScope>(Val: &it);
6034	// Fake uses can be safely emitted immediately prior to the tail call, so
6035	// we choose to emit them just before the call here.
6036	if (Cleanup && Cleanup->isFakeUse()) {
6037	CGBuilderTy::InsertPointGuard IPG(Builder);
6038	Builder.SetInsertPoint(CI);
6039	Cleanup->getCleanup()->Emit(CGF&: *this, flags: EHScopeStack::Cleanup::Flags ());
6040	} else if (!(Cleanup &&
6041	Cleanup->getCleanup()->isRedundantBeforeReturn())) {
6042	CGM.ErrorUnsupported(MustTailCall, "tail call skipping over cleanups");
6043	}
6044	}
6045	if (CI->getType()->isVoidTy())
6046	Builder.CreateRetVoid();
6047	else
6048	Builder.CreateRet(V: CI);
6049	Builder.ClearInsertionPoint();
6050	EnsureInsertPoint();
6051	return GetUndefRValue(Ty: RetTy);
6052	}
6053
6054	// Perform the swifterror writeback.
6055	if (swiftErrorTemp.isValid()) {
6056	llvm::Value *errorResult = Builder.CreateLoad(Addr: swiftErrorTemp);
6057	Builder.CreateStore(Val: errorResult, Addr: swiftErrorArg);
6058	}
6059
6060	// Emit any call-associated writebacks immediately. Arguably this
6061	// should happen after any return-value munging.
6062	if (CallArgs.hasWritebacks())
6063	EmitWritebacks(args: CallArgs);
6064
6065	// The stack cleanup for inalloca arguments has to run out of the normal
6066	// lexical order, so deactivate it and run it manually here.
6067	CallArgs.freeArgumentMemory(CGF&: *this);
6068
6069	// Extract the return value.
6070	RValue Ret;
6071
6072	// If the current function is a virtual function pointer thunk, avoid copying
6073	// the return value of the musttail call to a temporary.
6074	if (IsVirtualFunctionPointerThunk) {
6075	Ret = RValue::get(V: CI);
6076	} else {
6077	Ret = [&] {
6078	switch (RetAI.getKind()) {
6079	case ABIArgInfo::CoerceAndExpand: {
6080	auto coercionType = RetAI.getCoerceAndExpandType();
6081
6082	Address addr = SRetPtr.withElementType(ElemTy: coercionType);
6083
6084	assert(CI->getType() == RetAI.getUnpaddedCoerceAndExpandType());
6085	bool requiresExtract = isa<llvm::StructType>(Val: CI->getType());
6086
6087	unsigned unpaddedIndex = `0`;
6088	for (unsigned i = `0`, e = coercionType->getNumElements(); i != e; ++i) {
6089	llvm::Type *eltType = coercionType->getElementType(N: i);
6090	if (ABIArgInfo::isPaddingForCoerceAndExpand(eltType))
6091	continue;
6092	Address eltAddr = Builder.CreateStructGEP(Addr: addr, Index: i);
6093	llvm::Value *elt = CI;
6094	if (requiresExtract)
6095	elt = Builder.CreateExtractValue(Agg: elt, Idxs: unpaddedIndex++);
6096	else
6097	assert(unpaddedIndex == `0`);
6098	Builder.CreateStore(Val: elt, Addr: eltAddr);
6099	}
6100	[[fallthrough]];
6101	}
6102
6103	case ABIArgInfo::InAlloca:
6104	case ABIArgInfo::Indirect: {
6105	RValue ret = convertTempToRValue(addr: SRetPtr, type: RetTy, Loc: SourceLocation ());
6106	if (UnusedReturnSizePtr)
6107	PopCleanupBlock();
6108	return ret;
6109	}
6110
6111	case ABIArgInfo::Ignore:
6112	// If we are ignoring an argument that had a result, make sure to
6113	// construct the appropriate return value for our caller.
6114	return GetUndefRValue(Ty: RetTy);
6115
6116	case ABIArgInfo::Extend:
6117	case ABIArgInfo::Direct: {
6118	llvm::Type *RetIRTy = ConvertType(T: RetTy);
6119	if (RetAI.getCoerceToType() == RetIRTy &&
6120	RetAI.getDirectOffset() == `0`) {
6121	switch (getEvaluationKind(T: RetTy)) {
6122	case TEK_Complex: {
6123	llvm::Value *Real = Builder.CreateExtractValue(Agg: CI, Idxs: `0`);
6124	llvm::Value *Imag = Builder.CreateExtractValue(Agg: CI, Idxs: `1`);
6125	return RValue::getComplex(C: std::make_pair(x&: Real, y&: Imag));
6126	}
6127	case TEK_Aggregate:
6128	break;
6129	case TEK_Scalar: {
6130	// If the argument doesn't match, perform a bitcast to coerce it.
6131	// This can happen due to trivial type mismatches.
6132	llvm::Value *V = CI;
6133	if (V->getType() != RetIRTy)
6134	V = Builder.CreateBitCast(V, DestTy: RetIRTy);
6135	return RValue::get(V);
6136	}
6137	}
6138	}
6139
6140	// If coercing a fixed vector from a scalable vector for ABI
6141	// compatibility, and the types match, use the llvm.vector.extract
6142	// intrinsic to perform the conversion.
6143	if (auto *FixedDstTy = dyn_cast<llvm::FixedVectorType>(Val: RetIRTy)) {
6144	llvm::Value *V = CI;
6145	if (auto *ScalableSrcTy =
6146	dyn_cast<llvm::ScalableVectorType>(Val: V->getType())) {
6147	if (FixedDstTy->getElementType() ==
6148	ScalableSrcTy->getElementType()) {
6149	V = Builder.CreateExtractVector(DstType: FixedDstTy, SrcVec: V, Idx: uint64_t(`0`),
6150	Name: "cast.fixed");
6151	return RValue::get(V);
6152	}
6153	}
6154	}
6155
6156	Address DestPtr = ReturnValue.getValue();
6157	bool DestIsVolatile = ReturnValue.isVolatile();
6158	uint64_t DestSize =
6159	getContext().getTypeInfoDataSizeInChars(T: RetTy).Width.getQuantity();
6160
6161	if (!DestPtr.isValid()) {
6162	DestPtr = CreateMemTemp(T: RetTy, Name: "coerce");
6163	DestIsVolatile = false;
6164	DestSize = getContext().getTypeSizeInChars(T: RetTy).getQuantity();
6165	}
6166
6167	// An empty record can overlap other data (if declared with
6168	// no_unique_address); omit the store for such types - as there is no
6169	// actual data to store.
6170	if (!isEmptyRecord(Context&: getContext(), T: RetTy, AllowArrays: true)) {
6171	// If the value is offset in memory, apply the offset now.
6172	Address StorePtr = emitAddressAtOffset(CGF&: *this, addr: DestPtr, info: RetAI);
6173	CreateCoercedStore(
6174	Src: CI, Dst: StorePtr,
6175	DstSize: llvm::TypeSize::getFixed(ExactSize: DestSize - RetAI.getDirectOffset()),
6176	DstIsVolatile: DestIsVolatile);
6177	}
6178
6179	return convertTempToRValue(addr: DestPtr, type: RetTy, Loc: SourceLocation ());
6180	}
6181
6182	case ABIArgInfo::Expand:
6183	case ABIArgInfo::IndirectAliased:
6184	llvm_unreachable("Invalid ABI kind for return argument");
6185	}
6186
6187	llvm_unreachable("Unhandled ABIArgInfo::Kind");
6188	}();
6189	}
6190
6191	// Emit the assume_aligned check on the return value.
6192	if (Ret.isScalar() && TargetDecl) {
6193	AssumeAlignedAttrEmitter.EmitAsAnAssumption(Loc, RetTy, Ret);
6194	AllocAlignAttrEmitter.EmitAsAnAssumption(Loc, RetTy, Ret);
6195	}
6196
6197	// Explicitly call CallLifetimeEnd::Emit just to re-use the code even though
6198	// we can't use the full cleanup mechanism.
6199	for (CallLifetimeEnd &LifetimeEnd : CallLifetimeEndAfterCall)
6200	LifetimeEnd.Emit(CGF&: *this, /Flags=/flags: {});
6201
6202	if (!ReturnValue.isExternallyDestructed() &&
6203	RetTy.isDestructedType() == QualType::DK_nontrivial_c_struct)
6204	pushDestroy(dtorKind: QualType::DK_nontrivial_c_struct, addr: Ret.getAggregateAddress(),
6205	type: RetTy);
6206
6207	return Ret;
6208	}
6209
6210	CGCallee CGCallee::prepareConcreteCallee(CodeGenFunction &CGF) const {
6211	if (isVirtual()) {
6212	const CallExpr *CE = getVirtualCallExpr();
6213	return CGF.CGM.getCXXABI().getVirtualFunctionPointer(
6214	CGF, GD: getVirtualMethodDecl(), This: getThisAddress(), Ty: getVirtualFunctionType(),
6215	Loc: CE ? CE->getBeginLoc() : SourceLocation ());
6216	}
6217
6218	return *this;
6219	}
6220
6221	/ VarArg handling /
6222
6223	RValue CodeGenFunction::EmitVAArg(VAArgExpr *VE, Address &VAListAddr,
6224	AggValueSlot Slot) {
6225	VAListAddr = VE->isMicrosoftABI() ? EmitMSVAListRef(E: VE->getSubExpr())
6226	: EmitVAListRef(E: VE->getSubExpr());
6227	QualType Ty = VE->getType();
6228	if (Ty ->isVariablyModifiedType())
6229	EmitVariablyModifiedType(Ty);
6230	if (VE->isMicrosoftABI())
6231	return CGM.getABIInfo().EmitMSVAArg(CGF&: *this, VAListAddr, Ty, Slot);
6232	return CGM.getABIInfo().EmitVAArg(CGF&: *this, VAListAddr, Ty, Slot);
6233	}
6234

source code of clang/lib/CodeGen/CGCall.cpp