1	//===--- CGClass.cpp - Emit LLVM Code for C++ classes -----------*- C++ -*-===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This contains code dealing with C++ code generation of classes
10	//
11	//===----------------------------------------------------------------------===//
12
13	#include "ABIInfoImpl.h"
14	#include "CGBlocks.h"
15	#include "CGCXXABI.h"
16	#include "CGDebugInfo.h"
17	#include "CGRecordLayout.h"
18	#include "CodeGenFunction.h"
19	#include "TargetInfo.h"
20	#include "clang/AST/Attr.h"
21	#include "clang/AST/CXXInheritance.h"
22	#include "clang/AST/CharUnits.h"
23	#include "clang/AST/DeclTemplate.h"
24	#include "clang/AST/EvaluatedExprVisitor.h"
25	#include "clang/AST/RecordLayout.h"
26	#include "clang/AST/StmtCXX.h"
27	#include "clang/Basic/CodeGenOptions.h"
28	#include "clang/CodeGen/CGFunctionInfo.h"
29	#include "llvm/IR/Intrinsics.h"
30	#include "llvm/IR/Metadata.h"
31	#include "llvm/Support/SaveAndRestore.h"
32	#include "llvm/Transforms/Utils/SanitizerStats.h"
33	#include <optional>
34
35	using namespace clang;
36	using namespace CodeGen;
37
38	/// Return the best known alignment for an unknown pointer to a
39	/// particular class.
40	CharUnits CodeGenModule::getClassPointerAlignment(const CXXRecordDecl *RD) {
41	if (!RD->hasDefinition())
42	return CharUnits::One(); // Hopefully won't be used anywhere.
43
44	auto &layout = getContext().getASTRecordLayout(RD);
45
46	// If the class is final, then we know that the pointer points to an
47	// object of that type and can use the full alignment.
48	if (RD->isEffectivelyFinal())
49	return layout.getAlignment();
50
51	// Otherwise, we have to assume it could be a subclass.
52	return layout.getNonVirtualAlignment();
53	}
54
55	/// Return the smallest possible amount of storage that might be allocated
56	/// starting from the beginning of an object of a particular class.
57	///
58	/// This may be smaller than sizeof(RD) if RD has virtual base classes.
59	CharUnits CodeGenModule::getMinimumClassObjectSize(const CXXRecordDecl *RD) {
60	if (!RD->hasDefinition())
61	return CharUnits::One();
62
63	auto &layout = getContext().getASTRecordLayout(RD);
64
65	// If the class is final, then we know that the pointer points to an
66	// object of that type and can use the full alignment.
67	if (RD->isEffectivelyFinal())
68	return layout.getSize();
69
70	// Otherwise, we have to assume it could be a subclass.
71	return std::max(layout.getNonVirtualSize(), CharUnits::One());
72	}
73
74	/// Return the best known alignment for a pointer to a virtual base,
75	/// given the alignment of a pointer to the derived class.
76	CharUnits CodeGenModule::getVBaseAlignment(CharUnits actualDerivedAlign,
77	const CXXRecordDecl *derivedClass,
78	const CXXRecordDecl *vbaseClass) {
79	// The basic idea here is that an underaligned derived pointer might
80	// indicate an underaligned base pointer.
81
82	assert(vbaseClass->isCompleteDefinition());
83	auto &baseLayout = getContext().getASTRecordLayout(vbaseClass);
84	CharUnits expectedVBaseAlign = baseLayout.getNonVirtualAlignment();
85
86	return getDynamicOffsetAlignment(ActualAlign: actualDerivedAlign, Class: derivedClass,
87	ExpectedTargetAlign: expectedVBaseAlign);
88	}
89
90	CharUnits
91	CodeGenModule::getDynamicOffsetAlignment(CharUnits actualBaseAlign,
92	const CXXRecordDecl *baseDecl,
93	CharUnits expectedTargetAlign) {
94	// If the base is an incomplete type (which is, alas, possible with
95	// member pointers), be pessimistic.
96	if (!baseDecl->isCompleteDefinition())
97	return std::min(a: actualBaseAlign, b: expectedTargetAlign);
98
99	auto &baseLayout = getContext().getASTRecordLayout(baseDecl);
100	CharUnits expectedBaseAlign = baseLayout.getNonVirtualAlignment();
101
102	// If the class is properly aligned, assume the target offset is, too.
103	//
104	// This actually isn't necessarily the right thing to do --- if the
105	// class is a complete object, but it's only properly aligned for a
106	// base subobject, then the alignments of things relative to it are
107	// probably off as well. (Note that this requires the alignment of
108	// the target to be greater than the NV alignment of the derived
109	// class.)
110	//
111	// However, our approach to this kind of under-alignment can only
112	// ever be best effort; after all, we're never going to propagate
113	// alignments through variables or parameters. Note, in particular,
114	// that constructing a polymorphic type in an address that's less
115	// than pointer-aligned will generally trap in the constructor,
116	// unless we someday add some sort of attribute to change the
117	// assumed alignment of 'this'. So our goal here is pretty much
118	// just to allow the user to explicitly say that a pointer is
119	// under-aligned and then safely access its fields and vtables.
120	if (actualBaseAlign >= expectedBaseAlign) {
121	return expectedTargetAlign;
122	}
123
124	// Otherwise, we might be offset by an arbitrary multiple of the
125	// actual alignment. The correct adjustment is to take the min of
126	// the two alignments.
127	return std::min(a: actualBaseAlign, b: expectedTargetAlign);
128	}
129
130	Address CodeGenFunction::LoadCXXThisAddress() {
131	assert(CurFuncDecl && "loading 'this' without a func declaration?");
132	auto *MD = cast<CXXMethodDecl>(Val: CurFuncDecl);
133
134	// Lazily compute CXXThisAlignment.
135	if (CXXThisAlignment.isZero()) {
136	// Just use the best known alignment for the parent.
137	// TODO: if we're currently emitting a complete-object ctor/dtor,
138	// we can always use the complete-object alignment.
139	CXXThisAlignment = CGM.getClassPointerAlignment(RD: MD->getParent());
140	}
141
142	return makeNaturalAddressForPointer(
143	Ptr: LoadCXXThis(), T: MD->getFunctionObjectParameterType(), Alignment: CXXThisAlignment,
144	ForPointeeType: false, BaseInfo: nullptr, TBAAInfo: nullptr, IsKnownNonNull: KnownNonNull);
145	}
146
147	/// Emit the address of a field using a member data pointer.
148	///
149	/// \param E Only used for emergency diagnostics
150	Address CodeGenFunction::EmitCXXMemberDataPointerAddress(
151	const Expr *E, Address base, llvm::Value *memberPtr,
152	const MemberPointerType *memberPtrType, bool IsInBounds,
153	LValueBaseInfo *BaseInfo, TBAAAccessInfo *TBAAInfo) {
154	// Ask the ABI to compute the actual address.
155	llvm::Value *ptr = CGM.getCXXABI().EmitMemberDataPointerAddress(
156	CGF&: *this, E, Base: base, MemPtr: memberPtr, MPT: memberPtrType, IsInBounds);
157
158	QualType memberType = memberPtrType->getPointeeType();
159	CharUnits memberAlign =
160	CGM.getNaturalTypeAlignment(T: memberType, BaseInfo, TBAAInfo);
161	memberAlign = CGM.getDynamicOffsetAlignment(
162	actualBaseAlign: base.getAlignment(), baseDecl: memberPtrType->getMostRecentCXXRecordDecl(),
163	expectedTargetAlign: memberAlign);
164	return Address(ptr, ConvertTypeForMem(T: memberPtrType->getPointeeType()),
165	memberAlign);
166	}
167
168	CharUnits CodeGenModule::computeNonVirtualBaseClassOffset(
169	const CXXRecordDecl *DerivedClass, CastExpr::path_const_iterator Start,
170	CastExpr::path_const_iterator End) {
171	CharUnits Offset = CharUnits::Zero();
172
173	const ASTContext &Context = getContext();
174	const CXXRecordDecl *RD = DerivedClass;
175
176	for (CastExpr::path_const_iterator I = Start; I != End; ++I) {
177	const CXXBaseSpecifier *Base = *I;
178	assert(!Base->isVirtual() && "Should not see virtual bases here!");
179
180	// Get the layout.
181	const ASTRecordLayout &Layout = Context.getASTRecordLayout(RD);
182
183	const auto *BaseDecl =
184	cast<CXXRecordDecl>(Val: Base->getType()->castAs<RecordType>()->getDecl());
185
186	// Add the offset.
187	Offset += Layout.getBaseClassOffset(Base: BaseDecl);
188
189	RD = BaseDecl;
190	}
191
192	return Offset;
193	}
194
195	llvm::Constant *
196	CodeGenModule::GetNonVirtualBaseClassOffset(const CXXRecordDecl *ClassDecl,
197	CastExpr::path_const_iterator PathBegin,
198	CastExpr::path_const_iterator PathEnd) {
199	assert(PathBegin != PathEnd && "Base path should not be empty!");
200
201	CharUnits Offset =
202	computeNonVirtualBaseClassOffset(DerivedClass: ClassDecl, Start: PathBegin, End: PathEnd);
203	if (Offset.isZero())
204	return nullptr;
205
206	llvm::Type *PtrDiffTy =
207	getTypes().ConvertType(T: getContext().getPointerDiffType());
208
209	return llvm::ConstantInt::get(Ty: PtrDiffTy, V: Offset.getQuantity());
210	}
211
212	/// Gets the address of a direct base class within a complete object.
213	/// This should only be used for (1) non-virtual bases or (2) virtual bases
214	/// when the type is known to be complete (e.g. in complete destructors).
215	///
216	/// The object pointed to by 'This' is assumed to be non-null.
217	Address
218	CodeGenFunction::GetAddressOfDirectBaseInCompleteClass(Address This,
219	const CXXRecordDecl *Derived,
220	const CXXRecordDecl *Base,
221	bool BaseIsVirtual) {
222	// 'this' must be a pointer (in some address space) to Derived.
223	assert(This.getElementType() == ConvertType(Derived));
224
225	// Compute the offset of the virtual base.
226	CharUnits Offset;
227	const ASTRecordLayout &Layout = getContext().getASTRecordLayout(Derived);
228	if (BaseIsVirtual)
229	Offset = Layout.getVBaseClassOffset(VBase: Base);
230	else
231	Offset = Layout.getBaseClassOffset(Base);
232
233	// Shift and cast down to the base type.
234	// TODO: for complete types, this should be possible with a GEP.
235	Address V = This;
236	if (!Offset.isZero()) {
237	V = V.withElementType(ElemTy: Int8Ty);
238	V = Builder.CreateConstInBoundsByteGEP(Addr: V, Offset);
239	}
240	return V.withElementType(ElemTy: ConvertType(Base));
241	}
242
243	static Address
244	ApplyNonVirtualAndVirtualOffset(CodeGenFunction &CGF, Address addr,
245	CharUnits nonVirtualOffset,
246	llvm::Value *virtualOffset,
247	const CXXRecordDecl *derivedClass,
248	const CXXRecordDecl *nearestVBase) {
249	// Assert that we have something to do.
250	assert(!nonVirtualOffset.isZero() || virtualOffset != nullptr);
251
252	// Compute the offset from the static and dynamic components.
253	llvm::Value *baseOffset;
254	if (!nonVirtualOffset.isZero()) {
255	llvm::Type *OffsetType =
256	(CGF.CGM.getTarget().getCXXABI().isItaniumFamily() &&
257	CGF.CGM.getItaniumVTableContext().isRelativeLayout())
258	? CGF.Int32Ty
259	: CGF.PtrDiffTy;
260	baseOffset =
261	llvm::ConstantInt::get(Ty: OffsetType, V: nonVirtualOffset.getQuantity());
262	if (virtualOffset) {
263	baseOffset = CGF.Builder.CreateAdd(LHS: virtualOffset, RHS: baseOffset);
264	}
265	} else {
266	baseOffset = virtualOffset;
267	}
268
269	// Apply the base offset.
270	llvm::Value *ptr = addr.emitRawPointer(CGF);
271	ptr = CGF.Builder.CreateInBoundsGEP(Ty: CGF.Int8Ty, Ptr: ptr, IdxList: baseOffset, Name: "add.ptr");
272
273	// If we have a virtual component, the alignment of the result will
274	// be relative only to the known alignment of that vbase.
275	CharUnits alignment;
276	if (virtualOffset) {
277	assert(nearestVBase && "virtual offset without vbase?");
278	alignment = CGF.CGM.getVBaseAlignment(actualDerivedAlign: addr.getAlignment(),
279	derivedClass, vbaseClass: nearestVBase);
280	} else {
281	alignment = addr.getAlignment();
282	}
283	alignment = alignment.alignmentAtOffset(offset: nonVirtualOffset);
284
285	return Address(ptr, CGF.Int8Ty, alignment);
286	}
287
288	Address CodeGenFunction::GetAddressOfBaseClass(
289	Address Value, const CXXRecordDecl *Derived,
290	CastExpr::path_const_iterator PathBegin,
291	CastExpr::path_const_iterator PathEnd, bool NullCheckValue,
292	SourceLocation Loc) {
293	assert(PathBegin != PathEnd && "Base path should not be empty!");
294
295	CastExpr::path_const_iterator Start = PathBegin;
296	const CXXRecordDecl *VBase = nullptr;
297
298	// Sema has done some convenient canonicalization here: if the
299	// access path involved any virtual steps, the conversion path will
300	// *start* with a step down to the correct virtual base subobject,
301	// and hence will not require any further steps.
302	if ((*Start)->isVirtual()) {
303	VBase = cast<CXXRecordDecl>(
304	Val: (*Start)->getType()->castAs<RecordType>()->getDecl());
305	++Start;
306	}
307
308	// Compute the static offset of the ultimate destination within its
309	// allocating subobject (the virtual base, if there is one, or else
310	// the "complete" object that we see).
311	CharUnits NonVirtualOffset = CGM.computeNonVirtualBaseClassOffset(
312	DerivedClass: VBase ? VBase : Derived, Start, End: PathEnd);
313
314	// If there's a virtual step, we can sometimes "devirtualize" it.
315	// For now, that's limited to when the derived type is final.
316	// TODO: "devirtualize" this for accesses to known-complete objects.
317	if (VBase && Derived->hasAttr<FinalAttr>()) {
318	const ASTRecordLayout &layout = getContext().getASTRecordLayout(Derived);
319	CharUnits vBaseOffset = layout.getVBaseClassOffset(VBase);
320	NonVirtualOffset += vBaseOffset;
321	VBase = nullptr; // we no longer have a virtual step
322	}
323
324	// Get the base pointer type.
325	llvm::Type *BaseValueTy = ConvertType(T: (PathEnd[-1])->getType());
326	llvm::Type *PtrTy = llvm::PointerType::get(
327	C&: CGM.getLLVMContext(), AddressSpace: Value.getType()->getPointerAddressSpace());
328
329	QualType DerivedTy = getContext().getRecordType(Derived);
330	CharUnits DerivedAlign = CGM.getClassPointerAlignment(RD: Derived);
331
332	// If the static offset is zero and we don't have a virtual step,
333	// just do a bitcast; null checks are unnecessary.
334	if (NonVirtualOffset.isZero() && !VBase) {
335	if (sanitizePerformTypeCheck()) {
336	SanitizerSet SkippedChecks;
337	SkippedChecks.set(K: SanitizerKind::Null, Value: !NullCheckValue);
338	EmitTypeCheck(TCK: TCK_Upcast, Loc, V: Value.emitRawPointer(CGF&: *this), Type: DerivedTy,
339	Alignment: DerivedAlign, SkippedChecks);
340	}
341	return Value.withElementType(ElemTy: BaseValueTy);
342	}
343
344	llvm::BasicBlock *origBB = nullptr;
345	llvm::BasicBlock *endBB = nullptr;
346
347	// Skip over the offset (and the vtable load) if we're supposed to
348	// null-check the pointer.
349	if (NullCheckValue) {
350	origBB = Builder.GetInsertBlock();
351	llvm::BasicBlock *notNullBB = createBasicBlock(name: "cast.notnull");
352	endBB = createBasicBlock(name: "cast.end");
353
354	llvm::Value *isNull = Builder.CreateIsNull(Addr: Value);
355	Builder.CreateCondBr(Cond: isNull, True: endBB, False: notNullBB);
356	EmitBlock(BB: notNullBB);
357	}
358
359	if (sanitizePerformTypeCheck()) {
360	SanitizerSet SkippedChecks;
361	SkippedChecks.set(K: SanitizerKind::Null, Value: true);
362	EmitTypeCheck(TCK: VBase ? TCK_UpcastToVirtualBase : TCK_Upcast, Loc,
363	V: Value.emitRawPointer(CGF&: *this), Type: DerivedTy, Alignment: DerivedAlign,
364	SkippedChecks);
365	}
366
367	// Compute the virtual offset.
368	llvm::Value *VirtualOffset = nullptr;
369	if (VBase) {
370	VirtualOffset =
371	CGM.getCXXABI().GetVirtualBaseClassOffset(CGF&: *this, This: Value, ClassDecl: Derived, BaseClassDecl: VBase);
372	}
373
374	// Apply both offsets.
375	Value = ApplyNonVirtualAndVirtualOffset(CGF&: *this, addr: Value, nonVirtualOffset: NonVirtualOffset,
376	virtualOffset: VirtualOffset, derivedClass: Derived, nearestVBase: VBase);
377
378	// Cast to the destination type.
379	Value = Value.withElementType(ElemTy: BaseValueTy);
380
381	// Build a phi if we needed a null check.
382	if (NullCheckValue) {
383	llvm::BasicBlock *notNullBB = Builder.GetInsertBlock();
384	Builder.CreateBr(Dest: endBB);
385	EmitBlock(BB: endBB);
386
387	llvm::PHINode *PHI = Builder.CreatePHI(Ty: PtrTy, NumReservedValues: 2, Name: "cast.result");
388	PHI->addIncoming(V: Value.emitRawPointer(CGF&: *this), BB: notNullBB);
389	PHI->addIncoming(V: llvm::Constant::getNullValue(Ty: PtrTy), BB: origBB);
390	Value = Value.withPointer(NewPointer: PHI, IsKnownNonNull: NotKnownNonNull);
391	}
392
393	return Value;
394	}
395
396	Address
397	CodeGenFunction::GetAddressOfDerivedClass(Address BaseAddr,
398	const CXXRecordDecl *Derived,
399	CastExpr::path_const_iterator PathBegin,
400	CastExpr::path_const_iterator PathEnd,
401	bool NullCheckValue) {
402	assert(PathBegin != PathEnd && "Base path should not be empty!");
403
404	QualType DerivedTy =
405	getContext().getCanonicalType(T: getContext().getTagDeclType(Derived));
406	llvm::Type *DerivedValueTy = ConvertType(T: DerivedTy);
407
408	llvm::Value *NonVirtualOffset =
409	CGM.GetNonVirtualBaseClassOffset(ClassDecl: Derived, PathBegin, PathEnd);
410
411	if (!NonVirtualOffset) {
412	// No offset, we can just cast back.
413	return BaseAddr.withElementType(ElemTy: DerivedValueTy);
414	}
415
416	llvm::BasicBlock *CastNull = nullptr;
417	llvm::BasicBlock *CastNotNull = nullptr;
418	llvm::BasicBlock *CastEnd = nullptr;
419
420	if (NullCheckValue) {
421	CastNull = createBasicBlock(name: "cast.null");
422	CastNotNull = createBasicBlock(name: "cast.notnull");
423	CastEnd = createBasicBlock(name: "cast.end");
424
425	llvm::Value *IsNull = Builder.CreateIsNull(Addr: BaseAddr);
426	Builder.CreateCondBr(Cond: IsNull, True: CastNull, False: CastNotNull);
427	EmitBlock(BB: CastNotNull);
428	}
429
430	// Apply the offset.
431	Address Addr = BaseAddr.withElementType(ElemTy: Int8Ty);
432	Addr = Builder.CreateInBoundsGEP(
433	Addr, IdxList: Builder.CreateNeg(V: NonVirtualOffset), ElementType: Int8Ty,
434	Align: CGM.getClassPointerAlignment(RD: Derived), Name: "sub.ptr");
435
436	// Just cast.
437	Addr = Addr.withElementType(ElemTy: DerivedValueTy);
438
439	// Produce a PHI if we had a null-check.
440	if (NullCheckValue) {
441	Builder.CreateBr(Dest: CastEnd);
442	EmitBlock(BB: CastNull);
443	Builder.CreateBr(Dest: CastEnd);
444	EmitBlock(BB: CastEnd);
445
446	llvm::Value *Value = Addr.emitRawPointer(CGF&: *this);
447	llvm::PHINode *PHI = Builder.CreatePHI(Ty: Value->getType(), NumReservedValues: 2);
448	PHI->addIncoming(V: Value, BB: CastNotNull);
449	PHI->addIncoming(V: llvm::Constant::getNullValue(Ty: Value->getType()), BB: CastNull);
450	return Address(PHI, Addr.getElementType(),
451	CGM.getClassPointerAlignment(RD: Derived));
452	}
453
454	return Addr;
455	}
456
457	llvm::Value *CodeGenFunction::GetVTTParameter(GlobalDecl GD,
458	bool ForVirtualBase,
459	bool Delegating) {
460	if (!CGM.getCXXABI().NeedsVTTParameter(GD)) {
461	// This constructor/destructor does not need a VTT parameter.
462	return nullptr;
463	}
464
465	const CXXRecordDecl *RD = cast<CXXMethodDecl>(Val: CurCodeDecl)->getParent();
466	const CXXRecordDecl *Base = cast<CXXMethodDecl>(Val: GD.getDecl())->getParent();
467
468	uint64_t SubVTTIndex;
469
470	if (Delegating) {
471	// If this is a delegating constructor call, just load the VTT.
472	return LoadCXXVTT();
473	} else if (RD == Base) {
474	// If the record matches the base, this is the complete ctor/dtor
475	// variant calling the base variant in a class with virtual bases.
476	assert(!CGM.getCXXABI().NeedsVTTParameter(CurGD) &&
477	"doing no-op VTT offset in base dtor/ctor?");
478	assert(!ForVirtualBase && "Can't have same class as virtual base!");
479	SubVTTIndex = 0;
480	} else {
481	const ASTRecordLayout &Layout = getContext().getASTRecordLayout(RD);
482	CharUnits BaseOffset = ForVirtualBase ?
483	Layout.getVBaseClassOffset(VBase: Base) :
484	Layout.getBaseClassOffset(Base);
485
486	SubVTTIndex =
487	CGM.getVTables().getSubVTTIndex(RD, Base: BaseSubobject(Base, BaseOffset));
488	assert(SubVTTIndex != 0 && "Sub-VTT index must be greater than zero!");
489	}
490
491	if (CGM.getCXXABI().NeedsVTTParameter(GD: CurGD)) {
492	// A VTT parameter was passed to the constructor, use it.
493	llvm::Value *VTT = LoadCXXVTT();
494	return Builder.CreateConstInBoundsGEP1_64(Ty: VoidPtrTy, Ptr: VTT, Idx0: SubVTTIndex);
495	} else {
496	// We're the complete constructor, so get the VTT by name.
497	llvm::GlobalValue *VTT = CGM.getVTables().GetAddrOfVTT(RD);
498	return Builder.CreateConstInBoundsGEP2_64(
499	Ty: VTT->getValueType(), Ptr: VTT, Idx0: 0, Idx1: SubVTTIndex);
500	}
501	}
502
503	namespace {
504	/// Call the destructor for a direct base class.
505	struct CallBaseDtor final : EHScopeStack::Cleanup {
506	const CXXRecordDecl *BaseClass;
507	bool BaseIsVirtual;
508	CallBaseDtor(const CXXRecordDecl *Base, bool BaseIsVirtual)
509	: BaseClass(Base), BaseIsVirtual(BaseIsVirtual) {}
510
511	void Emit(CodeGenFunction &CGF, Flags flags) override {
512	const CXXRecordDecl *DerivedClass =
513	cast<CXXMethodDecl>(Val: CGF.CurCodeDecl)->getParent();
514
515	const CXXDestructorDecl *D = BaseClass->getDestructor();
516	// We are already inside a destructor, so presumably the object being
517	// destroyed should have the expected type.
518	QualType ThisTy = D->getFunctionObjectParameterType();
519	Address Addr =
520	CGF.GetAddressOfDirectBaseInCompleteClass(This: CGF.LoadCXXThisAddress(),
521	Derived: DerivedClass, Base: BaseClass,
522	BaseIsVirtual);
523	CGF.EmitCXXDestructorCall(D, Type: Dtor_Base, ForVirtualBase: BaseIsVirtual,
524	/*Delegating=*/false, This: Addr, ThisTy);
525	}
526	};
527
528	/// A visitor which checks whether an initializer uses 'this' in a
529	/// way which requires the vtable to be properly set.
530	struct DynamicThisUseChecker : ConstEvaluatedExprVisitor<DynamicThisUseChecker> {
531	typedef ConstEvaluatedExprVisitor<DynamicThisUseChecker> super;
532
533	bool UsesThis;
534
535	DynamicThisUseChecker(const ASTContext &C) : super(C), UsesThis(false) {}
536
537	// Black-list all explicit and implicit references to 'this'.
538	//
539	// Do we need to worry about external references to 'this' derived
540	// from arbitrary code? If so, then anything which runs arbitrary
541	// external code might potentially access the vtable.
542	void VisitCXXThisExpr(const CXXThisExpr *E) { UsesThis = true; }
543	};
544	} // end anonymous namespace
545
546	static bool BaseInitializerUsesThis(ASTContext &C, const Expr *Init) {
547	DynamicThisUseChecker Checker(C);
548	Checker.Visit(Init);
549	return Checker.UsesThis;
550	}
551
552	static void EmitBaseInitializer(CodeGenFunction &CGF,
553	const CXXRecordDecl *ClassDecl,
554	CXXCtorInitializer *BaseInit) {
555	assert(BaseInit->isBaseInitializer() &&
556	"Must have base initializer!");
557
558	Address ThisPtr = CGF.LoadCXXThisAddress();
559
560	const Type *BaseType = BaseInit->getBaseClass();
561	const auto *BaseClassDecl =
562	cast<CXXRecordDecl>(Val: BaseType->castAs<RecordType>()->getDecl());
563
564	bool isBaseVirtual = BaseInit->isBaseVirtual();
565
566	// If the initializer for the base (other than the constructor
567	// itself) accesses 'this' in any way, we need to initialize the
568	// vtables.
569	if (BaseInitializerUsesThis(C&: CGF.getContext(), Init: BaseInit->getInit()))
570	CGF.InitializeVTablePointers(ClassDecl);
571
572	// We can pretend to be a complete class because it only matters for
573	// virtual bases, and we only do virtual bases for complete ctors.
574	Address V =
575	CGF.GetAddressOfDirectBaseInCompleteClass(This: ThisPtr, Derived: ClassDecl,
576	Base: BaseClassDecl,
577	BaseIsVirtual: isBaseVirtual);
578	AggValueSlot AggSlot =
579	AggValueSlot::forAddr(
580	addr: V, quals: Qualifiers(),
581	isDestructed: AggValueSlot::IsDestructed,
582	needsGC: AggValueSlot::DoesNotNeedGCBarriers,
583	isAliased: AggValueSlot::IsNotAliased,
584	mayOverlap: CGF.getOverlapForBaseInit(RD: ClassDecl, BaseRD: BaseClassDecl, IsVirtual: isBaseVirtual));
585
586	CGF.EmitAggExpr(E: BaseInit->getInit(), AS: AggSlot);
587
588	if (CGF.CGM.getLangOpts().Exceptions &&
589	!BaseClassDecl->hasTrivialDestructor())
590	CGF.EHStack.pushCleanup<CallBaseDtor>(Kind: EHCleanup, A: BaseClassDecl,
591	A: isBaseVirtual);
592	}
593
594	static bool isMemcpyEquivalentSpecialMember(const CXXMethodDecl *D) {
595	auto *CD = dyn_cast<CXXConstructorDecl>(Val: D);
596	if (!(CD && CD->isCopyOrMoveConstructor()) &&
597	!D->isCopyAssignmentOperator() && !D->isMoveAssignmentOperator())
598	return false;
599
600	// We can emit a memcpy for a trivial copy or move constructor/assignment.
601	if (D->isTrivial() && !D->getParent()->mayInsertExtraPadding())
602	return true;
603
604	// We *must* emit a memcpy for a defaulted union copy or move op.
605	if (D->getParent()->isUnion() && D->isDefaulted())
606	return true;
607
608	return false;
609	}
610
611	static void EmitLValueForAnyFieldInitialization(CodeGenFunction &CGF,
612	CXXCtorInitializer *MemberInit,
613	LValue &LHS) {
614	FieldDecl *Field = MemberInit->getAnyMember();
615	if (MemberInit->isIndirectMemberInitializer()) {
616	// If we are initializing an anonymous union field, drill down to the field.
617	IndirectFieldDecl *IndirectField = MemberInit->getIndirectMember();
618	for (const auto *I : IndirectField->chain())
619	LHS = CGF.EmitLValueForFieldInitialization(Base: LHS, Field: cast<FieldDecl>(Val: I));
620	} else {
621	LHS = CGF.EmitLValueForFieldInitialization(Base: LHS, Field);
622	}
623	}
624
625	static void EmitMemberInitializer(CodeGenFunction &CGF,
626	const CXXRecordDecl *ClassDecl,
627	CXXCtorInitializer *MemberInit,
628	const CXXConstructorDecl *Constructor,
629	FunctionArgList &Args) {
630	ApplyDebugLocation Loc(CGF, MemberInit->getSourceLocation());
631	assert(MemberInit->isAnyMemberInitializer() &&
632	"Must have member initializer!");
633	assert(MemberInit->getInit() && "Must have initializer!");
634
635	// non-static data member initializers.
636	FieldDecl *Field = MemberInit->getAnyMember();
637	QualType FieldType = Field->getType();
638
639	llvm::Value *ThisPtr = CGF.LoadCXXThis();
640	QualType RecordTy = CGF.getContext().getTypeDeclType(ClassDecl);
641	LValue LHS;
642
643	// If a base constructor is being emitted, create an LValue that has the
644	// non-virtual alignment.
645	if (CGF.CurGD.getCtorType() == Ctor_Base)
646	LHS = CGF.MakeNaturalAlignPointeeAddrLValue(V: ThisPtr, T: RecordTy);
647	else
648	LHS = CGF.MakeNaturalAlignAddrLValue(V: ThisPtr, T: RecordTy);
649
650	EmitLValueForAnyFieldInitialization(CGF, MemberInit, LHS);
651
652	// Special case: if we are in a copy or move constructor, and we are copying
653	// an array of PODs or classes with trivial copy constructors, ignore the
654	// AST and perform the copy we know is equivalent.
655	// FIXME: This is hacky at best... if we had a bit more explicit information
656	// in the AST, we could generalize it more easily.
657	const ConstantArrayType *Array
658	= CGF.getContext().getAsConstantArrayType(T: FieldType);
659	if (Array && Constructor->isDefaulted() &&
660	Constructor->isCopyOrMoveConstructor()) {
661	QualType BaseElementTy = CGF.getContext().getBaseElementType(Array);
662	CXXConstructExpr *CE = dyn_cast<CXXConstructExpr>(Val: MemberInit->getInit());
663	if (BaseElementTy.isPODType(Context: CGF.getContext()) ||
664	(CE && isMemcpyEquivalentSpecialMember(CE->getConstructor()))) {
665	unsigned SrcArgIndex =
666	CGF.CGM.getCXXABI().getSrcArgforCopyCtor(Constructor, Args);
667	llvm::Value *SrcPtr
668	= CGF.Builder.CreateLoad(Addr: CGF.GetAddrOfLocalVar(VD: Args[SrcArgIndex]));
669	LValue ThisRHSLV = CGF.MakeNaturalAlignAddrLValue(V: SrcPtr, T: RecordTy);
670	LValue Src = CGF.EmitLValueForFieldInitialization(Base: ThisRHSLV, Field);
671
672	// Copy the aggregate.
673	CGF.EmitAggregateCopy(Dest: LHS, Src, EltTy: FieldType, MayOverlap: CGF.getOverlapForFieldInit(FD: Field),
674	isVolatile: LHS.isVolatileQualified());
675	// Ensure that we destroy the objects if an exception is thrown later in
676	// the constructor.
677	QualType::DestructionKind dtorKind = FieldType.isDestructedType();
678	if (CGF.needsEHCleanup(kind: dtorKind))
679	CGF.pushEHDestroy(dtorKind, addr: LHS.getAddress(), type: FieldType);
680	return;
681	}
682	}
683
684	CGF.EmitInitializerForField(Field, LHS, Init: MemberInit->getInit());
685	}
686
687	void CodeGenFunction::EmitInitializerForField(FieldDecl *Field, LValue LHS,
688	Expr *Init) {
689	QualType FieldType = Field->getType();
690	switch (getEvaluationKind(T: FieldType)) {
691	case TEK_Scalar:
692	if (LHS.isSimple()) {
693	EmitExprAsInit(Init, Field, LHS, false);
694	} else {
695	RValue RHS = RValue::get(V: EmitScalarExpr(E: Init));
696	EmitStoreThroughLValue(Src: RHS, Dst: LHS);
697	}
698	break;
699	case TEK_Complex:
700	EmitComplexExprIntoLValue(E: Init, dest: LHS, /*isInit*/ true);
701	break;
702	case TEK_Aggregate: {
703	AggValueSlot Slot = AggValueSlot::forLValue(
704	LV: LHS, isDestructed: AggValueSlot::IsDestructed, needsGC: AggValueSlot::DoesNotNeedGCBarriers,
705	isAliased: AggValueSlot::IsNotAliased, mayOverlap: getOverlapForFieldInit(FD: Field),
706	isZeroed: AggValueSlot::IsNotZeroed,
707	// Checks are made by the code that calls constructor.
708	isChecked: AggValueSlot::IsSanitizerChecked);
709	EmitAggExpr(E: Init, AS: Slot);
710	break;
711	}
712	}
713
714	// Ensure that we destroy this object if an exception is thrown
715	// later in the constructor.
716	QualType::DestructionKind dtorKind = FieldType.isDestructedType();
717	if (needsEHCleanup(kind: dtorKind))
718	pushEHDestroy(dtorKind, addr: LHS.getAddress(), type: FieldType);
719	}
720
721	/// Checks whether the given constructor is a valid subject for the
722	/// complete-to-base constructor delegation optimization, i.e.
723	/// emitting the complete constructor as a simple call to the base
724	/// constructor.
725	bool CodeGenFunction::IsConstructorDelegationValid(
726	const CXXConstructorDecl *Ctor) {
727
728	// Currently we disable the optimization for classes with virtual
729	// bases because (1) the addresses of parameter variables need to be
730	// consistent across all initializers but (2) the delegate function
731	// call necessarily creates a second copy of the parameter variable.
732	//
733	// The limiting example (purely theoretical AFAIK):
734	// struct A { A(int &c) { c++; } };
735	// struct B : virtual A {
736	// B(int count) : A(count) { printf("%d\n", count); }
737	// };
738	// ...although even this example could in principle be emitted as a
739	// delegation since the address of the parameter doesn't escape.
740	if (Ctor->getParent()->getNumVBases()) {
741	// TODO: white-list trivial vbase initializers. This case wouldn't
742	// be subject to the restrictions below.
743
744	// TODO: white-list cases where:
745	// - there are no non-reference parameters to the constructor
746	// - the initializers don't access any non-reference parameters
747	// - the initializers don't take the address of non-reference
748	// parameters
749	// - etc.
750	// If we ever add any of the above cases, remember that:
751	// - function-try-blocks will always exclude this optimization
752	// - we need to perform the constructor prologue and cleanup in
753	// EmitConstructorBody.
754
755	return false;
756	}
757
758	// We also disable the optimization for variadic functions because
759	// it's impossible to "re-pass" varargs.
760	if (Ctor->getType()->castAs<FunctionProtoType>()->isVariadic())
761	return false;
762
763	// FIXME: Decide if we can do a delegation of a delegating constructor.
764	if (Ctor->isDelegatingConstructor())
765	return false;
766
767	return true;
768	}
769
770	// Emit code in ctor (Prologue==true) or dtor (Prologue==false)
771	// to poison the extra field paddings inserted under
772	// -fsanitize-address-field-padding=1|2.
773	void CodeGenFunction::EmitAsanPrologueOrEpilogue(bool Prologue) {
774	ASTContext &Context = getContext();
775	const CXXRecordDecl *ClassDecl =
776	Prologue ? cast<CXXConstructorDecl>(Val: CurGD.getDecl())->getParent()
777	: cast<CXXDestructorDecl>(Val: CurGD.getDecl())->getParent();
778	if (!ClassDecl->mayInsertExtraPadding()) return;
779
780	struct SizeAndOffset {
781	uint64_t Size;
782	uint64_t Offset;
783	};
784
785	unsigned PtrSize = CGM.getDataLayout().getPointerSizeInBits();
786	const ASTRecordLayout &Info = Context.getASTRecordLayout(ClassDecl);
787
788	// Populate sizes and offsets of fields.
789	SmallVector<SizeAndOffset, 16> SSV(Info.getFieldCount());
790	for (unsigned i = 0, e = Info.getFieldCount(); i != e; ++i)
791	SSV[i].Offset =
792	Context.toCharUnitsFromBits(BitSize: Info.getFieldOffset(FieldNo: i)).getQuantity();
793
794	size_t NumFields = 0;
795	for (const auto *Field : ClassDecl->fields()) {
796	const FieldDecl *D = Field;
797	auto FieldInfo = Context.getTypeInfoInChars(D->getType());
798	CharUnits FieldSize = FieldInfo.Width;
799	assert(NumFields < SSV.size());
800	SSV[NumFields].Size = D->isBitField() ? 0 : FieldSize.getQuantity();
801	NumFields++;
802	}
803	assert(NumFields == SSV.size());
804	if (SSV.size() <= 1) return;
805
806	// We will insert calls to __asan_* run-time functions.
807	// LLVM AddressSanitizer pass may decide to inline them later.
808	llvm::Type *Args[2] = {IntPtrTy, IntPtrTy};
809	llvm::FunctionType *FTy =
810	llvm::FunctionType::get(Result: CGM.VoidTy, Params: Args, isVarArg: false);
811	llvm::FunctionCallee F = CGM.CreateRuntimeFunction(
812	Ty: FTy, Name: Prologue ? "__asan_poison_intra_object_redzone"
813	: "__asan_unpoison_intra_object_redzone");
814
815	llvm::Value *ThisPtr = LoadCXXThis();
816	ThisPtr = Builder.CreatePtrToInt(V: ThisPtr, DestTy: IntPtrTy);
817	uint64_t TypeSize = Info.getNonVirtualSize().getQuantity();
818	// For each field check if it has sufficient padding,
819	// if so (un)poison it with a call.
820	for (size_t i = 0; i < SSV.size(); i++) {
821	uint64_t AsanAlignment = 8;
822	uint64_t NextField = i == SSV.size() - 1 ? TypeSize : SSV[i + 1].Offset;
823	uint64_t PoisonSize = NextField - SSV[i].Offset - SSV[i].Size;
824	uint64_t EndOffset = SSV[i].Offset + SSV[i].Size;
825	if (PoisonSize < AsanAlignment || !SSV[i].Size ||
826	(NextField % AsanAlignment) != 0)
827	continue;
828	Builder.CreateCall(
829	Callee: F, Args: {Builder.CreateAdd(LHS: ThisPtr, RHS: Builder.getIntN(N: PtrSize, C: EndOffset)),
830	Builder.getIntN(N: PtrSize, C: PoisonSize)});
831	}
832	}
833
834	/// EmitConstructorBody - Emits the body of the current constructor.
835	void CodeGenFunction::EmitConstructorBody(FunctionArgList &Args) {
836	EmitAsanPrologueOrEpilogue(Prologue: true);
837	const CXXConstructorDecl *Ctor = cast<CXXConstructorDecl>(Val: CurGD.getDecl());
838	CXXCtorType CtorType = CurGD.getCtorType();
839
840	assert((CGM.getTarget().getCXXABI().hasConstructorVariants() ||
841	CtorType == Ctor_Complete) &&
842	"can only generate complete ctor for this ABI");
843
844	// Before we go any further, try the complete->base constructor
845	// delegation optimization.
846	if (CtorType == Ctor_Complete && IsConstructorDelegationValid(Ctor) &&
847	CGM.getTarget().getCXXABI().hasConstructorVariants()) {
848	EmitDelegateCXXConstructorCall(Ctor, CtorType: Ctor_Base, Args, Loc: Ctor->getEndLoc());
849	return;
850	}
851
852	const FunctionDecl *Definition = nullptr;
853	Stmt *Body = Ctor->getBody(Definition);
854	assert(Definition == Ctor && "emitting wrong constructor body");
855
856	// Enter the function-try-block before the constructor prologue if
857	// applicable.
858	bool IsTryBody = isa_and_nonnull<CXXTryStmt>(Val: Body);
859	if (IsTryBody)
860	EnterCXXTryStmt(S: *cast<CXXTryStmt>(Val: Body), IsFnTryBlock: true);
861
862	incrementProfileCounter(S: Body);
863	maybeCreateMCDCCondBitmap();
864
865	RunCleanupsScope RunCleanups(*this);
866
867	// TODO: in restricted cases, we can emit the vbase initializers of
868	// a complete ctor and then delegate to the base ctor.
869
870	// Emit the constructor prologue, i.e. the base and member
871	// initializers.
872	EmitCtorPrologue(CD: Ctor, Type: CtorType, Args);
873
874	// Emit the body of the statement.
875	if (IsTryBody)
876	EmitStmt(cast<CXXTryStmt>(Val: Body)->getTryBlock());
877	else if (Body)
878	EmitStmt(S: Body);
879
880	// Emit any cleanup blocks associated with the member or base
881	// initializers, which includes (along the exceptional path) the
882	// destructors for those members and bases that were fully
883	// constructed.
884	RunCleanups.ForceCleanup();
885
886	if (IsTryBody)
887	ExitCXXTryStmt(S: *cast<CXXTryStmt>(Val: Body), IsFnTryBlock: true);
888	}
889
890	namespace {
891	/// RAII object to indicate that codegen is copying the value representation
892	/// instead of the object representation. Useful when copying a struct or
893	/// class which has uninitialized members and we're only performing
894	/// lvalue-to-rvalue conversion on the object but not its members.
895	class CopyingValueRepresentation {
896	public:
897	explicit CopyingValueRepresentation(CodeGenFunction &CGF)
898	: CGF(CGF), OldSanOpts(CGF.SanOpts) {
899	CGF.SanOpts.set(K: SanitizerKind::Bool, Value: false);
900	CGF.SanOpts.set(K: SanitizerKind::Enum, Value: false);
901	}
902	~CopyingValueRepresentation() {
903	CGF.SanOpts = OldSanOpts;
904	}
905	private:
906	CodeGenFunction &CGF;
907	SanitizerSet OldSanOpts;
908	};
909	} // end anonymous namespace
910
911	namespace {
912	class FieldMemcpyizer {
913	public:
914	FieldMemcpyizer(CodeGenFunction &CGF, const CXXRecordDecl *ClassDecl,
915	const VarDecl *SrcRec)
916	: CGF(CGF), ClassDecl(ClassDecl), SrcRec(SrcRec),
917	RecLayout(CGF.getContext().getASTRecordLayout(ClassDecl)),
918	FirstField(nullptr), LastField(nullptr), FirstFieldOffset(0),
919	LastFieldOffset(0), LastAddedFieldIndex(0) {}
920
921	bool isMemcpyableField(FieldDecl *F) const {
922	// Never memcpy fields when we are adding poisoned paddings.
923	if (CGF.getContext().getLangOpts().SanitizeAddressFieldPadding)
924	return false;
925	Qualifiers Qual = F->getType().getQualifiers();
926	if (Qual.hasVolatile() || Qual.hasObjCLifetime())
927	return false;
928	if (PointerAuthQualifier Q = F->getType().getPointerAuth();
929	Q && Q.isAddressDiscriminated())
930	return false;
931	return true;
932	}
933
934	void addMemcpyableField(FieldDecl *F) {
935	if (isEmptyFieldForLayout(Context: CGF.getContext(), FD: F))
936	return;
937	if (!FirstField)
938	addInitialField(F);
939	else
940	addNextField(F);
941	}
942
943	CharUnits getMemcpySize(uint64_t FirstByteOffset) const {
944	ASTContext &Ctx = CGF.getContext();
945	unsigned LastFieldSize =
946	LastField->isBitField()
947	? LastField->getBitWidthValue()
948	: Ctx.toBits(
949	CharSize: Ctx.getTypeInfoDataSizeInChars(T: LastField->getType()).Width);
950	uint64_t MemcpySizeBits = LastFieldOffset + LastFieldSize -
951	FirstByteOffset + Ctx.getCharWidth() - 1;
952	CharUnits MemcpySize = Ctx.toCharUnitsFromBits(BitSize: MemcpySizeBits);
953	return MemcpySize;
954	}
955
956	void emitMemcpy() {
957	// Give the subclass a chance to bail out if it feels the memcpy isn't
958	// worth it (e.g. Hasn't aggregated enough data).
959	if (!FirstField) {
960	return;
961	}
962
963	uint64_t FirstByteOffset;
964	if (FirstField->isBitField()) {
965	const CGRecordLayout &RL =
966	CGF.getTypes().getCGRecordLayout(FirstField->getParent());
967	const CGBitFieldInfo &BFInfo = RL.getBitFieldInfo(FD: FirstField);
968	// FirstFieldOffset is not appropriate for bitfields,
969	// we need to use the storage offset instead.
970	FirstByteOffset = CGF.getContext().toBits(CharSize: BFInfo.StorageOffset);
971	} else {
972	FirstByteOffset = FirstFieldOffset;
973	}
974
975	CharUnits MemcpySize = getMemcpySize(FirstByteOffset);
976	QualType RecordTy = CGF.getContext().getTypeDeclType(ClassDecl);
977	Address ThisPtr = CGF.LoadCXXThisAddress();
978	LValue DestLV = CGF.MakeAddrLValue(Addr: ThisPtr, T: RecordTy);
979	LValue Dest = CGF.EmitLValueForFieldInitialization(Base: DestLV, Field: FirstField);
980	llvm::Value *SrcPtr = CGF.Builder.CreateLoad(Addr: CGF.GetAddrOfLocalVar(VD: SrcRec));
981	LValue SrcLV = CGF.MakeNaturalAlignAddrLValue(V: SrcPtr, T: RecordTy);
982	LValue Src = CGF.EmitLValueForFieldInitialization(Base: SrcLV, Field: FirstField);
983
984	emitMemcpyIR(
985	DestPtr: Dest.isBitField() ? Dest.getBitFieldAddress() : Dest.getAddress(),
986	SrcPtr: Src.isBitField() ? Src.getBitFieldAddress() : Src.getAddress(),
987	Size: MemcpySize);
988	reset();
989	}
990
991	void reset() {
992	FirstField = nullptr;
993	}
994
995	protected:
996	CodeGenFunction &CGF;
997	const CXXRecordDecl *ClassDecl;
998
999	private:
1000	void emitMemcpyIR(Address DestPtr, Address SrcPtr, CharUnits Size) {
1001	DestPtr = DestPtr.withElementType(ElemTy: CGF.Int8Ty);
1002	SrcPtr = SrcPtr.withElementType(ElemTy: CGF.Int8Ty);
1003	CGF.Builder.CreateMemCpy(Dest: DestPtr, Src: SrcPtr, Size: Size.getQuantity());
1004	}
1005
1006	void addInitialField(FieldDecl *F) {
1007	FirstField = F;
1008	LastField = F;
1009	FirstFieldOffset = RecLayout.getFieldOffset(FieldNo: F->getFieldIndex());
1010	LastFieldOffset = FirstFieldOffset;
1011	LastAddedFieldIndex = F->getFieldIndex();
1012	}
1013
1014	void addNextField(FieldDecl *F) {
1015	// For the most part, the following invariant will hold:
1016	// F->getFieldIndex() == LastAddedFieldIndex + 1
1017	// The one exception is that Sema won't add a copy-initializer for an
1018	// unnamed bitfield, which will show up here as a gap in the sequence.
1019	assert(F->getFieldIndex() >= LastAddedFieldIndex + 1 &&
1020	"Cannot aggregate fields out of order.");
1021	LastAddedFieldIndex = F->getFieldIndex();
1022
1023	// The 'first' and 'last' fields are chosen by offset, rather than field
1024	// index. This allows the code to support bitfields, as well as regular
1025	// fields.
1026	uint64_t FOffset = RecLayout.getFieldOffset(FieldNo: F->getFieldIndex());
1027	if (FOffset < FirstFieldOffset) {
1028	FirstField = F;
1029	FirstFieldOffset = FOffset;
1030	} else if (FOffset >= LastFieldOffset) {
1031	LastField = F;
1032	LastFieldOffset = FOffset;
1033	}
1034	}
1035
1036	const VarDecl *SrcRec;
1037	const ASTRecordLayout &RecLayout;
1038	FieldDecl *FirstField;
1039	FieldDecl *LastField;
1040	uint64_t FirstFieldOffset, LastFieldOffset;
1041	unsigned LastAddedFieldIndex;
1042	};
1043
1044	class ConstructorMemcpyizer : public FieldMemcpyizer {
1045	private:
1046	/// Get source argument for copy constructor. Returns null if not a copy
1047	/// constructor.
1048	static const VarDecl *getTrivialCopySource(CodeGenFunction &CGF,
1049	const CXXConstructorDecl *CD,
1050	FunctionArgList &Args) {
1051	if (CD->isCopyOrMoveConstructor() && CD->isDefaulted())
1052	return Args[CGF.CGM.getCXXABI().getSrcArgforCopyCtor(CD, Args)];
1053	return nullptr;
1054	}
1055
1056	// Returns true if a CXXCtorInitializer represents a member initialization
1057	// that can be rolled into a memcpy.
1058	bool isMemberInitMemcpyable(CXXCtorInitializer *MemberInit) const {
1059	if (!MemcpyableCtor)
1060	return false;
1061	FieldDecl *Field = MemberInit->getMember();
1062	assert(Field && "No field for member init.");
1063	QualType FieldType = Field->getType();
1064	CXXConstructExpr *CE = dyn_cast<CXXConstructExpr>(Val: MemberInit->getInit());
1065
1066	// Bail out on non-memcpyable, not-trivially-copyable members.
1067	if (!(CE && isMemcpyEquivalentSpecialMember(CE->getConstructor())) &&
1068	!(FieldType.isTriviallyCopyableType(Context: CGF.getContext()) ||
1069	FieldType->isReferenceType()))
1070	return false;
1071
1072	// Bail out on volatile fields.
1073	if (!isMemcpyableField(F: Field))
1074	return false;
1075
1076	// Otherwise we're good.
1077	return true;
1078	}
1079
1080	public:
1081	ConstructorMemcpyizer(CodeGenFunction &CGF, const CXXConstructorDecl *CD,
1082	FunctionArgList &Args)
1083	: FieldMemcpyizer(CGF, CD->getParent(), getTrivialCopySource(CGF, CD, Args)),
1084	ConstructorDecl(CD),
1085	MemcpyableCtor(CD->isDefaulted() &&
1086	CD->isCopyOrMoveConstructor() &&
1087	CGF.getLangOpts().getGC() == LangOptions::NonGC),
1088	Args(Args) { }
1089
1090	void addMemberInitializer(CXXCtorInitializer *MemberInit) {
1091	if (isMemberInitMemcpyable(MemberInit)) {
1092	AggregatedInits.push_back(Elt: MemberInit);
1093	addMemcpyableField(F: MemberInit->getMember());
1094	} else {
1095	emitAggregatedInits();
1096	EmitMemberInitializer(CGF, ConstructorDecl->getParent(), MemberInit,
1097	ConstructorDecl, Args);
1098	}
1099	}
1100
1101	void emitAggregatedInits() {
1102	if (AggregatedInits.size() <= 1) {
1103	// This memcpy is too small to be worthwhile. Fall back on default
1104	// codegen.
1105	if (!AggregatedInits.empty()) {
1106	CopyingValueRepresentation CVR(CGF);
1107	EmitMemberInitializer(CGF, ConstructorDecl->getParent(),
1108	AggregatedInits[0], ConstructorDecl, Args);
1109	AggregatedInits.clear();
1110	}
1111	reset();
1112	return;
1113	}
1114
1115	pushEHDestructors();
1116	emitMemcpy();
1117	AggregatedInits.clear();
1118	}
1119
1120	void pushEHDestructors() {
1121	Address ThisPtr = CGF.LoadCXXThisAddress();
1122	QualType RecordTy = CGF.getContext().getTypeDeclType(ClassDecl);
1123	LValue LHS = CGF.MakeAddrLValue(Addr: ThisPtr, T: RecordTy);
1124
1125	for (unsigned i = 0; i < AggregatedInits.size(); ++i) {
1126	CXXCtorInitializer *MemberInit = AggregatedInits[i];
1127	QualType FieldType = MemberInit->getAnyMember()->getType();
1128	QualType::DestructionKind dtorKind = FieldType.isDestructedType();
1129	if (!CGF.needsEHCleanup(kind: dtorKind))
1130	continue;
1131	LValue FieldLHS = LHS;
1132	EmitLValueForAnyFieldInitialization(CGF, MemberInit, LHS&: FieldLHS);
1133	CGF.pushEHDestroy(dtorKind, addr: FieldLHS.getAddress(), type: FieldType);
1134	}
1135	}
1136
1137	void finish() {
1138	emitAggregatedInits();
1139	}
1140
1141	private:
1142	const CXXConstructorDecl *ConstructorDecl;
1143	bool MemcpyableCtor;
1144	FunctionArgList &Args;
1145	SmallVector<CXXCtorInitializer*, 16> AggregatedInits;
1146	};
1147
1148	class AssignmentMemcpyizer : public FieldMemcpyizer {
1149	private:
1150	// Returns the memcpyable field copied by the given statement, if one
1151	// exists. Otherwise returns null.
1152	FieldDecl *getMemcpyableField(Stmt *S) {
1153	if (!AssignmentsMemcpyable)
1154	return nullptr;
1155	if (BinaryOperator *BO = dyn_cast<BinaryOperator>(Val: S)) {
1156	// Recognise trivial assignments.
1157	if (BO->getOpcode() != BO_Assign)
1158	return nullptr;
1159	MemberExpr *ME = dyn_cast<MemberExpr>(Val: BO->getLHS());
1160	if (!ME)
1161	return nullptr;
1162	FieldDecl *Field = dyn_cast<FieldDecl>(Val: ME->getMemberDecl());
1163	if (!Field || !isMemcpyableField(F: Field))
1164	return nullptr;
1165	Stmt *RHS = BO->getRHS();
1166	if (ImplicitCastExpr *EC = dyn_cast<ImplicitCastExpr>(Val: RHS))
1167	RHS = EC->getSubExpr();
1168	if (!RHS)
1169	return nullptr;
1170	if (MemberExpr *ME2 = dyn_cast<MemberExpr>(Val: RHS)) {
1171	if (ME2->getMemberDecl() == Field)
1172	return Field;
1173	}
1174	return nullptr;
1175	} else if (CXXMemberCallExpr *MCE = dyn_cast<CXXMemberCallExpr>(Val: S)) {
1176	CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(MCE->getCalleeDecl());
1177	if (!(MD && isMemcpyEquivalentSpecialMember(D: MD)))
1178	return nullptr;
1179	MemberExpr *IOA = dyn_cast<MemberExpr>(Val: MCE->getImplicitObjectArgument());
1180	if (!IOA)
1181	return nullptr;
1182	FieldDecl *Field = dyn_cast<FieldDecl>(Val: IOA->getMemberDecl());
1183	if (!Field || !isMemcpyableField(F: Field))
1184	return nullptr;
1185	MemberExpr *Arg0 = dyn_cast<MemberExpr>(MCE->getArg(0));
1186	if (!Arg0 || Field != dyn_cast<FieldDecl>(Val: Arg0->getMemberDecl()))
1187	return nullptr;
1188	return Field;
1189	} else if (CallExpr *CE = dyn_cast<CallExpr>(Val: S)) {
1190	FunctionDecl *FD = dyn_cast<FunctionDecl>(Val: CE->getCalleeDecl());
1191	if (!FD || FD->getBuiltinID() != Builtin::BI__builtin_memcpy)
1192	return nullptr;
1193	Expr *DstPtr = CE->getArg(Arg: 0);
1194	if (ImplicitCastExpr *DC = dyn_cast<ImplicitCastExpr>(Val: DstPtr))
1195	DstPtr = DC->getSubExpr();
1196	UnaryOperator *DUO = dyn_cast<UnaryOperator>(Val: DstPtr);
1197	if (!DUO || DUO->getOpcode() != UO_AddrOf)
1198	return nullptr;
1199	MemberExpr *ME = dyn_cast<MemberExpr>(Val: DUO->getSubExpr());
1200	if (!ME)
1201	return nullptr;
1202	FieldDecl *Field = dyn_cast<FieldDecl>(Val: ME->getMemberDecl());
1203	if (!Field || !isMemcpyableField(F: Field))
1204	return nullptr;
1205	Expr *SrcPtr = CE->getArg(Arg: 1);
1206	if (ImplicitCastExpr *SC = dyn_cast<ImplicitCastExpr>(Val: SrcPtr))
1207	SrcPtr = SC->getSubExpr();
1208	UnaryOperator *SUO = dyn_cast<UnaryOperator>(Val: SrcPtr);
1209	if (!SUO || SUO->getOpcode() != UO_AddrOf)
1210	return nullptr;
1211	MemberExpr *ME2 = dyn_cast<MemberExpr>(Val: SUO->getSubExpr());
1212	if (!ME2 || Field != dyn_cast<FieldDecl>(Val: ME2->getMemberDecl()))
1213	return nullptr;
1214	return Field;
1215	}
1216
1217	return nullptr;
1218	}
1219
1220	bool AssignmentsMemcpyable;
1221	SmallVector<Stmt*, 16> AggregatedStmts;
1222
1223	public:
1224	AssignmentMemcpyizer(CodeGenFunction &CGF, const CXXMethodDecl *AD,
1225	FunctionArgList &Args)
1226	: FieldMemcpyizer(CGF, AD->getParent(), Args[Args.size() - 1]),
1227	AssignmentsMemcpyable(CGF.getLangOpts().getGC() == LangOptions::NonGC) {
1228	assert(Args.size() == 2);
1229	}
1230
1231	void emitAssignment(Stmt *S) {
1232	FieldDecl *F = getMemcpyableField(S);
1233	if (F) {
1234	addMemcpyableField(F);
1235	AggregatedStmts.push_back(Elt: S);
1236	} else {
1237	emitAggregatedStmts();
1238	CGF.EmitStmt(S);
1239	}
1240	}
1241
1242	void emitAggregatedStmts() {
1243	if (AggregatedStmts.size() <= 1) {
1244	if (!AggregatedStmts.empty()) {
1245	CopyingValueRepresentation CVR(CGF);
1246	CGF.EmitStmt(S: AggregatedStmts[0]);
1247	}
1248	reset();
1249	}
1250
1251	emitMemcpy();
1252	AggregatedStmts.clear();
1253	}
1254
1255	void finish() {
1256	emitAggregatedStmts();
1257	}
1258	};
1259	} // end anonymous namespace
1260
1261	static bool isInitializerOfDynamicClass(const CXXCtorInitializer *BaseInit) {
1262	const Type *BaseType = BaseInit->getBaseClass();
1263	const auto *BaseClassDecl =
1264	cast<CXXRecordDecl>(Val: BaseType->castAs<RecordType>()->getDecl());
1265	return BaseClassDecl->isDynamicClass();
1266	}
1267
1268	/// EmitCtorPrologue - This routine generates necessary code to initialize
1269	/// base classes and non-static data members belonging to this constructor.
1270	void CodeGenFunction::EmitCtorPrologue(const CXXConstructorDecl *CD,
1271	CXXCtorType CtorType,
1272	FunctionArgList &Args) {
1273	if (CD->isDelegatingConstructor())
1274	return EmitDelegatingCXXConstructorCall(Ctor: CD, Args);
1275
1276	const CXXRecordDecl *ClassDecl = CD->getParent();
1277
1278	CXXConstructorDecl::init_const_iterator B = CD->init_begin(),
1279	E = CD->init_end();
1280
1281	// Virtual base initializers first, if any. They aren't needed if:
1282	// - This is a base ctor variant
1283	// - There are no vbases
1284	// - The class is abstract, so a complete object of it cannot be constructed
1285	//
1286	// The check for an abstract class is necessary because sema may not have
1287	// marked virtual base destructors referenced.
1288	bool ConstructVBases = CtorType != Ctor_Base &&
1289	ClassDecl->getNumVBases() != 0 &&
1290	!ClassDecl->isAbstract();
1291
1292	// In the Microsoft C++ ABI, there are no constructor variants. Instead, the
1293	// constructor of a class with virtual bases takes an additional parameter to
1294	// conditionally construct the virtual bases. Emit that check here.
1295	llvm::BasicBlock *BaseCtorContinueBB = nullptr;
1296	if (ConstructVBases &&
1297	!CGM.getTarget().getCXXABI().hasConstructorVariants()) {
1298	BaseCtorContinueBB =
1299	CGM.getCXXABI().EmitCtorCompleteObjectHandler(CGF&: *this, RD: ClassDecl);
1300	assert(BaseCtorContinueBB);
1301	}
1302
1303	for (; B != E && (*B)->isBaseInitializer() && (*B)->isBaseVirtual(); B++) {
1304	if (!ConstructVBases)
1305	continue;
1306	SaveAndRestore ThisRAII(CXXThisValue);
1307	if (CGM.getCodeGenOpts().StrictVTablePointers &&
1308	CGM.getCodeGenOpts().OptimizationLevel > 0 &&
1309	isInitializerOfDynamicClass(BaseInit: *B))
1310	CXXThisValue = Builder.CreateLaunderInvariantGroup(Ptr: LoadCXXThis());
1311	EmitBaseInitializer(CGF&: *this, ClassDecl, BaseInit: *B);
1312	}
1313
1314	if (BaseCtorContinueBB) {
1315	// Complete object handler should continue to the remaining initializers.
1316	Builder.CreateBr(Dest: BaseCtorContinueBB);
1317	EmitBlock(BB: BaseCtorContinueBB);
1318	}
1319
1320	// Then, non-virtual base initializers.
1321	for (; B != E && (*B)->isBaseInitializer(); B++) {
1322	assert(!(*B)->isBaseVirtual());
1323	SaveAndRestore ThisRAII(CXXThisValue);
1324	if (CGM.getCodeGenOpts().StrictVTablePointers &&
1325	CGM.getCodeGenOpts().OptimizationLevel > 0 &&
1326	isInitializerOfDynamicClass(BaseInit: *B))
1327	CXXThisValue = Builder.CreateLaunderInvariantGroup(Ptr: LoadCXXThis());
1328	EmitBaseInitializer(CGF&: *this, ClassDecl, BaseInit: *B);
1329	}
1330
1331	InitializeVTablePointers(ClassDecl);
1332
1333	// And finally, initialize class members.
1334	FieldConstructionScope FCS(*this, LoadCXXThisAddress());
1335	ConstructorMemcpyizer CM(*this, CD, Args);
1336	for (; B != E; B++) {
1337	CXXCtorInitializer *Member = (*B);
1338	assert(!Member->isBaseInitializer());
1339	assert(Member->isAnyMemberInitializer() &&
1340	"Delegating initializer on non-delegating constructor");
1341	ApplyAtomGroup Grp(getDebugInfo());
1342	CM.addMemberInitializer(MemberInit: Member);
1343	}
1344	CM.finish();
1345	}
1346
1347	static bool
1348	FieldHasTrivialDestructorBody(ASTContext &Context, const FieldDecl *Field);
1349
1350	static bool
1351	HasTrivialDestructorBody(ASTContext &Context,
1352	const CXXRecordDecl *BaseClassDecl,
1353	const CXXRecordDecl *MostDerivedClassDecl)
1354	{
1355	// If the destructor is trivial we don't have to check anything else.
1356	if (BaseClassDecl->hasTrivialDestructor())
1357	return true;
1358
1359	if (!BaseClassDecl->getDestructor()->hasTrivialBody())
1360	return false;
1361
1362	// Check fields.
1363	for (const auto *Field : BaseClassDecl->fields())
1364	if (!FieldHasTrivialDestructorBody(Context, Field))
1365	return false;
1366
1367	// Check non-virtual bases.
1368	for (const auto &I : BaseClassDecl->bases()) {
1369	if (I.isVirtual())
1370	continue;
1371
1372	const CXXRecordDecl *NonVirtualBase =
1373	cast<CXXRecordDecl>(Val: I.getType()->castAs<RecordType>()->getDecl());
1374	if (!HasTrivialDestructorBody(Context, BaseClassDecl: NonVirtualBase,
1375	MostDerivedClassDecl))
1376	return false;
1377	}
1378
1379	if (BaseClassDecl == MostDerivedClassDecl) {
1380	// Check virtual bases.
1381	for (const auto &I : BaseClassDecl->vbases()) {
1382	const CXXRecordDecl *VirtualBase =
1383	cast<CXXRecordDecl>(Val: I.getType()->castAs<RecordType>()->getDecl());
1384	if (!HasTrivialDestructorBody(Context, BaseClassDecl: VirtualBase,
1385	MostDerivedClassDecl))
1386	return false;
1387	}
1388	}
1389
1390	return true;
1391	}
1392
1393	static bool
1394	FieldHasTrivialDestructorBody(ASTContext &Context,
1395	const FieldDecl *Field)
1396	{
1397	QualType FieldBaseElementType = Context.getBaseElementType(Field->getType());
1398
1399	const RecordType *RT = FieldBaseElementType->getAs<RecordType>();
1400	if (!RT)
1401	return true;
1402
1403	CXXRecordDecl *FieldClassDecl = cast<CXXRecordDecl>(Val: RT->getDecl());
1404
1405	// The destructor for an implicit anonymous union member is never invoked.
1406	if (FieldClassDecl->isUnion() && FieldClassDecl->isAnonymousStructOrUnion())
1407	return true;
1408
1409	return HasTrivialDestructorBody(Context, BaseClassDecl: FieldClassDecl, MostDerivedClassDecl: FieldClassDecl);
1410	}
1411
1412	/// CanSkipVTablePointerInitialization - Check whether we need to initialize
1413	/// any vtable pointers before calling this destructor.
1414	static bool CanSkipVTablePointerInitialization(CodeGenFunction &CGF,
1415	const CXXDestructorDecl *Dtor) {
1416	const CXXRecordDecl *ClassDecl = Dtor->getParent();
1417	if (!ClassDecl->isDynamicClass())
1418	return true;
1419
1420	// For a final class, the vtable pointer is known to already point to the
1421	// class's vtable.
1422	if (ClassDecl->isEffectivelyFinal())
1423	return true;
1424
1425	if (!Dtor->hasTrivialBody())
1426	return false;
1427
1428	// Check the fields.
1429	for (const auto *Field : ClassDecl->fields())
1430	if (!FieldHasTrivialDestructorBody(CGF.getContext(), Field))
1431	return false;
1432
1433	return true;
1434	}
1435
1436	/// EmitDestructorBody - Emits the body of the current destructor.
1437	void CodeGenFunction::EmitDestructorBody(FunctionArgList &Args) {
1438	const CXXDestructorDecl *Dtor = cast<CXXDestructorDecl>(Val: CurGD.getDecl());
1439	CXXDtorType DtorType = CurGD.getDtorType();
1440
1441	// For an abstract class, non-base destructors are never used (and can't
1442	// be emitted in general, because vbase dtors may not have been validated
1443	// by Sema), but the Itanium ABI doesn't make them optional and Clang may
1444	// in fact emit references to them from other compilations, so emit them
1445	// as functions containing a trap instruction.
1446	if (DtorType != Dtor_Base && Dtor->getParent()->isAbstract()) {
1447	llvm::CallInst *TrapCall = EmitTrapCall(llvm::Intrinsic::trap);
1448	TrapCall->setDoesNotReturn();
1449	TrapCall->setDoesNotThrow();
1450	Builder.CreateUnreachable();
1451	Builder.ClearInsertionPoint();
1452	return;
1453	}
1454
1455	Stmt *Body = Dtor->getBody();
1456	if (Body) {
1457	incrementProfileCounter(S: Body);
1458	maybeCreateMCDCCondBitmap();
1459	}
1460
1461	// The call to operator delete in a deleting destructor happens
1462	// outside of the function-try-block, which means it's always
1463	// possible to delegate the destructor body to the complete
1464	// destructor. Do so.
1465	if (DtorType == Dtor_Deleting) {
1466	RunCleanupsScope DtorEpilogue(*this);
1467	EnterDtorCleanups(Dtor, Type: Dtor_Deleting);
1468	if (HaveInsertPoint()) {
1469	QualType ThisTy = Dtor->getFunctionObjectParameterType();
1470	EmitCXXDestructorCall(D: Dtor, Type: Dtor_Complete, /*ForVirtualBase=*/false,
1471	/*Delegating=*/false, This: LoadCXXThisAddress(), ThisTy);
1472	}
1473	return;
1474	}
1475
1476	// If the body is a function-try-block, enter the try before
1477	// anything else.
1478	bool isTryBody = isa_and_nonnull<CXXTryStmt>(Val: Body);
1479	if (isTryBody)
1480	EnterCXXTryStmt(S: *cast<CXXTryStmt>(Val: Body), IsFnTryBlock: true);
1481	EmitAsanPrologueOrEpilogue(Prologue: false);
1482
1483	// Enter the epilogue cleanups.
1484	RunCleanupsScope DtorEpilogue(*this);
1485
1486	// If this is the complete variant, just invoke the base variant;
1487	// the epilogue will destruct the virtual bases. But we can't do
1488	// this optimization if the body is a function-try-block, because
1489	// we'd introduce *two* handler blocks. In the Microsoft ABI, we
1490	// always delegate because we might not have a definition in this TU.
1491	switch (DtorType) {
1492	case Dtor_Comdat: llvm_unreachable("not expecting a COMDAT");
1493	case Dtor_Deleting: llvm_unreachable("already handled deleting case");
1494
1495	case Dtor_Complete:
1496	assert((Body || getTarget().getCXXABI().isMicrosoft()) &&
1497	"can't emit a dtor without a body for non-Microsoft ABIs");
1498
1499	// Enter the cleanup scopes for virtual bases.
1500	EnterDtorCleanups(Dtor, Type: Dtor_Complete);
1501
1502	if (!isTryBody) {
1503	QualType ThisTy = Dtor->getFunctionObjectParameterType();
1504	EmitCXXDestructorCall(D: Dtor, Type: Dtor_Base, /*ForVirtualBase=*/false,
1505	/*Delegating=*/false, This: LoadCXXThisAddress(), ThisTy);
1506	break;
1507	}
1508
1509	// Fallthrough: act like we're in the base variant.
1510	[[fallthrough]];
1511
1512	case Dtor_Base:
1513	assert(Body);
1514
1515	// Enter the cleanup scopes for fields and non-virtual bases.
1516	EnterDtorCleanups(Dtor, Type: Dtor_Base);
1517
1518	// Initialize the vtable pointers before entering the body.
1519	if (!CanSkipVTablePointerInitialization(CGF&: *this, Dtor)) {
1520	// Insert the llvm.launder.invariant.group intrinsic before initializing
1521	// the vptrs to cancel any previous assumptions we might have made.
1522	if (CGM.getCodeGenOpts().StrictVTablePointers &&
1523	CGM.getCodeGenOpts().OptimizationLevel > 0)
1524	CXXThisValue = Builder.CreateLaunderInvariantGroup(Ptr: LoadCXXThis());
1525	InitializeVTablePointers(ClassDecl: Dtor->getParent());
1526	}
1527
1528	if (isTryBody)
1529	EmitStmt(cast<CXXTryStmt>(Val: Body)->getTryBlock());
1530	else if (Body)
1531	EmitStmt(S: Body);
1532	else {
1533	assert(Dtor->isImplicit() && "bodyless dtor not implicit");
1534	// nothing to do besides what's in the epilogue
1535	}
1536	// -fapple-kext must inline any call to this dtor into
1537	// the caller's body.
1538	if (getLangOpts().AppleKext)
1539	CurFn->addFnAttr(llvm::Attribute::AlwaysInline);
1540
1541	break;
1542	}
1543
1544	// Jump out through the epilogue cleanups.
1545	DtorEpilogue.ForceCleanup();
1546
1547	// Exit the try if applicable.
1548	if (isTryBody)
1549	ExitCXXTryStmt(S: *cast<CXXTryStmt>(Val: Body), IsFnTryBlock: true);
1550	}
1551
1552	void CodeGenFunction::emitImplicitAssignmentOperatorBody(FunctionArgList &Args) {
1553	const CXXMethodDecl *AssignOp = cast<CXXMethodDecl>(Val: CurGD.getDecl());
1554	const Stmt *RootS = AssignOp->getBody();
1555	assert(isa<CompoundStmt>(RootS) &&
1556	"Body of an implicit assignment operator should be compound stmt.");
1557	const CompoundStmt *RootCS = cast<CompoundStmt>(Val: RootS);
1558
1559	LexicalScope Scope(*this, RootCS->getSourceRange());
1560
1561	incrementProfileCounter(RootCS);
1562	maybeCreateMCDCCondBitmap();
1563	AssignmentMemcpyizer AM(*this, AssignOp, Args);
1564	for (auto *I : RootCS->body())
1565	AM.emitAssignment(I);
1566	AM.finish();
1567	}
1568
1569	namespace {
1570	llvm::Value *LoadThisForDtorDelete(CodeGenFunction &CGF,
1571	const CXXDestructorDecl *DD) {
1572	if (Expr *ThisArg = DD->getOperatorDeleteThisArg())
1573	return CGF.EmitScalarExpr(E: ThisArg);
1574	return CGF.LoadCXXThis();
1575	}
1576
1577	/// Call the operator delete associated with the current destructor.
1578	struct CallDtorDelete final : EHScopeStack::Cleanup {
1579	CallDtorDelete() {}
1580
1581	void Emit(CodeGenFunction &CGF, Flags flags) override {
1582	const CXXDestructorDecl *Dtor = cast<CXXDestructorDecl>(Val: CGF.CurCodeDecl);
1583	const CXXRecordDecl *ClassDecl = Dtor->getParent();
1584	CGF.EmitDeleteCall(DeleteFD: Dtor->getOperatorDelete(),
1585	Ptr: LoadThisForDtorDelete(CGF, DD: Dtor),
1586	DeleteTy: CGF.getContext().getTagDeclType(ClassDecl));
1587	}
1588	};
1589
1590	void EmitConditionalDtorDeleteCall(CodeGenFunction &CGF,
1591	llvm::Value *ShouldDeleteCondition,
1592	bool ReturnAfterDelete) {
1593	llvm::BasicBlock *callDeleteBB = CGF.createBasicBlock(name: "dtor.call_delete");
1594	llvm::BasicBlock *continueBB = CGF.createBasicBlock(name: "dtor.continue");
1595	llvm::Value *ShouldCallDelete
1596	= CGF.Builder.CreateIsNull(Arg: ShouldDeleteCondition);
1597	CGF.Builder.CreateCondBr(Cond: ShouldCallDelete, True: continueBB, False: callDeleteBB);
1598
1599	CGF.EmitBlock(BB: callDeleteBB);
1600	const CXXDestructorDecl *Dtor = cast<CXXDestructorDecl>(Val: CGF.CurCodeDecl);
1601	const CXXRecordDecl *ClassDecl = Dtor->getParent();
1602	CGF.EmitDeleteCall(DeleteFD: Dtor->getOperatorDelete(),
1603	Ptr: LoadThisForDtorDelete(CGF, DD: Dtor),
1604	DeleteTy: CGF.getContext().getTagDeclType(ClassDecl));
1605	assert(Dtor->getOperatorDelete()->isDestroyingOperatorDelete() ==
1606	ReturnAfterDelete &&
1607	"unexpected value for ReturnAfterDelete");
1608	if (ReturnAfterDelete)
1609	CGF.EmitBranchThroughCleanup(Dest: CGF.ReturnBlock);
1610	else
1611	CGF.Builder.CreateBr(Dest: continueBB);
1612
1613	CGF.EmitBlock(BB: continueBB);
1614	}
1615
1616	struct CallDtorDeleteConditional final : EHScopeStack::Cleanup {
1617	llvm::Value *ShouldDeleteCondition;
1618
1619	public:
1620	CallDtorDeleteConditional(llvm::Value *ShouldDeleteCondition)
1621	: ShouldDeleteCondition(ShouldDeleteCondition) {
1622	assert(ShouldDeleteCondition != nullptr);
1623	}
1624
1625	void Emit(CodeGenFunction &CGF, Flags flags) override {
1626	EmitConditionalDtorDeleteCall(CGF, ShouldDeleteCondition,
1627	/*ReturnAfterDelete*/false);
1628	}
1629	};
1630
1631	class DestroyField final : public EHScopeStack::Cleanup {
1632	const FieldDecl *field;
1633	CodeGenFunction::Destroyer *destroyer;
1634	bool useEHCleanupForArray;
1635
1636	public:
1637	DestroyField(const FieldDecl *field, CodeGenFunction::Destroyer *destroyer,
1638	bool useEHCleanupForArray)
1639	: field(field), destroyer(destroyer),
1640	useEHCleanupForArray(useEHCleanupForArray) {}
1641
1642	void Emit(CodeGenFunction &CGF, Flags flags) override {
1643	// Find the address of the field.
1644	Address thisValue = CGF.LoadCXXThisAddress();
1645	QualType RecordTy = CGF.getContext().getTagDeclType(field->getParent());
1646	LValue ThisLV = CGF.MakeAddrLValue(Addr: thisValue, T: RecordTy);
1647	LValue LV = CGF.EmitLValueForField(Base: ThisLV, Field: field);
1648	assert(LV.isSimple());
1649
1650	CGF.emitDestroy(addr: LV.getAddress(), type: field->getType(), destroyer,
1651	useEHCleanupForArray: flags.isForNormalCleanup() && useEHCleanupForArray);
1652	}
1653	};
1654
1655	class DeclAsInlineDebugLocation {
1656	CGDebugInfo *DI;
1657	llvm::MDNode *InlinedAt;
1658	std::optional<ApplyDebugLocation> Location;
1659
1660	public:
1661	DeclAsInlineDebugLocation(CodeGenFunction &CGF, const NamedDecl &Decl)
1662	: DI(CGF.getDebugInfo()) {
1663	if (!DI)
1664	return;
1665	InlinedAt = DI->getInlinedAt();
1666	DI->setInlinedAt(CGF.Builder.getCurrentDebugLocation());
1667	Location.emplace(CGF, Decl.getLocation());
1668	}
1669
1670	~DeclAsInlineDebugLocation() {
1671	if (!DI)
1672	return;
1673	Location.reset();
1674	DI->setInlinedAt(InlinedAt);
1675	}
1676	};
1677
1678	static void EmitSanitizerDtorCallback(
1679	CodeGenFunction &CGF, StringRef Name, llvm::Value *Ptr,
1680	std::optional<CharUnits::QuantityType> PoisonSize = {}) {
1681	CodeGenFunction::SanitizerScope SanScope(&CGF);
1682	// Pass in void pointer and size of region as arguments to runtime
1683	// function
1684	SmallVector<llvm::Value *, 2> Args = {Ptr};
1685	SmallVector<llvm::Type *, 2> ArgTypes = {CGF.VoidPtrTy};
1686
1687	if (PoisonSize.has_value()) {
1688	Args.emplace_back(Args: llvm::ConstantInt::get(Ty: CGF.SizeTy, V: *PoisonSize));
1689	ArgTypes.emplace_back(Args&: CGF.SizeTy);
1690	}
1691
1692	llvm::FunctionType *FnType =
1693	llvm::FunctionType::get(Result: CGF.VoidTy, Params: ArgTypes, isVarArg: false);
1694	llvm::FunctionCallee Fn = CGF.CGM.CreateRuntimeFunction(Ty: FnType, Name);
1695
1696	CGF.EmitNounwindRuntimeCall(callee: Fn, args: Args);
1697	}
1698
1699	static void
1700	EmitSanitizerDtorFieldsCallback(CodeGenFunction &CGF, llvm::Value *Ptr,
1701	CharUnits::QuantityType PoisonSize) {
1702	EmitSanitizerDtorCallback(CGF, Name: "__sanitizer_dtor_callback_fields", Ptr,
1703	PoisonSize);
1704	}
1705
1706	/// Poison base class with a trivial destructor.
1707	struct SanitizeDtorTrivialBase final : EHScopeStack::Cleanup {
1708	const CXXRecordDecl *BaseClass;
1709	bool BaseIsVirtual;
1710	SanitizeDtorTrivialBase(const CXXRecordDecl *Base, bool BaseIsVirtual)
1711	: BaseClass(Base), BaseIsVirtual(BaseIsVirtual) {}
1712
1713	void Emit(CodeGenFunction &CGF, Flags flags) override {
1714	const CXXRecordDecl *DerivedClass =
1715	cast<CXXMethodDecl>(Val: CGF.CurCodeDecl)->getParent();
1716
1717	Address Addr = CGF.GetAddressOfDirectBaseInCompleteClass(
1718	This: CGF.LoadCXXThisAddress(), Derived: DerivedClass, Base: BaseClass, BaseIsVirtual);
1719
1720	const ASTRecordLayout &BaseLayout =
1721	CGF.getContext().getASTRecordLayout(BaseClass);
1722	CharUnits BaseSize = BaseLayout.getSize();
1723
1724	if (!BaseSize.isPositive())
1725	return;
1726
1727	// Use the base class declaration location as inline DebugLocation. All
1728	// fields of the class are destroyed.
1729	DeclAsInlineDebugLocation InlineHere(CGF, *BaseClass);
1730	EmitSanitizerDtorFieldsCallback(CGF, Ptr: Addr.emitRawPointer(CGF),
1731	PoisonSize: BaseSize.getQuantity());
1732
1733	// Prevent the current stack frame from disappearing from the stack trace.
1734	CGF.CurFn->addFnAttr(Kind: "disable-tail-calls", Val: "true");
1735	}
1736	};
1737
1738	class SanitizeDtorFieldRange final : public EHScopeStack::Cleanup {
1739	const CXXDestructorDecl *Dtor;
1740	unsigned StartIndex;
1741	unsigned EndIndex;
1742
1743	public:
1744	SanitizeDtorFieldRange(const CXXDestructorDecl *Dtor, unsigned StartIndex,
1745	unsigned EndIndex)
1746	: Dtor(Dtor), StartIndex(StartIndex), EndIndex(EndIndex) {}
1747
1748	// Generate function call for handling object poisoning.
1749	// Disables tail call elimination, to prevent the current stack frame
1750	// from disappearing from the stack trace.
1751	void Emit(CodeGenFunction &CGF, Flags flags) override {
1752	const ASTContext &Context = CGF.getContext();
1753	const ASTRecordLayout &Layout =
1754	Context.getASTRecordLayout(D: Dtor->getParent());
1755
1756	// It's a first trivial field so it should be at the begining of a char,
1757	// still round up start offset just in case.
1758	CharUnits PoisonStart = Context.toCharUnitsFromBits(
1759	BitSize: Layout.getFieldOffset(FieldNo: StartIndex) + Context.getCharWidth() - 1);
1760	llvm::ConstantInt *OffsetSizePtr =
1761	llvm::ConstantInt::get(Ty: CGF.SizeTy, V: PoisonStart.getQuantity());
1762
1763	llvm::Value *OffsetPtr =
1764	CGF.Builder.CreateGEP(Ty: CGF.Int8Ty, Ptr: CGF.LoadCXXThis(), IdxList: OffsetSizePtr);
1765
1766	CharUnits PoisonEnd;
1767	if (EndIndex >= Layout.getFieldCount()) {
1768	PoisonEnd = Layout.getNonVirtualSize();
1769	} else {
1770	PoisonEnd =
1771	Context.toCharUnitsFromBits(BitSize: Layout.getFieldOffset(FieldNo: EndIndex));
1772	}
1773	CharUnits PoisonSize = PoisonEnd - PoisonStart;
1774	if (!PoisonSize.isPositive())
1775	return;
1776
1777	// Use the top field declaration location as inline DebugLocation.
1778	DeclAsInlineDebugLocation InlineHere(
1779	CGF, **std::next(Dtor->getParent()->field_begin(), StartIndex));
1780	EmitSanitizerDtorFieldsCallback(CGF, Ptr: OffsetPtr, PoisonSize: PoisonSize.getQuantity());
1781
1782	// Prevent the current stack frame from disappearing from the stack trace.
1783	CGF.CurFn->addFnAttr(Kind: "disable-tail-calls", Val: "true");
1784	}
1785	};
1786
1787	class SanitizeDtorVTable final : public EHScopeStack::Cleanup {
1788	const CXXDestructorDecl *Dtor;
1789
1790	public:
1791	SanitizeDtorVTable(const CXXDestructorDecl *Dtor) : Dtor(Dtor) {}
1792
1793	// Generate function call for handling vtable pointer poisoning.
1794	void Emit(CodeGenFunction &CGF, Flags flags) override {
1795	assert(Dtor->getParent()->isDynamicClass());
1796	(void)Dtor;
1797	// Poison vtable and vtable ptr if they exist for this class.
1798	llvm::Value *VTablePtr = CGF.LoadCXXThis();
1799
1800	// Pass in void pointer and size of region as arguments to runtime
1801	// function
1802	EmitSanitizerDtorCallback(CGF, Name: "__sanitizer_dtor_callback_vptr",
1803	Ptr: VTablePtr);
1804	}
1805	};
1806
1807	class SanitizeDtorCleanupBuilder {
1808	ASTContext &Context;
1809	EHScopeStack &EHStack;
1810	const CXXDestructorDecl *DD;
1811	std::optional<unsigned> StartIndex;
1812
1813	public:
1814	SanitizeDtorCleanupBuilder(ASTContext &Context, EHScopeStack &EHStack,
1815	const CXXDestructorDecl *DD)
1816	: Context(Context), EHStack(EHStack), DD(DD), StartIndex(std::nullopt) {}
1817	void PushCleanupForField(const FieldDecl *Field) {
1818	if (isEmptyFieldForLayout(Context, FD: Field))
1819	return;
1820	unsigned FieldIndex = Field->getFieldIndex();
1821	if (FieldHasTrivialDestructorBody(Context, Field)) {
1822	if (!StartIndex)
1823	StartIndex = FieldIndex;
1824	} else if (StartIndex) {
1825	EHStack.pushCleanup<SanitizeDtorFieldRange>(Kind: NormalAndEHCleanup, A: DD,
1826	A: *StartIndex, A: FieldIndex);
1827	StartIndex = std::nullopt;
1828	}
1829	}
1830	void End() {
1831	if (StartIndex)
1832	EHStack.pushCleanup<SanitizeDtorFieldRange>(Kind: NormalAndEHCleanup, A: DD,
1833	A: *StartIndex, A: -1);
1834	}
1835	};
1836	} // end anonymous namespace
1837
1838	/// Emit all code that comes at the end of class's
1839	/// destructor. This is to call destructors on members and base classes
1840	/// in reverse order of their construction.
1841	///
1842	/// For a deleting destructor, this also handles the case where a destroying
1843	/// operator delete completely overrides the definition.
1844	void CodeGenFunction::EnterDtorCleanups(const CXXDestructorDecl *DD,
1845	CXXDtorType DtorType) {
1846	assert((!DD->isTrivial() || DD->hasAttr<DLLExportAttr>()) &&
1847	"Should not emit dtor epilogue for non-exported trivial dtor!");
1848
1849	// The deleting-destructor phase just needs to call the appropriate
1850	// operator delete that Sema picked up.
1851	if (DtorType == Dtor_Deleting) {
1852	assert(DD->getOperatorDelete() &&
1853	"operator delete missing - EnterDtorCleanups");
1854	if (CXXStructorImplicitParamValue) {
1855	// If there is an implicit param to the deleting dtor, it's a boolean
1856	// telling whether this is a deleting destructor.
1857	if (DD->getOperatorDelete()->isDestroyingOperatorDelete())
1858	EmitConditionalDtorDeleteCall(CGF&: *this, ShouldDeleteCondition: CXXStructorImplicitParamValue,
1859	/*ReturnAfterDelete*/true);
1860	else
1861	EHStack.pushCleanup<CallDtorDeleteConditional>(
1862	Kind: NormalAndEHCleanup, A: CXXStructorImplicitParamValue);
1863	} else {
1864	if (DD->getOperatorDelete()->isDestroyingOperatorDelete()) {
1865	const CXXRecordDecl *ClassDecl = DD->getParent();
1866	EmitDeleteCall(DeleteFD: DD->getOperatorDelete(),
1867	Ptr: LoadThisForDtorDelete(CGF&: *this, DD),
1868	DeleteTy: getContext().getTagDeclType(ClassDecl));
1869	EmitBranchThroughCleanup(Dest: ReturnBlock);
1870	} else {
1871	EHStack.pushCleanup<CallDtorDelete>(Kind: NormalAndEHCleanup);
1872	}
1873	}
1874	return;
1875	}
1876
1877	const CXXRecordDecl *ClassDecl = DD->getParent();
1878
1879	// Unions have no bases and do not call field destructors.
1880	if (ClassDecl->isUnion())
1881	return;
1882
1883	// The complete-destructor phase just destructs all the virtual bases.
1884	if (DtorType == Dtor_Complete) {
1885	// Poison the vtable pointer such that access after the base
1886	// and member destructors are invoked is invalid.
1887	if (CGM.getCodeGenOpts().SanitizeMemoryUseAfterDtor &&
1888	SanOpts.has(K: SanitizerKind::Memory) && ClassDecl->getNumVBases() &&
1889	ClassDecl->isPolymorphic())
1890	EHStack.pushCleanup<SanitizeDtorVTable>(Kind: NormalAndEHCleanup, A: DD);
1891
1892	// We push them in the forward order so that they'll be popped in
1893	// the reverse order.
1894	for (const auto &Base : ClassDecl->vbases()) {
1895	auto *BaseClassDecl =
1896	cast<CXXRecordDecl>(Base.getType()->castAs<RecordType>()->getDecl());
1897
1898	if (BaseClassDecl->hasTrivialDestructor()) {
1899	// Under SanitizeMemoryUseAfterDtor, poison the trivial base class
1900	// memory. For non-trival base classes the same is done in the class
1901	// destructor.
1902	if (CGM.getCodeGenOpts().SanitizeMemoryUseAfterDtor &&
1903	SanOpts.has(SanitizerKind::Memory) && !BaseClassDecl->isEmpty())
1904	EHStack.pushCleanup<SanitizeDtorTrivialBase>(NormalAndEHCleanup,
1905	BaseClassDecl,
1906	/*BaseIsVirtual*/ true);
1907	} else {
1908	EHStack.pushCleanup<CallBaseDtor>(NormalAndEHCleanup, BaseClassDecl,
1909	/*BaseIsVirtual*/ true);
1910	}
1911	}
1912
1913	return;
1914	}
1915
1916	assert(DtorType == Dtor_Base);
1917	// Poison the vtable pointer if it has no virtual bases, but inherits
1918	// virtual functions.
1919	if (CGM.getCodeGenOpts().SanitizeMemoryUseAfterDtor &&
1920	SanOpts.has(K: SanitizerKind::Memory) && !ClassDecl->getNumVBases() &&
1921	ClassDecl->isPolymorphic())
1922	EHStack.pushCleanup<SanitizeDtorVTable>(Kind: NormalAndEHCleanup, A: DD);
1923
1924	// Destroy non-virtual bases.
1925	for (const auto &Base : ClassDecl->bases()) {
1926	// Ignore virtual bases.
1927	if (Base.isVirtual())
1928	continue;
1929
1930	CXXRecordDecl *BaseClassDecl = Base.getType()->getAsCXXRecordDecl();
1931
1932	if (BaseClassDecl->hasTrivialDestructor()) {
1933	if (CGM.getCodeGenOpts().SanitizeMemoryUseAfterDtor &&
1934	SanOpts.has(SanitizerKind::Memory) && !BaseClassDecl->isEmpty())
1935	EHStack.pushCleanup<SanitizeDtorTrivialBase>(NormalAndEHCleanup,
1936	BaseClassDecl,
1937	/*BaseIsVirtual*/ false);
1938	} else {
1939	EHStack.pushCleanup<CallBaseDtor>(NormalAndEHCleanup, BaseClassDecl,
1940	/*BaseIsVirtual*/ false);
1941	}
1942	}
1943
1944	// Poison fields such that access after their destructors are
1945	// invoked, and before the base class destructor runs, is invalid.
1946	bool SanitizeFields = CGM.getCodeGenOpts().SanitizeMemoryUseAfterDtor &&
1947	SanOpts.has(K: SanitizerKind::Memory);
1948	SanitizeDtorCleanupBuilder SanitizeBuilder(getContext(), EHStack, DD);
1949
1950	// Destroy direct fields.
1951	for (const auto *Field : ClassDecl->fields()) {
1952	if (SanitizeFields)
1953	SanitizeBuilder.PushCleanupForField(Field);
1954
1955	QualType type = Field->getType();
1956	QualType::DestructionKind dtorKind = type.isDestructedType();
1957	if (!dtorKind)
1958	continue;
1959
1960	// Anonymous union members do not have their destructors called.
1961	const RecordType *RT = type->getAsUnionType();
1962	if (RT && RT->getDecl()->isAnonymousStructOrUnion())
1963	continue;
1964
1965	CleanupKind cleanupKind = getCleanupKind(dtorKind);
1966	EHStack.pushCleanup<DestroyField>(
1967	cleanupKind, Field, getDestroyer(dtorKind), cleanupKind & EHCleanup);
1968	}
1969
1970	if (SanitizeFields)
1971	SanitizeBuilder.End();
1972	}
1973
1974	/// EmitCXXAggrConstructorCall - Emit a loop to call a particular
1975	/// constructor for each of several members of an array.
1976	///
1977	/// \param ctor the constructor to call for each element
1978	/// \param arrayType the type of the array to initialize
1979	/// \param arrayBegin an arrayType*
1980	/// \param zeroInitialize true if each element should be
1981	/// zero-initialized before it is constructed
1982	void CodeGenFunction::EmitCXXAggrConstructorCall(
1983	const CXXConstructorDecl *ctor, const ArrayType *arrayType,
1984	Address arrayBegin, const CXXConstructExpr *E, bool NewPointerIsChecked,
1985	bool zeroInitialize) {
1986	QualType elementType;
1987	llvm::Value *numElements =
1988	emitArrayLength(arrayType, baseType&: elementType, addr&: arrayBegin);
1989
1990	EmitCXXAggrConstructorCall(D: ctor, NumElements: numElements, ArrayPtr: arrayBegin, E,
1991	NewPointerIsChecked, ZeroInitialization: zeroInitialize);
1992	}
1993
1994	/// EmitCXXAggrConstructorCall - Emit a loop to call a particular
1995	/// constructor for each of several members of an array.
1996	///
1997	/// \param ctor the constructor to call for each element
1998	/// \param numElements the number of elements in the array;
1999	/// may be zero
2000	/// \param arrayBase a T*, where T is the type constructed by ctor
2001	/// \param zeroInitialize true if each element should be
2002	/// zero-initialized before it is constructed
2003	void CodeGenFunction::EmitCXXAggrConstructorCall(const CXXConstructorDecl *ctor,
2004	llvm::Value *numElements,
2005	Address arrayBase,
2006	const CXXConstructExpr *E,
2007	bool NewPointerIsChecked,
2008	bool zeroInitialize) {
2009	// It's legal for numElements to be zero. This can happen both
2010	// dynamically, because x can be zero in 'new A[x]', and statically,
2011	// because of GCC extensions that permit zero-length arrays. There
2012	// are probably legitimate places where we could assume that this
2013	// doesn't happen, but it's not clear that it's worth it.
2014	llvm::BranchInst *zeroCheckBranch = nullptr;
2015
2016	// Optimize for a constant count.
2017	llvm::ConstantInt *constantCount
2018	= dyn_cast<llvm::ConstantInt>(Val: numElements);
2019	if (constantCount) {
2020	// Just skip out if the constant count is zero.
2021	if (constantCount->isZero()) return;
2022
2023	// Otherwise, emit the check.
2024	} else {
2025	llvm::BasicBlock *loopBB = createBasicBlock(name: "new.ctorloop");
2026	llvm::Value *iszero = Builder.CreateIsNull(Arg: numElements, Name: "isempty");
2027	zeroCheckBranch = Builder.CreateCondBr(Cond: iszero, True: loopBB, False: loopBB);
2028	EmitBlock(BB: loopBB);
2029	}
2030
2031	// Find the end of the array.
2032	llvm::Type *elementType = arrayBase.getElementType();
2033	llvm::Value *arrayBegin = arrayBase.emitRawPointer(CGF&: *this);
2034	llvm::Value *arrayEnd = Builder.CreateInBoundsGEP(
2035	Ty: elementType, Ptr: arrayBegin, IdxList: numElements, Name: "arrayctor.end");
2036
2037	// Enter the loop, setting up a phi for the current location to initialize.
2038	llvm::BasicBlock *entryBB = Builder.GetInsertBlock();
2039	llvm::BasicBlock *loopBB = createBasicBlock(name: "arrayctor.loop");
2040	EmitBlock(BB: loopBB);
2041	llvm::PHINode *cur = Builder.CreatePHI(Ty: arrayBegin->getType(), NumReservedValues: 2,
2042	Name: "arrayctor.cur");
2043	cur->addIncoming(V: arrayBegin, BB: entryBB);
2044
2045	// Inside the loop body, emit the constructor call on the array element.
2046	if (CGM.shouldEmitConvergenceTokens())
2047	ConvergenceTokenStack.push_back(Elt: emitConvergenceLoopToken(BB: loopBB));
2048
2049	// The alignment of the base, adjusted by the size of a single element,
2050	// provides a conservative estimate of the alignment of every element.
2051	// (This assumes we never start tracking offsetted alignments.)
2052	//
2053	// Note that these are complete objects and so we don't need to
2054	// use the non-virtual size or alignment.
2055	QualType type = getContext().getTypeDeclType(Decl: ctor->getParent());
2056	CharUnits eltAlignment =
2057	arrayBase.getAlignment()
2058	.alignmentOfArrayElement(elementSize: getContext().getTypeSizeInChars(T: type));
2059	Address curAddr = Address(cur, elementType, eltAlignment);
2060
2061	// Zero initialize the storage, if requested.
2062	if (zeroInitialize)
2063	EmitNullInitialization(DestPtr: curAddr, Ty: type);
2064
2065	// C++ [class.temporary]p4:
2066	// There are two contexts in which temporaries are destroyed at a different
2067	// point than the end of the full-expression. The first context is when a
2068	// default constructor is called to initialize an element of an array.
2069	// If the constructor has one or more default arguments, the destruction of
2070	// every temporary created in a default argument expression is sequenced
2071	// before the construction of the next array element, if any.
2072
2073	{
2074	RunCleanupsScope Scope(*this);
2075
2076	// Evaluate the constructor and its arguments in a regular
2077	// partial-destroy cleanup.
2078	if (getLangOpts().Exceptions &&
2079	!ctor->getParent()->hasTrivialDestructor()) {
2080	Destroyer *destroyer = destroyCXXObject;
2081	pushRegularPartialArrayCleanup(arrayBegin, arrayEnd: cur, elementType: type, elementAlignment: eltAlignment,
2082	destroyer: *destroyer);
2083	}
2084	auto currAVS = AggValueSlot::forAddr(
2085	addr: curAddr, quals: type.getQualifiers(), isDestructed: AggValueSlot::IsDestructed,
2086	needsGC: AggValueSlot::DoesNotNeedGCBarriers, isAliased: AggValueSlot::IsNotAliased,
2087	mayOverlap: AggValueSlot::DoesNotOverlap, isZeroed: AggValueSlot::IsNotZeroed,
2088	isChecked: NewPointerIsChecked ? AggValueSlot::IsSanitizerChecked
2089	: AggValueSlot::IsNotSanitizerChecked);
2090	EmitCXXConstructorCall(ctor, Ctor_Complete, /*ForVirtualBase=*/false,
2091	/*Delegating=*/false, currAVS, E);
2092	}
2093
2094	// Go to the next element.
2095	llvm::Value *next = Builder.CreateInBoundsGEP(
2096	Ty: elementType, Ptr: cur, IdxList: llvm::ConstantInt::get(Ty: SizeTy, V: 1), Name: "arrayctor.next");
2097	cur->addIncoming(V: next, BB: Builder.GetInsertBlock());
2098
2099	// Check whether that's the end of the loop.
2100	llvm::Value *done = Builder.CreateICmpEQ(LHS: next, RHS: arrayEnd, Name: "arrayctor.done");
2101	llvm::BasicBlock *contBB = createBasicBlock(name: "arrayctor.cont");
2102	Builder.CreateCondBr(Cond: done, True: contBB, False: loopBB);
2103
2104	// Patch the earlier check to skip over the loop.
2105	if (zeroCheckBranch) zeroCheckBranch->setSuccessor(idx: 0, NewSucc: contBB);
2106
2107	if (CGM.shouldEmitConvergenceTokens())
2108	ConvergenceTokenStack.pop_back();
2109
2110	EmitBlock(BB: contBB);
2111	}
2112
2113	void CodeGenFunction::destroyCXXObject(CodeGenFunction &CGF,
2114	Address addr,
2115	QualType type) {
2116	const RecordType *rtype = type->castAs<RecordType>();
2117	const CXXRecordDecl *record = cast<CXXRecordDecl>(Val: rtype->getDecl());
2118	const CXXDestructorDecl *dtor = record->getDestructor();
2119	assert(!dtor->isTrivial());
2120	CGF.EmitCXXDestructorCall(D: dtor, Type: Dtor_Complete, /*for vbase*/ ForVirtualBase: false,
2121	/*Delegating=*/false, This: addr, ThisTy: type);
2122	}
2123
2124	void CodeGenFunction::EmitCXXConstructorCall(const CXXConstructorDecl *D,
2125	CXXCtorType Type,
2126	bool ForVirtualBase,
2127	bool Delegating,
2128	AggValueSlot ThisAVS,
2129	const CXXConstructExpr *E) {
2130	CallArgList Args;
2131	Address This = ThisAVS.getAddress();
2132	LangAS SlotAS = ThisAVS.getQualifiers().getAddressSpace();
2133	LangAS ThisAS = D->getFunctionObjectParameterType().getAddressSpace();
2134	llvm::Value *ThisPtr =
2135	getAsNaturalPointerTo(Addr: This, PointeeType: D->getThisType()->getPointeeType());
2136
2137	if (SlotAS != ThisAS) {
2138	unsigned TargetThisAS = getContext().getTargetAddressSpace(AS: ThisAS);
2139	llvm::Type *NewType =
2140	llvm::PointerType::get(C&: getLLVMContext(), AddressSpace: TargetThisAS);
2141	ThisPtr =
2142	getTargetHooks().performAddrSpaceCast(CGF&: *this, V: ThisPtr, SrcAddr: ThisAS, DestTy: NewType);
2143	}
2144
2145	// Push the this ptr.
2146	Args.add(rvalue: RValue::get(V: ThisPtr), type: D->getThisType());
2147
2148	// If this is a trivial constructor, emit a memcpy now before we lose
2149	// the alignment information on the argument.
2150	// FIXME: It would be better to preserve alignment information into CallArg.
2151	if (isMemcpyEquivalentSpecialMember(D)) {
2152	assert(E->getNumArgs() == 1 && "unexpected argcount for trivial ctor");
2153
2154	const Expr *Arg = E->getArg(Arg: 0);
2155	LValue Src = EmitLValue(E: Arg);
2156	QualType DestTy = getContext().getTypeDeclType(Decl: D->getParent());
2157	LValue Dest = MakeAddrLValue(Addr: This, T: DestTy);
2158	EmitAggregateCopyCtor(Dest, Src, MayOverlap: ThisAVS.mayOverlap());
2159	return;
2160	}
2161
2162	// Add the rest of the user-supplied arguments.
2163	const FunctionProtoType *FPT = D->getType()->castAs<FunctionProtoType>();
2164	EvaluationOrder Order = E->isListInitialization()
2165	? EvaluationOrder::ForceLeftToRight
2166	: EvaluationOrder::Default;
2167	EmitCallArgs(Args, Prototype: FPT, ArgRange: E->arguments(), AC: E->getConstructor(),
2168	/*ParamsToSkip*/ 0, Order);
2169
2170	EmitCXXConstructorCall(D, Type, ForVirtualBase, Delegating, This, Args,
2171	ThisAVS.mayOverlap(), E->getExprLoc(),
2172	ThisAVS.isSanitizerChecked());
2173	}
2174
2175	static bool canEmitDelegateCallArgs(CodeGenFunction &CGF,
2176	const CXXConstructorDecl *Ctor,
2177	CXXCtorType Type, CallArgList &Args) {
2178	// We can't forward a variadic call.
2179	if (Ctor->isVariadic())
2180	return false;
2181
2182	if (CGF.getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee()) {
2183	// If the parameters are callee-cleanup, it's not safe to forward.
2184	for (auto *P : Ctor->parameters())
2185	if (P->needsDestruction(CGF.getContext()))
2186	return false;
2187
2188	// Likewise if they're inalloca.
2189	const CGFunctionInfo &Info =
2190	CGF.CGM.getTypes().arrangeCXXConstructorCall(Args, D: Ctor, CtorKind: Type, ExtraPrefixArgs: 0, ExtraSuffixArgs: 0);
2191	if (Info.usesInAlloca())
2192	return false;
2193	}
2194
2195	// Anything else should be OK.
2196	return true;
2197	}
2198
2199	void CodeGenFunction::EmitCXXConstructorCall(
2200	const CXXConstructorDecl *D, CXXCtorType Type, bool ForVirtualBase,
2201	bool Delegating, Address This, CallArgList &Args,
2202	AggValueSlot::Overlap_t Overlap, SourceLocation Loc,
2203	bool NewPointerIsChecked, llvm::CallBase **CallOrInvoke) {
2204	const CXXRecordDecl *ClassDecl = D->getParent();
2205
2206	if (!NewPointerIsChecked)
2207	EmitTypeCheck(TCK: CodeGenFunction::TCK_ConstructorCall, Loc, Addr: This,
2208	Type: getContext().getRecordType(ClassDecl), Alignment: CharUnits::Zero());
2209
2210	if (D->isTrivial() && D->isDefaultConstructor()) {
2211	assert(Args.size() == 1 && "trivial default ctor with args");
2212	return;
2213	}
2214
2215	// If this is a trivial constructor, just emit what's needed. If this is a
2216	// union copy constructor, we must emit a memcpy, because the AST does not
2217	// model that copy.
2218	if (isMemcpyEquivalentSpecialMember(D)) {
2219	assert(Args.size() == 2 && "unexpected argcount for trivial ctor");
2220	QualType SrcTy = D->getParamDecl(0)->getType().getNonReferenceType();
2221	Address Src = makeNaturalAddressForPointer(
2222	Ptr: Args[1].getRValue(CGF&: *this).getScalarVal(), T: SrcTy);
2223	LValue SrcLVal = MakeAddrLValue(Addr: Src, T: SrcTy);
2224	QualType DestTy = getContext().getTypeDeclType(ClassDecl);
2225	LValue DestLVal = MakeAddrLValue(Addr: This, T: DestTy);
2226	EmitAggregateCopyCtor(Dest: DestLVal, Src: SrcLVal, MayOverlap: Overlap);
2227	return;
2228	}
2229
2230	bool PassPrototypeArgs = true;
2231	// Check whether we can actually emit the constructor before trying to do so.
2232	if (auto Inherited = D->getInheritedConstructor()) {
2233	PassPrototypeArgs = getTypes().inheritingCtorHasParams(Inherited, Type);
2234	if (PassPrototypeArgs && !canEmitDelegateCallArgs(CGF&: *this, Ctor: D, Type, Args)) {
2235	EmitInlinedInheritingCXXConstructorCall(Ctor: D, CtorType: Type, ForVirtualBase,
2236	Delegating, Args);
2237	return;
2238	}
2239	}
2240
2241	// Insert any ABI-specific implicit constructor arguments.
2242	CGCXXABI::AddedStructorArgCounts ExtraArgs =
2243	CGM.getCXXABI().addImplicitConstructorArgs(CGF&: *this, D, Type, ForVirtualBase,
2244	Delegating, Args);
2245
2246	// Emit the call.
2247	llvm::Constant *CalleePtr = CGM.getAddrOfCXXStructor(GD: GlobalDecl(D, Type));
2248	const CGFunctionInfo &Info = CGM.getTypes().arrangeCXXConstructorCall(
2249	Args, D, CtorKind: Type, ExtraPrefixArgs: ExtraArgs.Prefix, ExtraSuffixArgs: ExtraArgs.Suffix, PassProtoArgs: PassPrototypeArgs);
2250	CGCallee Callee = CGCallee::forDirect(functionPtr: CalleePtr, abstractInfo: GlobalDecl(D, Type));
2251	EmitCall(CallInfo: Info, Callee, ReturnValue: ReturnValueSlot(), Args, CallOrInvoke, IsMustTail: false, Loc);
2252
2253	// Generate vtable assumptions if we're constructing a complete object
2254	// with a vtable. We don't do this for base subobjects for two reasons:
2255	// first, it's incorrect for classes with virtual bases, and second, we're
2256	// about to overwrite the vptrs anyway.
2257	// We also have to make sure if we can refer to vtable:
2258	// - Otherwise we can refer to vtable if it's safe to speculatively emit.
2259	// FIXME: If vtable is used by ctor/dtor, or if vtable is external and we are
2260	// sure that definition of vtable is not hidden,
2261	// then we are always safe to refer to it.
2262	// FIXME: It looks like InstCombine is very inefficient on dealing with
2263	// assumes. Make assumption loads require -fstrict-vtable-pointers temporarily.
2264	if (CGM.getCodeGenOpts().OptimizationLevel > 0 &&
2265	ClassDecl->isDynamicClass() && Type != Ctor_Base &&
2266	CGM.getCXXABI().canSpeculativelyEmitVTable(RD: ClassDecl) &&
2267	CGM.getCodeGenOpts().StrictVTablePointers)
2268	EmitVTableAssumptionLoads(ClassDecl, This);
2269	}
2270
2271	void CodeGenFunction::EmitInheritedCXXConstructorCall(
2272	const CXXConstructorDecl *D, bool ForVirtualBase, Address This,
2273	bool InheritedFromVBase, const CXXInheritedCtorInitExpr *E) {
2274	CallArgList Args;
2275	CallArg ThisArg(RValue::get(V: getAsNaturalPointerTo(
2276	Addr: This, PointeeType: D->getThisType()->getPointeeType())),
2277	D->getThisType());
2278
2279	// Forward the parameters.
2280	if (InheritedFromVBase &&
2281	CGM.getTarget().getCXXABI().hasConstructorVariants()) {
2282	// Nothing to do; this construction is not responsible for constructing
2283	// the base class containing the inherited constructor.
2284	// FIXME: Can we just pass undef's for the remaining arguments if we don't
2285	// have constructor variants?
2286	Args.push_back(Elt: ThisArg);
2287	} else if (!CXXInheritedCtorInitExprArgs.empty()) {
2288	// The inheriting constructor was inlined; just inject its arguments.
2289	assert(CXXInheritedCtorInitExprArgs.size() >= D->getNumParams() &&
2290	"wrong number of parameters for inherited constructor call");
2291	Args = CXXInheritedCtorInitExprArgs;
2292	Args[0] = ThisArg;
2293	} else {
2294	// The inheriting constructor was not inlined. Emit delegating arguments.
2295	Args.push_back(Elt: ThisArg);
2296	const auto *OuterCtor = cast<CXXConstructorDecl>(Val: CurCodeDecl);
2297	assert(OuterCtor->getNumParams() == D->getNumParams());
2298	assert(!OuterCtor->isVariadic() && "should have been inlined");
2299
2300	for (const auto *Param : OuterCtor->parameters()) {
2301	assert(getContext().hasSameUnqualifiedType(
2302	OuterCtor->getParamDecl(Param->getFunctionScopeIndex())->getType(),
2303	Param->getType()));
2304	EmitDelegateCallArg(Args, Param, E->getLocation());
2305
2306	// Forward __attribute__(pass_object_size).
2307	if (Param->hasAttr<PassObjectSizeAttr>()) {
2308	auto *POSParam = SizeArguments[Param];
2309	assert(POSParam && "missing pass_object_size value for forwarding");
2310	EmitDelegateCallArg(Args, POSParam, E->getLocation());
2311	}
2312	}
2313	}
2314
2315	EmitCXXConstructorCall(D, Type: Ctor_Base, ForVirtualBase, /*Delegating*/false,
2316	This, Args, Overlap: AggValueSlot::MayOverlap,
2317	Loc: E->getLocation(), /*NewPointerIsChecked*/true);
2318	}
2319
2320	void CodeGenFunction::EmitInlinedInheritingCXXConstructorCall(
2321	const CXXConstructorDecl *Ctor, CXXCtorType CtorType, bool ForVirtualBase,
2322	bool Delegating, CallArgList &Args) {
2323	GlobalDecl GD(Ctor, CtorType);
2324	InlinedInheritingConstructorScope Scope(*this, GD);
2325	ApplyInlineDebugLocation DebugScope(*this, GD);
2326	RunCleanupsScope RunCleanups(*this);
2327
2328	// Save the arguments to be passed to the inherited constructor.
2329	CXXInheritedCtorInitExprArgs = Args;
2330
2331	FunctionArgList Params;
2332	QualType RetType = BuildFunctionArgList(GD: CurGD, Args&: Params);
2333	FnRetTy = RetType;
2334
2335	// Insert any ABI-specific implicit constructor arguments.
2336	CGM.getCXXABI().addImplicitConstructorArgs(CGF&: *this, D: Ctor, Type: CtorType,
2337	ForVirtualBase, Delegating, Args);
2338
2339	// Emit a simplified prolog. We only need to emit the implicit params.
2340	assert(Args.size() >= Params.size() && "too few arguments for call");
2341	for (unsigned I = 0, N = Args.size(); I != N; ++I) {
2342	if (I < Params.size() && isa<ImplicitParamDecl>(Val: Params[I])) {
2343	const RValue &RV = Args[I].getRValue(CGF&: *this);
2344	assert(!RV.isComplex() && "complex indirect params not supported");
2345	ParamValue Val = RV.isScalar()
2346	? ParamValue::forDirect(value: RV.getScalarVal())
2347	: ParamValue::forIndirect(addr: RV.getAggregateAddress());
2348	EmitParmDecl(D: *Params[I], Arg: Val, ArgNo: I + 1);
2349	}
2350	}
2351
2352	// Create a return value slot if the ABI implementation wants one.
2353	// FIXME: This is dumb, we should ask the ABI not to try to set the return
2354	// value instead.
2355	if (!RetType->isVoidType())
2356	ReturnValue = CreateIRTemp(T: RetType, Name: "retval.inhctor");
2357
2358	CGM.getCXXABI().EmitInstanceFunctionProlog(CGF&: *this);
2359	CXXThisValue = CXXABIThisValue;
2360
2361	// Directly emit the constructor initializers.
2362	EmitCtorPrologue(CD: Ctor, CtorType, Args&: Params);
2363	}
2364
2365	void CodeGenFunction::EmitVTableAssumptionLoad(const VPtr &Vptr, Address This) {
2366	llvm::Value *VTableGlobal =
2367	CGM.getCXXABI().getVTableAddressPoint(Base: Vptr.Base, VTableClass: Vptr.VTableClass);
2368	if (!VTableGlobal)
2369	return;
2370
2371	// We can just use the base offset in the complete class.
2372	CharUnits NonVirtualOffset = Vptr.Base.getBaseOffset();
2373
2374	if (!NonVirtualOffset.isZero())
2375	This =
2376	ApplyNonVirtualAndVirtualOffset(CGF&: *this, addr: This, nonVirtualOffset: NonVirtualOffset, virtualOffset: nullptr,
2377	derivedClass: Vptr.VTableClass, nearestVBase: Vptr.NearestVBase);
2378
2379	llvm::Value *VPtrValue =
2380	GetVTablePtr(This, VTableTy: VTableGlobal->getType(), VTableClass: Vptr.VTableClass);
2381	llvm::Value *Cmp =
2382	Builder.CreateICmpEQ(LHS: VPtrValue, RHS: VTableGlobal, Name: "cmp.vtables");
2383	Builder.CreateAssumption(Cond: Cmp);
2384	}
2385
2386	void CodeGenFunction::EmitVTableAssumptionLoads(const CXXRecordDecl *ClassDecl,
2387	Address This) {
2388	if (CGM.getCXXABI().doStructorsInitializeVPtrs(VTableClass: ClassDecl))
2389	for (const VPtr &Vptr : getVTablePointers(VTableClass: ClassDecl))
2390	EmitVTableAssumptionLoad(Vptr, This);
2391	}
2392
2393	void
2394	CodeGenFunction::EmitSynthesizedCXXCopyCtorCall(const CXXConstructorDecl *D,
2395	Address This, Address Src,
2396	const CXXConstructExpr *E) {
2397	const FunctionProtoType *FPT = D->getType()->castAs<FunctionProtoType>();
2398
2399	CallArgList Args;
2400
2401	// Push the this ptr.
2402	Args.add(rvalue: RValue::get(V: getAsNaturalPointerTo(Addr: This, PointeeType: D->getThisType())),
2403	type: D->getThisType());
2404
2405	// Push the src ptr.
2406	QualType QT = *(FPT->param_type_begin());
2407	llvm::Type *t = CGM.getTypes().ConvertType(T: QT);
2408	llvm::Value *Val = getAsNaturalPointerTo(Addr: Src, PointeeType: D->getThisType());
2409	llvm::Value *SrcVal = Builder.CreateBitCast(V: Val, DestTy: t);
2410	Args.add(rvalue: RValue::get(V: SrcVal), type: QT);
2411
2412	// Skip over first argument (Src).
2413	EmitCallArgs(Args, Prototype: FPT, ArgRange: drop_begin(E->arguments(), 1), AC: E->getConstructor(),
2414	/*ParamsToSkip*/ 1);
2415
2416	EmitCXXConstructorCall(D, Ctor_Complete, /*ForVirtualBase*/false,
2417	/*Delegating*/false, This, Args,
2418	AggValueSlot::MayOverlap, E->getExprLoc(),
2419	/*NewPointerIsChecked*/false);
2420	}
2421
2422	void
2423	CodeGenFunction::EmitDelegateCXXConstructorCall(const CXXConstructorDecl *Ctor,
2424	CXXCtorType CtorType,
2425	const FunctionArgList &Args,
2426	SourceLocation Loc) {
2427	CallArgList DelegateArgs;
2428
2429	FunctionArgList::const_iterator I = Args.begin(), E = Args.end();
2430	assert(I != E && "no parameters to constructor");
2431
2432	// this
2433	Address This = LoadCXXThisAddress();
2434	DelegateArgs.add(rvalue: RValue::get(getAsNaturalPointerTo(
2435	Addr: This, PointeeType: (*I)->getType()->getPointeeType())),
2436	type: (*I)->getType());
2437	++I;
2438
2439	// FIXME: The location of the VTT parameter in the parameter list is
2440	// specific to the Itanium ABI and shouldn't be hardcoded here.
2441	if (CGM.getCXXABI().NeedsVTTParameter(GD: CurGD)) {
2442	assert(I != E && "cannot skip vtt parameter, already done with args");
2443	assert((*I)->getType()->isPointerType() &&
2444	"skipping parameter not of vtt type");
2445	++I;
2446	}
2447
2448	// Explicit arguments.
2449	for (; I != E; ++I) {
2450	const VarDecl *param = *I;
2451	// FIXME: per-argument source location
2452	EmitDelegateCallArg(args&: DelegateArgs, param, loc: Loc);
2453	}
2454
2455	EmitCXXConstructorCall(D: Ctor, Type: CtorType, /*ForVirtualBase=*/false,
2456	/*Delegating=*/true, This, Args&: DelegateArgs,
2457	Overlap: AggValueSlot::MayOverlap, Loc,
2458	/*NewPointerIsChecked=*/true);
2459	}
2460
2461	namespace {
2462	struct CallDelegatingCtorDtor final : EHScopeStack::Cleanup {
2463	const CXXDestructorDecl *Dtor;
2464	Address Addr;
2465	CXXDtorType Type;
2466
2467	CallDelegatingCtorDtor(const CXXDestructorDecl *D, Address Addr,
2468	CXXDtorType Type)
2469	: Dtor(D), Addr(Addr), Type(Type) {}
2470
2471	void Emit(CodeGenFunction &CGF, Flags flags) override {
2472	// We are calling the destructor from within the constructor.
2473	// Therefore, "this" should have the expected type.
2474	QualType ThisTy = Dtor->getFunctionObjectParameterType();
2475	CGF.EmitCXXDestructorCall(D: Dtor, Type, /*ForVirtualBase=*/false,
2476	/*Delegating=*/true, This: Addr, ThisTy);
2477	}
2478	};
2479	} // end anonymous namespace
2480
2481	void
2482	CodeGenFunction::EmitDelegatingCXXConstructorCall(const CXXConstructorDecl *Ctor,
2483	const FunctionArgList &Args) {
2484	assert(Ctor->isDelegatingConstructor());
2485
2486	Address ThisPtr = LoadCXXThisAddress();
2487
2488	AggValueSlot AggSlot =
2489	AggValueSlot::forAddr(addr: ThisPtr, quals: Qualifiers(),
2490	isDestructed: AggValueSlot::IsDestructed,
2491	needsGC: AggValueSlot::DoesNotNeedGCBarriers,
2492	isAliased: AggValueSlot::IsNotAliased,
2493	mayOverlap: AggValueSlot::MayOverlap,
2494	isZeroed: AggValueSlot::IsNotZeroed,
2495	// Checks are made by the code that calls constructor.
2496	isChecked: AggValueSlot::IsSanitizerChecked);
2497
2498	EmitAggExpr(E: Ctor->init_begin()[0]->getInit(), AS: AggSlot);
2499
2500	const CXXRecordDecl *ClassDecl = Ctor->getParent();
2501	if (CGM.getLangOpts().Exceptions && !ClassDecl->hasTrivialDestructor()) {
2502	CXXDtorType Type =
2503	CurGD.getCtorType() == Ctor_Complete ? Dtor_Complete : Dtor_Base;
2504
2505	EHStack.pushCleanup<CallDelegatingCtorDtor>(Kind: EHCleanup,
2506	A: ClassDecl->getDestructor(),
2507	A: ThisPtr, A: Type);
2508	}
2509	}
2510
2511	void CodeGenFunction::EmitCXXDestructorCall(const CXXDestructorDecl *DD,
2512	CXXDtorType Type,
2513	bool ForVirtualBase,
2514	bool Delegating, Address This,
2515	QualType ThisTy) {
2516	CGM.getCXXABI().EmitDestructorCall(CGF&: *this, DD, Type, ForVirtualBase,
2517	Delegating, This, ThisTy);
2518	}
2519
2520	namespace {
2521	struct CallLocalDtor final : EHScopeStack::Cleanup {
2522	const CXXDestructorDecl *Dtor;
2523	Address Addr;
2524	QualType Ty;
2525
2526	CallLocalDtor(const CXXDestructorDecl *D, Address Addr, QualType Ty)
2527	: Dtor(D), Addr(Addr), Ty(Ty) {}
2528
2529	void Emit(CodeGenFunction &CGF, Flags flags) override {
2530	CGF.EmitCXXDestructorCall(Dtor, Dtor_Complete,
2531	/*ForVirtualBase=*/false,
2532	/*Delegating=*/false, Addr, Ty);
2533	}
2534	};
2535	} // end anonymous namespace
2536
2537	void CodeGenFunction::PushDestructorCleanup(const CXXDestructorDecl *D,
2538	QualType T, Address Addr) {
2539	EHStack.pushCleanup<CallLocalDtor>(Kind: NormalAndEHCleanup, A: D, A: Addr, A: T);
2540	}
2541
2542	void CodeGenFunction::PushDestructorCleanup(QualType T, Address Addr) {
2543	CXXRecordDecl *ClassDecl = T->getAsCXXRecordDecl();
2544	if (!ClassDecl) return;
2545	if (ClassDecl->hasTrivialDestructor()) return;
2546
2547	const CXXDestructorDecl *D = ClassDecl->getDestructor();
2548	assert(D && D->isUsed() && "destructor not marked as used!");
2549	PushDestructorCleanup(D, T, Addr);
2550	}
2551
2552	void CodeGenFunction::InitializeVTablePointer(const VPtr &Vptr) {
2553	// Compute the address point.
2554	llvm::Value *VTableAddressPoint =
2555	CGM.getCXXABI().getVTableAddressPointInStructor(
2556	CGF&: *this, RD: Vptr.VTableClass, Base: Vptr.Base, NearestVBase: Vptr.NearestVBase);
2557
2558	if (!VTableAddressPoint)
2559	return;
2560
2561	// Compute where to store the address point.
2562	llvm::Value *VirtualOffset = nullptr;
2563	CharUnits NonVirtualOffset = CharUnits::Zero();
2564
2565	if (CGM.getCXXABI().isVirtualOffsetNeededForVTableField(CGF&: *this, Vptr)) {
2566	// We need to use the virtual base offset offset because the virtual base
2567	// might have a different offset in the most derived class.
2568
2569	VirtualOffset = CGM.getCXXABI().GetVirtualBaseClassOffset(
2570	CGF&: *this, This: LoadCXXThisAddress(), ClassDecl: Vptr.VTableClass, BaseClassDecl: Vptr.NearestVBase);
2571	NonVirtualOffset = Vptr.OffsetFromNearestVBase;
2572	} else {
2573	// We can just use the base offset in the complete class.
2574	NonVirtualOffset = Vptr.Base.getBaseOffset();
2575	}
2576
2577	// Apply the offsets.
2578	Address VTableField = LoadCXXThisAddress();
2579	if (!NonVirtualOffset.isZero() || VirtualOffset)
2580	VTableField = ApplyNonVirtualAndVirtualOffset(
2581	CGF&: *this, addr: VTableField, nonVirtualOffset: NonVirtualOffset, virtualOffset: VirtualOffset, derivedClass: Vptr.VTableClass,
2582	nearestVBase: Vptr.NearestVBase);
2583
2584	// Finally, store the address point. Use the same LLVM types as the field to
2585	// support optimization.
2586	unsigned GlobalsAS = CGM.getDataLayout().getDefaultGlobalsAddressSpace();
2587	llvm::Type *PtrTy = llvm::PointerType::get(C&: CGM.getLLVMContext(), AddressSpace: GlobalsAS);
2588	// vtable field is derived from `this` pointer, therefore they should be in
2589	// the same addr space. Note that this might not be LLVM address space 0.
2590	VTableField = VTableField.withElementType(ElemTy: PtrTy);
2591
2592	if (auto AuthenticationInfo = CGM.getVTablePointerAuthInfo(
2593	Context: this, Record: Vptr.Base.getBase(), StorageAddress: VTableField.emitRawPointer(CGF&: *this)))
2594	VTableAddressPoint =
2595	EmitPointerAuthSign(Info: *AuthenticationInfo, Pointer: VTableAddressPoint);
2596
2597	llvm::StoreInst *Store = Builder.CreateStore(Val: VTableAddressPoint, Addr: VTableField);
2598	TBAAAccessInfo TBAAInfo = CGM.getTBAAVTablePtrAccessInfo(VTablePtrType: PtrTy);
2599	CGM.DecorateInstructionWithTBAA(Inst: Store, TBAAInfo);
2600	if (CGM.getCodeGenOpts().OptimizationLevel > 0 &&
2601	CGM.getCodeGenOpts().StrictVTablePointers)
2602	CGM.DecorateInstructionWithInvariantGroup(I: Store, RD: Vptr.VTableClass);
2603	}
2604
2605	CodeGenFunction::VPtrsVector
2606	CodeGenFunction::getVTablePointers(const CXXRecordDecl *VTableClass) {
2607	CodeGenFunction::VPtrsVector VPtrsResult;
2608	VisitedVirtualBasesSetTy VBases;
2609	getVTablePointers(Base: BaseSubobject(VTableClass, CharUnits::Zero()),
2610	/*NearestVBase=*/nullptr,
2611	/*OffsetFromNearestVBase=*/CharUnits::Zero(),
2612	/*BaseIsNonVirtualPrimaryBase=*/false, VTableClass, VBases,
2613	vptrs&: VPtrsResult);
2614	return VPtrsResult;
2615	}
2616
2617	void CodeGenFunction::getVTablePointers(BaseSubobject Base,
2618	const CXXRecordDecl *NearestVBase,
2619	CharUnits OffsetFromNearestVBase,
2620	bool BaseIsNonVirtualPrimaryBase,
2621	const CXXRecordDecl *VTableClass,
2622	VisitedVirtualBasesSetTy &VBases,
2623	VPtrsVector &Vptrs) {
2624	// If this base is a non-virtual primary base the address point has already
2625	// been set.
2626	if (!BaseIsNonVirtualPrimaryBase) {
2627	// Initialize the vtable pointer for this base.
2628	VPtr Vptr = {.Base: Base, .NearestVBase: NearestVBase, .OffsetFromNearestVBase: OffsetFromNearestVBase, .VTableClass: VTableClass};
2629	Vptrs.push_back(Elt: Vptr);
2630	}
2631
2632	const CXXRecordDecl *RD = Base.getBase();
2633
2634	// Traverse bases.
2635	for (const auto &I : RD->bases()) {
2636	auto *BaseDecl =
2637	cast<CXXRecordDecl>(Val: I.getType()->castAs<RecordType>()->getDecl());
2638
2639	// Ignore classes without a vtable.
2640	if (!BaseDecl->isDynamicClass())
2641	continue;
2642
2643	CharUnits BaseOffset;
2644	CharUnits BaseOffsetFromNearestVBase;
2645	bool BaseDeclIsNonVirtualPrimaryBase;
2646
2647	if (I.isVirtual()) {
2648	// Check if we've visited this virtual base before.
2649	if (!VBases.insert(Ptr: BaseDecl).second)
2650	continue;
2651
2652	const ASTRecordLayout &Layout =
2653	getContext().getASTRecordLayout(VTableClass);
2654
2655	BaseOffset = Layout.getVBaseClassOffset(VBase: BaseDecl);
2656	BaseOffsetFromNearestVBase = CharUnits::Zero();
2657	BaseDeclIsNonVirtualPrimaryBase = false;
2658	} else {
2659	const ASTRecordLayout &Layout = getContext().getASTRecordLayout(RD);
2660
2661	BaseOffset = Base.getBaseOffset() + Layout.getBaseClassOffset(Base: BaseDecl);
2662	BaseOffsetFromNearestVBase =
2663	OffsetFromNearestVBase + Layout.getBaseClassOffset(Base: BaseDecl);
2664	BaseDeclIsNonVirtualPrimaryBase = Layout.getPrimaryBase() == BaseDecl;
2665	}
2666
2667	getVTablePointers(
2668	Base: BaseSubobject(BaseDecl, BaseOffset),
2669	NearestVBase: I.isVirtual() ? BaseDecl : NearestVBase, OffsetFromNearestVBase: BaseOffsetFromNearestVBase,
2670	BaseIsNonVirtualPrimaryBase: BaseDeclIsNonVirtualPrimaryBase, VTableClass, VBases, Vptrs);
2671	}
2672	}
2673
2674	void CodeGenFunction::InitializeVTablePointers(const CXXRecordDecl *RD) {
2675	// Ignore classes without a vtable.
2676	if (!RD->isDynamicClass())
2677	return;
2678
2679	// Initialize the vtable pointers for this class and all of its bases.
2680	if (CGM.getCXXABI().doStructorsInitializeVPtrs(VTableClass: RD))
2681	for (const VPtr &Vptr : getVTablePointers(VTableClass: RD))
2682	InitializeVTablePointer(Vptr);
2683
2684	if (RD->getNumVBases())
2685	CGM.getCXXABI().initializeHiddenVirtualInheritanceMembers(CGF&: *this, RD);
2686	}
2687
2688	llvm::Value *CodeGenFunction::GetVTablePtr(Address This,
2689	llvm::Type *VTableTy,
2690	const CXXRecordDecl *RD,
2691	VTableAuthMode AuthMode) {
2692	Address VTablePtrSrc = This.withElementType(ElemTy: VTableTy);
2693	llvm::Instruction *VTable = Builder.CreateLoad(Addr: VTablePtrSrc, Name: "vtable");
2694	TBAAAccessInfo TBAAInfo = CGM.getTBAAVTablePtrAccessInfo(VTablePtrType: VTableTy);
2695	CGM.DecorateInstructionWithTBAA(Inst: VTable, TBAAInfo);
2696
2697	if (auto AuthenticationInfo =
2698	CGM.getVTablePointerAuthInfo(Context: this, Record: RD, StorageAddress: This.emitRawPointer(CGF&: *this))) {
2699	if (AuthMode != VTableAuthMode::UnsafeUbsanStrip) {
2700	VTable = cast<llvm::Instruction>(
2701	Val: EmitPointerAuthAuth(Info: *AuthenticationInfo, Pointer: VTable));
2702	if (AuthMode == VTableAuthMode::MustTrap) {
2703	// This is clearly suboptimal but until we have an ability
2704	// to rely on the authentication intrinsic trapping and force
2705	// an authentication to occur we don't really have a choice.
2706	VTable =
2707	cast<llvm::Instruction>(Val: Builder.CreateBitCast(V: VTable, DestTy: Int8PtrTy));
2708	Builder.CreateLoad(Addr: RawAddress(VTable, Int8Ty, CGM.getPointerAlign()),
2709	/* IsVolatile */ true);
2710	}
2711	} else {
2712	VTable = cast<llvm::Instruction>(Val: EmitPointerAuthAuth(
2713	Info: CGPointerAuthInfo(0, PointerAuthenticationMode::Strip, false, false,
2714	nullptr),
2715	Pointer: VTable));
2716	}
2717	}
2718
2719	if (CGM.getCodeGenOpts().OptimizationLevel > 0 &&
2720	CGM.getCodeGenOpts().StrictVTablePointers)
2721	CGM.DecorateInstructionWithInvariantGroup(I: VTable, RD);
2722
2723	return VTable;
2724	}
2725
2726	// If a class has a single non-virtual base and does not introduce or override
2727	// virtual member functions or fields, it will have the same layout as its base.
2728	// This function returns the least derived such class.
2729	//
2730	// Casting an instance of a base class to such a derived class is technically
2731	// undefined behavior, but it is a relatively common hack for introducing member
2732	// functions on class instances with specific properties (e.g. llvm::Operator)
2733	// that works under most compilers and should not have security implications, so
2734	// we allow it by default. It can be disabled with -fsanitize=cfi-cast-strict.
2735	static const CXXRecordDecl *
2736	LeastDerivedClassWithSameLayout(const CXXRecordDecl *RD) {
2737	if (!RD->field_empty())
2738	return RD;
2739
2740	if (RD->getNumVBases() != 0)
2741	return RD;
2742
2743	if (RD->getNumBases() != 1)
2744	return RD;
2745
2746	for (const CXXMethodDecl *MD : RD->methods()) {
2747	if (MD->isVirtual()) {
2748	// Virtual member functions are only ok if they are implicit destructors
2749	// because the implicit destructor will have the same semantics as the
2750	// base class's destructor if no fields are added.
2751	if (isa<CXXDestructorDecl>(Val: MD) && MD->isImplicit())
2752	continue;
2753	return RD;
2754	}
2755	}
2756
2757	return LeastDerivedClassWithSameLayout(
2758	RD: RD->bases_begin()->getType()->getAsCXXRecordDecl());
2759	}
2760
2761	void CodeGenFunction::EmitTypeMetadataCodeForVCall(const CXXRecordDecl *RD,
2762	llvm::Value *VTable,
2763	SourceLocation Loc) {
2764	if (SanOpts.has(K: SanitizerKind::CFIVCall))
2765	EmitVTablePtrCheckForCall(RD, VTable, TCK: CodeGenFunction::CFITCK_VCall, Loc);
2766	else if (CGM.getCodeGenOpts().WholeProgramVTables &&
2767	// Don't insert type test assumes if we are forcing public
2768	// visibility.
2769	!CGM.AlwaysHasLTOVisibilityPublic(RD)) {
2770	QualType Ty = QualType(RD->getTypeForDecl(), 0);
2771	llvm::Metadata *MD = CGM.CreateMetadataIdentifierForType(T: Ty);
2772	llvm::Value *TypeId =
2773	llvm::MetadataAsValue::get(Context&: CGM.getLLVMContext(), MD);
2774
2775	// If we already know that the call has hidden LTO visibility, emit
2776	// @llvm.type.test(). Otherwise emit @llvm.public.type.test(), which WPD
2777	// will convert to @llvm.type.test() if we assert at link time that we have
2778	// whole program visibility.
2779	llvm::Intrinsic::ID IID = CGM.HasHiddenLTOVisibility(RD)
2780	? llvm::Intrinsic::type_test
2781	: llvm::Intrinsic::public_type_test;
2782	llvm::Value *TypeTest =
2783	Builder.CreateCall(Callee: CGM.getIntrinsic(IID), Args: {VTable, TypeId});
2784	Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::assume), TypeTest);
2785	}
2786	}
2787
2788	/// Converts the CFITypeCheckKind into SanitizerKind::SanitizerOrdinal and
2789	/// llvm::SanitizerStatKind.
2790	static std::pair<SanitizerKind::SanitizerOrdinal, llvm::SanitizerStatKind>
2791	SanitizerInfoFromCFICheckKind(CodeGenFunction::CFITypeCheckKind TCK) {
2792	switch (TCK) {
2793	case CodeGenFunction::CFITCK_VCall:
2794	return std::make_pair(x: SanitizerKind::SO_CFIVCall, y: llvm::SanStat_CFI_VCall);
2795	case CodeGenFunction::CFITCK_NVCall:
2796	return std::make_pair(x: SanitizerKind::SO_CFINVCall,
2797	y: llvm::SanStat_CFI_NVCall);
2798	case CodeGenFunction::CFITCK_DerivedCast:
2799	return std::make_pair(x: SanitizerKind::SO_CFIDerivedCast,
2800	y: llvm::SanStat_CFI_DerivedCast);
2801	case CodeGenFunction::CFITCK_UnrelatedCast:
2802	return std::make_pair(x: SanitizerKind::SO_CFIUnrelatedCast,
2803	y: llvm::SanStat_CFI_UnrelatedCast);
2804	case CodeGenFunction::CFITCK_ICall:
2805	case CodeGenFunction::CFITCK_NVMFCall:
2806	case CodeGenFunction::CFITCK_VMFCall:
2807	llvm_unreachable("unexpected sanitizer kind");
2808	}
2809	llvm_unreachable("Unknown CFITypeCheckKind enum");
2810	}
2811
2812	void CodeGenFunction::EmitVTablePtrCheckForCall(const CXXRecordDecl *RD,
2813	llvm::Value *VTable,
2814	CFITypeCheckKind TCK,
2815	SourceLocation Loc) {
2816	if (!SanOpts.has(K: SanitizerKind::CFICastStrict))
2817	RD = LeastDerivedClassWithSameLayout(RD);
2818
2819	auto [Ordinal, _] = SanitizerInfoFromCFICheckKind(TCK);
2820	SanitizerDebugLocation SanScope(this, {Ordinal},
2821	SanitizerHandler::CFICheckFail);
2822
2823	EmitVTablePtrCheck(RD, VTable, TCK, Loc);
2824	}
2825
2826	void CodeGenFunction::EmitVTablePtrCheckForCast(QualType T, Address Derived,
2827	bool MayBeNull,
2828	CFITypeCheckKind TCK,
2829	SourceLocation Loc) {
2830	if (!getLangOpts().CPlusPlus)
2831	return;
2832
2833	auto *ClassTy = T->getAs<RecordType>();
2834	if (!ClassTy)
2835	return;
2836
2837	const CXXRecordDecl *ClassDecl = cast<CXXRecordDecl>(Val: ClassTy->getDecl());
2838
2839	if (!ClassDecl->isCompleteDefinition() || !ClassDecl->isDynamicClass())
2840	return;
2841
2842	if (!SanOpts.has(K: SanitizerKind::CFICastStrict))
2843	ClassDecl = LeastDerivedClassWithSameLayout(RD: ClassDecl);
2844
2845	auto [Ordinal, _] = SanitizerInfoFromCFICheckKind(TCK);
2846	SanitizerDebugLocation SanScope(this, {Ordinal},
2847	SanitizerHandler::CFICheckFail);
2848
2849	llvm::BasicBlock *ContBlock = nullptr;
2850
2851	if (MayBeNull) {
2852	llvm::Value *DerivedNotNull =
2853	Builder.CreateIsNotNull(Arg: Derived.emitRawPointer(CGF&: *this), Name: "cast.nonnull");
2854
2855	llvm::BasicBlock *CheckBlock = createBasicBlock(name: "cast.check");
2856	ContBlock = createBasicBlock(name: "cast.cont");
2857
2858	Builder.CreateCondBr(Cond: DerivedNotNull, True: CheckBlock, False: ContBlock);
2859
2860	EmitBlock(BB: CheckBlock);
2861	}
2862
2863	llvm::Value *VTable;
2864	std::tie(args&: VTable, args&: ClassDecl) =
2865	CGM.getCXXABI().LoadVTablePtr(CGF&: *this, This: Derived, RD: ClassDecl);
2866
2867	EmitVTablePtrCheck(RD: ClassDecl, VTable, TCK, Loc);
2868
2869	if (MayBeNull) {
2870	Builder.CreateBr(Dest: ContBlock);
2871	EmitBlock(BB: ContBlock);
2872	}
2873	}
2874
2875	void CodeGenFunction::EmitVTablePtrCheck(const CXXRecordDecl *RD,
2876	llvm::Value *VTable,
2877	CFITypeCheckKind TCK,
2878	SourceLocation Loc) {
2879	assert(IsSanitizerScope);
2880
2881	if (!CGM.getCodeGenOpts().SanitizeCfiCrossDso &&
2882	!CGM.HasHiddenLTOVisibility(RD))
2883	return;
2884
2885	auto [M, SSK] = SanitizerInfoFromCFICheckKind(TCK);
2886
2887	std::string TypeName = RD->getQualifiedNameAsString();
2888	if (getContext().getNoSanitizeList().containsType(
2889	Mask: SanitizerMask::bitPosToMask(Pos: M), MangledTypeName: TypeName))
2890	return;
2891
2892	EmitSanitizerStatReport(SSK);
2893
2894	llvm::Metadata *MD =
2895	CGM.CreateMetadataIdentifierForType(T: QualType(RD->getTypeForDecl(), 0));
2896	llvm::Value *TypeId = llvm::MetadataAsValue::get(Context&: getLLVMContext(), MD);
2897
2898	llvm::Value *TypeTest = Builder.CreateCall(
2899	CGM.getIntrinsic(llvm::Intrinsic::type_test), {VTable, TypeId});
2900
2901	llvm::Constant *StaticData[] = {
2902	llvm::ConstantInt::get(Ty: Int8Ty, V: TCK),
2903	EmitCheckSourceLocation(Loc),
2904	EmitCheckTypeDescriptor(T: QualType(RD->getTypeForDecl(), 0)),
2905	};
2906
2907	auto CrossDsoTypeId = CGM.CreateCrossDsoCfiTypeId(MD);
2908	if (CGM.getCodeGenOpts().SanitizeCfiCrossDso && CrossDsoTypeId) {
2909	EmitCfiSlowPathCheck(Ordinal: M, Cond: TypeTest, TypeId: CrossDsoTypeId, Ptr: VTable, StaticArgs: StaticData);
2910	return;
2911	}
2912
2913	if (CGM.getCodeGenOpts().SanitizeTrap.has(O: M)) {
2914	bool NoMerge = !CGM.getCodeGenOpts().SanitizeMergeHandlers.has(O: M);
2915	EmitTrapCheck(Checked: TypeTest, CheckHandlerID: SanitizerHandler::CFICheckFail, NoMerge);
2916	return;
2917	}
2918
2919	llvm::Value *AllVtables = llvm::MetadataAsValue::get(
2920	Context&: CGM.getLLVMContext(),
2921	MD: llvm::MDString::get(Context&: CGM.getLLVMContext(), Str: "all-vtables"));
2922	llvm::Value *ValidVtable = Builder.CreateCall(
2923	CGM.getIntrinsic(llvm::Intrinsic::type_test), {VTable, AllVtables});
2924	EmitCheck(Checked: std::make_pair(x&: TypeTest, y&: M), Check: SanitizerHandler::CFICheckFail,
2925	StaticArgs: StaticData, DynamicArgs: {VTable, ValidVtable});
2926	}
2927
2928	bool CodeGenFunction::ShouldEmitVTableTypeCheckedLoad(const CXXRecordDecl *RD) {
2929	if (!CGM.getCodeGenOpts().WholeProgramVTables ||
2930	!CGM.HasHiddenLTOVisibility(RD))
2931	return false;
2932
2933	if (CGM.getCodeGenOpts().VirtualFunctionElimination)
2934	return true;
2935
2936	if (!SanOpts.has(K: SanitizerKind::CFIVCall) ||
2937	!CGM.getCodeGenOpts().SanitizeTrap.has(K: SanitizerKind::CFIVCall))
2938	return false;
2939
2940	std::string TypeName = RD->getQualifiedNameAsString();
2941	return !getContext().getNoSanitizeList().containsType(Mask: SanitizerKind::CFIVCall,
2942	MangledTypeName: TypeName);
2943	}
2944
2945	llvm::Value *CodeGenFunction::EmitVTableTypeCheckedLoad(
2946	const CXXRecordDecl *RD, llvm::Value *VTable, llvm::Type *VTableTy,
2947	uint64_t VTableByteOffset) {
2948	auto CheckOrdinal = SanitizerKind::SO_CFIVCall;
2949	auto CheckHandler = SanitizerHandler::CFICheckFail;
2950	SanitizerDebugLocation SanScope(this, {CheckOrdinal}, CheckHandler);
2951
2952	EmitSanitizerStatReport(SSK: llvm::SanStat_CFI_VCall);
2953
2954	llvm::Metadata *MD =
2955	CGM.CreateMetadataIdentifierForType(T: QualType(RD->getTypeForDecl(), 0));
2956	llvm::Value *TypeId = llvm::MetadataAsValue::get(Context&: CGM.getLLVMContext(), MD);
2957
2958	auto CheckedLoadIntrinsic = CGM.getVTables().useRelativeLayout()
2959	? llvm::Intrinsic::type_checked_load_relative
2960	: llvm::Intrinsic::type_checked_load;
2961	llvm::Value *CheckedLoad = Builder.CreateCall(
2962	CGM.getIntrinsic(IID: CheckedLoadIntrinsic),
2963	{VTable, llvm::ConstantInt::get(Ty: Int32Ty, V: VTableByteOffset), TypeId});
2964
2965	llvm::Value *CheckResult = Builder.CreateExtractValue(Agg: CheckedLoad, Idxs: 1);
2966
2967	std::string TypeName = RD->getQualifiedNameAsString();
2968	if (SanOpts.has(K: SanitizerKind::CFIVCall) &&
2969	!getContext().getNoSanitizeList().containsType(Mask: SanitizerKind::CFIVCall,
2970	MangledTypeName: TypeName)) {
2971	EmitCheck(Checked: std::make_pair(x&: CheckResult, y&: CheckOrdinal), Check: CheckHandler, StaticArgs: {}, DynamicArgs: {});
2972	}
2973
2974	return Builder.CreateBitCast(V: Builder.CreateExtractValue(Agg: CheckedLoad, Idxs: 0),
2975	DestTy: VTableTy);
2976	}
2977
2978	void CodeGenFunction::EmitForwardingCallToLambda(
2979	const CXXMethodDecl *callOperator, CallArgList &callArgs,
2980	const CGFunctionInfo *calleeFnInfo, llvm::Constant *calleePtr) {
2981	// Get the address of the call operator.
2982	if (!calleeFnInfo)
2983	calleeFnInfo = &CGM.getTypes().arrangeCXXMethodDeclaration(MD: callOperator);
2984
2985	if (!calleePtr)
2986	calleePtr =
2987	CGM.GetAddrOfFunction(GD: GlobalDecl(callOperator),
2988	Ty: CGM.getTypes().GetFunctionType(Info: *calleeFnInfo));
2989
2990	// Prepare the return slot.
2991	const FunctionProtoType *FPT =
2992	callOperator->getType()->castAs<FunctionProtoType>();
2993	QualType resultType = FPT->getReturnType();
2994	ReturnValueSlot returnSlot;
2995	if (!resultType->isVoidType() &&
2996	calleeFnInfo->getReturnInfo().getKind() == ABIArgInfo::Indirect &&
2997	!hasScalarEvaluationKind(T: calleeFnInfo->getReturnType()))
2998	returnSlot =
2999	ReturnValueSlot(ReturnValue, resultType.isVolatileQualified(),
3000	/*IsUnused=*/false, /*IsExternallyDestructed=*/true);
3001
3002	// We don't need to separately arrange the call arguments because
3003	// the call can't be variadic anyway --- it's impossible to forward
3004	// variadic arguments.
3005
3006	// Now emit our call.
3007	auto callee = CGCallee::forDirect(functionPtr: calleePtr, abstractInfo: GlobalDecl(callOperator));
3008	RValue RV = EmitCall(*calleeFnInfo, callee, returnSlot, callArgs);
3009
3010	// If necessary, copy the returned value into the slot.
3011	if (!resultType->isVoidType() && returnSlot.isNull()) {
3012	if (getLangOpts().ObjCAutoRefCount && resultType->isObjCRetainableType()) {
3013	RV = RValue::get(V: EmitARCRetainAutoreleasedReturnValue(value: RV.getScalarVal()));
3014	}
3015	EmitReturnOfRValue(RV, Ty: resultType);
3016	} else
3017	EmitBranchThroughCleanup(Dest: ReturnBlock);
3018	}
3019
3020	void CodeGenFunction::EmitLambdaBlockInvokeBody() {
3021	const BlockDecl *BD = BlockInfo->getBlockDecl();
3022	const VarDecl *variable = BD->capture_begin()->getVariable();
3023	const CXXRecordDecl *Lambda = variable->getType()->getAsCXXRecordDecl();
3024	const CXXMethodDecl *CallOp = Lambda->getLambdaCallOperator();
3025
3026	if (CallOp->isVariadic()) {
3027	// FIXME: Making this work correctly is nasty because it requires either
3028	// cloning the body of the call operator or making the call operator
3029	// forward.
3030	CGM.ErrorUnsupported(D: CurCodeDecl, Type: "lambda conversion to variadic function");
3031	return;
3032	}
3033
3034	// Start building arguments for forwarding call
3035	CallArgList CallArgs;
3036
3037	QualType ThisType = getContext().getPointerType(T: getContext().getRecordType(Lambda));
3038	Address ThisPtr = GetAddrOfBlockDecl(var: variable);
3039	CallArgs.add(rvalue: RValue::get(V: getAsNaturalPointerTo(Addr: ThisPtr, PointeeType: ThisType)), type: ThisType);
3040
3041	// Add the rest of the parameters.
3042	for (auto *param : BD->parameters())
3043	EmitDelegateCallArg(args&: CallArgs, param, loc: param->getBeginLoc());
3044
3045	assert(!Lambda->isGenericLambda() &&
3046	"generic lambda interconversion to block not implemented");
3047	EmitForwardingCallToLambda(callOperator: CallOp, callArgs&: CallArgs);
3048	}
3049
3050	void CodeGenFunction::EmitLambdaStaticInvokeBody(const CXXMethodDecl *MD) {
3051	if (MD->isVariadic()) {
3052	// FIXME: Making this work correctly is nasty because it requires either
3053	// cloning the body of the call operator or making the call operator
3054	// forward.
3055	CGM.ErrorUnsupported(MD, "lambda conversion to variadic function");
3056	return;
3057	}
3058
3059	const CXXRecordDecl *Lambda = MD->getParent();
3060
3061	// Start building arguments for forwarding call
3062	CallArgList CallArgs;
3063
3064	QualType LambdaType = getContext().getRecordType(Lambda);
3065	QualType ThisType = getContext().getPointerType(T: LambdaType);
3066	Address ThisPtr = CreateMemTemp(T: LambdaType, Name: "unused.capture");
3067	CallArgs.add(rvalue: RValue::get(V: ThisPtr.emitRawPointer(CGF&: *this)), type: ThisType);
3068
3069	EmitLambdaDelegatingInvokeBody(MD, CallArgs);
3070	}
3071
3072	void CodeGenFunction::EmitLambdaDelegatingInvokeBody(const CXXMethodDecl *MD,
3073	CallArgList &CallArgs) {
3074	// Add the rest of the forwarded parameters.
3075	for (auto *Param : MD->parameters())
3076	EmitDelegateCallArg(CallArgs, Param, Param->getBeginLoc());
3077
3078	const CXXRecordDecl *Lambda = MD->getParent();
3079	const CXXMethodDecl *CallOp = Lambda->getLambdaCallOperator();
3080	// For a generic lambda, find the corresponding call operator specialization
3081	// to which the call to the static-invoker shall be forwarded.
3082	if (Lambda->isGenericLambda()) {
3083	assert(MD->isFunctionTemplateSpecialization());
3084	const TemplateArgumentList *TAL = MD->getTemplateSpecializationArgs();
3085	FunctionTemplateDecl *CallOpTemplate = CallOp->getDescribedFunctionTemplate();
3086	void *InsertPos = nullptr;
3087	FunctionDecl *CorrespondingCallOpSpecialization =
3088	CallOpTemplate->findSpecialization(Args: TAL->asArray(), InsertPos);
3089	assert(CorrespondingCallOpSpecialization);
3090	CallOp = cast<CXXMethodDecl>(Val: CorrespondingCallOpSpecialization);
3091	}
3092
3093	// Special lambda forwarding when there are inalloca parameters.
3094	if (hasInAllocaArg(MD)) {
3095	const CGFunctionInfo *ImplFnInfo = nullptr;
3096	llvm::Function *ImplFn = nullptr;
3097	EmitLambdaInAllocaImplFn(CallOp, ImplFnInfo: &ImplFnInfo, ImplFn: &ImplFn);
3098
3099	EmitForwardingCallToLambda(callOperator: CallOp, callArgs&: CallArgs, calleeFnInfo: ImplFnInfo, calleePtr: ImplFn);
3100	return;
3101	}
3102
3103	EmitForwardingCallToLambda(callOperator: CallOp, callArgs&: CallArgs);
3104	}
3105
3106	void CodeGenFunction::EmitLambdaInAllocaCallOpBody(const CXXMethodDecl *MD) {
3107	if (MD->isVariadic()) {
3108	// FIXME: Making this work correctly is nasty because it requires either
3109	// cloning the body of the call operator or making the call operator forward.
3110	CGM.ErrorUnsupported(MD, "lambda conversion to variadic function");
3111	return;
3112	}
3113
3114	// Forward %this argument.
3115	CallArgList CallArgs;
3116	QualType LambdaType = getContext().getRecordType(MD->getParent());
3117	QualType ThisType = getContext().getPointerType(T: LambdaType);
3118	llvm::Value *ThisArg = CurFn->getArg(i: 0);
3119	CallArgs.add(rvalue: RValue::get(V: ThisArg), type: ThisType);
3120
3121	EmitLambdaDelegatingInvokeBody(MD, CallArgs);
3122	}
3123
3124	void CodeGenFunction::EmitLambdaInAllocaImplFn(
3125	const CXXMethodDecl *CallOp, const CGFunctionInfo **ImplFnInfo,
3126	llvm::Function **ImplFn) {
3127	const CGFunctionInfo &FnInfo =
3128	CGM.getTypes().arrangeCXXMethodDeclaration(MD: CallOp);
3129	llvm::Function *CallOpFn =
3130	cast<llvm::Function>(Val: CGM.GetAddrOfFunction(GD: GlobalDecl(CallOp)));
3131
3132	// Emit function containing the original call op body. __invoke will delegate
3133	// to this function.
3134	SmallVector<CanQualType, 4> ArgTypes;
3135	for (auto I = FnInfo.arg_begin(); I != FnInfo.arg_end(); ++I)
3136	ArgTypes.push_back(Elt: I->type);
3137	*ImplFnInfo = &CGM.getTypes().arrangeLLVMFunctionInfo(
3138	returnType: FnInfo.getReturnType(), opts: FnInfoOpts::IsDelegateCall, argTypes: ArgTypes,
3139	info: FnInfo.getExtInfo(), paramInfos: {}, args: FnInfo.getRequiredArgs());
3140
3141	// Create mangled name as if this was a method named __impl. If for some
3142	// reason the name doesn't look as expected then just tack __impl to the
3143	// front.
3144	// TODO: Use the name mangler to produce the right name instead of using
3145	// string replacement.
3146	StringRef CallOpName = CallOpFn->getName();
3147	std::string ImplName;
3148	if (size_t Pos = CallOpName.find_first_of(Chars: "<lambda"))
3149	ImplName = ("?__impl@" + CallOpName.drop_front(N: Pos)).str();
3150	else
3151	ImplName = ("__impl" + CallOpName).str();
3152
3153	llvm::Function *Fn = CallOpFn->getParent()->getFunction(Name: ImplName);
3154	if (!Fn) {
3155	Fn = llvm::Function::Create(Ty: CGM.getTypes().GetFunctionType(Info: **ImplFnInfo),
3156	Linkage: llvm::GlobalValue::InternalLinkage, N: ImplName,
3157	M&: CGM.getModule());
3158	CGM.SetInternalFunctionAttributes(CallOp, Fn, **ImplFnInfo);
3159
3160	const GlobalDecl &GD = GlobalDecl(CallOp);
3161	const auto *D = cast<FunctionDecl>(Val: GD.getDecl());
3162	CodeGenFunction(CGM).GenerateCode(GD, Fn, FnInfo: **ImplFnInfo);
3163	CGM.SetLLVMFunctionAttributesForDefinition(D: D, F: Fn);
3164	}
3165	*ImplFn = Fn;
3166	}
3167