1	//===--- CGClass.cpp - Emit LLVM Code for C++ classes -----------*- C++ -*-===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This contains code dealing with C++ code generation of classes
10	//
11	//===----------------------------------------------------------------------===//
12
13	#include "CGBlocks.h"
14	#include "CGCXXABI.h"
15	#include "CGDebugInfo.h"
16	#include "CGRecordLayout.h"
17	#include "CodeGenFunction.h"
18	#include "TargetInfo.h"
19	#include "clang/AST/Attr.h"
20	#include "clang/AST/CXXInheritance.h"
21	#include "clang/AST/CharUnits.h"
22	#include "clang/AST/DeclTemplate.h"
23	#include "clang/AST/EvaluatedExprVisitor.h"
24	#include "clang/AST/RecordLayout.h"
25	#include "clang/AST/StmtCXX.h"
26	#include "clang/Basic/CodeGenOptions.h"
27	#include "clang/Basic/TargetBuiltins.h"
28	#include "clang/CodeGen/CGFunctionInfo.h"
29	#include "llvm/IR/Intrinsics.h"
30	#include "llvm/IR/Metadata.h"
31	#include "llvm/Support/SaveAndRestore.h"
32	#include "llvm/Transforms/Utils/SanitizerStats.h"
33	#include <optional>
34
35	using namespace clang;
36	using namespace CodeGen;
37
38	/// Return the best known alignment for an unknown pointer to a
39	/// particular class.
40	CharUnits CodeGenModule::getClassPointerAlignment(const CXXRecordDecl *RD) {
41	if (!RD->hasDefinition())
42	return CharUnits::One(); // Hopefully won't be used anywhere.
43
44	auto &layout = getContext().getASTRecordLayout(RD);
45
46	// If the class is final, then we know that the pointer points to an
47	// object of that type and can use the full alignment.
48	if (RD->isEffectivelyFinal())
49	return layout.getAlignment();
50
51	// Otherwise, we have to assume it could be a subclass.
52	return layout.getNonVirtualAlignment();
53	}
54
55	/// Return the smallest possible amount of storage that might be allocated
56	/// starting from the beginning of an object of a particular class.
57	///
58	/// This may be smaller than sizeof(RD) if RD has virtual base classes.
59	CharUnits CodeGenModule::getMinimumClassObjectSize(const CXXRecordDecl *RD) {
60	if (!RD->hasDefinition())
61	return CharUnits::One();
62
63	auto &layout = getContext().getASTRecordLayout(RD);
64
65	// If the class is final, then we know that the pointer points to an
66	// object of that type and can use the full alignment.
67	if (RD->isEffectivelyFinal())
68	return layout.getSize();
69
70	// Otherwise, we have to assume it could be a subclass.
71	return std::max(layout.getNonVirtualSize(), CharUnits::One());
72	}
73
74	/// Return the best known alignment for a pointer to a virtual base,
75	/// given the alignment of a pointer to the derived class.
76	CharUnits CodeGenModule::getVBaseAlignment(CharUnits actualDerivedAlign,
77	const CXXRecordDecl *derivedClass,
78	const CXXRecordDecl *vbaseClass) {
79	// The basic idea here is that an underaligned derived pointer might
80	// indicate an underaligned base pointer.
81
82	assert(vbaseClass->isCompleteDefinition());
83	auto &baseLayout = getContext().getASTRecordLayout(vbaseClass);
84	CharUnits expectedVBaseAlign = baseLayout.getNonVirtualAlignment();
85
86	return getDynamicOffsetAlignment(ActualAlign: actualDerivedAlign, Class: derivedClass,
87	ExpectedTargetAlign: expectedVBaseAlign);
88	}
89
90	CharUnits
91	CodeGenModule::getDynamicOffsetAlignment(CharUnits actualBaseAlign,
92	const CXXRecordDecl *baseDecl,
93	CharUnits expectedTargetAlign) {
94	// If the base is an incomplete type (which is, alas, possible with
95	// member pointers), be pessimistic.
96	if (!baseDecl->isCompleteDefinition())
97	return std::min(a: actualBaseAlign, b: expectedTargetAlign);
98
99	auto &baseLayout = getContext().getASTRecordLayout(baseDecl);
100	CharUnits expectedBaseAlign = baseLayout.getNonVirtualAlignment();
101
102	// If the class is properly aligned, assume the target offset is, too.
103	//
104	// This actually isn't necessarily the right thing to do --- if the
105	// class is a complete object, but it's only properly aligned for a
106	// base subobject, then the alignments of things relative to it are
107	// probably off as well. (Note that this requires the alignment of
108	// the target to be greater than the NV alignment of the derived
109	// class.)
110	//
111	// However, our approach to this kind of under-alignment can only
112	// ever be best effort; after all, we're never going to propagate
113	// alignments through variables or parameters. Note, in particular,
114	// that constructing a polymorphic type in an address that's less
115	// than pointer-aligned will generally trap in the constructor,
116	// unless we someday add some sort of attribute to change the
117	// assumed alignment of 'this'. So our goal here is pretty much
118	// just to allow the user to explicitly say that a pointer is
119	// under-aligned and then safely access its fields and vtables.
120	if (actualBaseAlign >= expectedBaseAlign) {
121	return expectedTargetAlign;
122	}
123
124	// Otherwise, we might be offset by an arbitrary multiple of the
125	// actual alignment. The correct adjustment is to take the min of
126	// the two alignments.
127	return std::min(a: actualBaseAlign, b: expectedTargetAlign);
128	}
129
130	Address CodeGenFunction::LoadCXXThisAddress() {
131	assert(CurFuncDecl && "loading 'this' without a func declaration?");
132	auto *MD = cast<CXXMethodDecl>(Val: CurFuncDecl);
133
134	// Lazily compute CXXThisAlignment.
135	if (CXXThisAlignment.isZero()) {
136	// Just use the best known alignment for the parent.
137	// TODO: if we're currently emitting a complete-object ctor/dtor,
138	// we can always use the complete-object alignment.
139	CXXThisAlignment = CGM.getClassPointerAlignment(RD: MD->getParent());
140	}
141
142	return makeNaturalAddressForPointer(
143	Ptr: LoadCXXThis(), T: MD->getFunctionObjectParameterType(), Alignment: CXXThisAlignment,
144	ForPointeeType: false, BaseInfo: nullptr, TBAAInfo: nullptr, IsKnownNonNull: KnownNonNull);
145	}
146
147	/// Emit the address of a field using a member data pointer.
148	///
149	/// \param E Only used for emergency diagnostics
150	Address
151	CodeGenFunction::EmitCXXMemberDataPointerAddress(const Expr *E, Address base,
152	llvm::Value *memberPtr,
153	const MemberPointerType *memberPtrType,
154	LValueBaseInfo *BaseInfo,
155	TBAAAccessInfo *TBAAInfo) {
156	// Ask the ABI to compute the actual address.
157	llvm::Value *ptr =
158	CGM.getCXXABI().EmitMemberDataPointerAddress(CGF&: *this, E, Base: base,
159	MemPtr: memberPtr, MPT: memberPtrType);
160
161	QualType memberType = memberPtrType->getPointeeType();
162	CharUnits memberAlign =
163	CGM.getNaturalTypeAlignment(T: memberType, BaseInfo, TBAAInfo);
164	memberAlign =
165	CGM.getDynamicOffsetAlignment(actualBaseAlign: base.getAlignment(),
166	baseDecl: memberPtrType->getClass()->getAsCXXRecordDecl(),
167	expectedTargetAlign: memberAlign);
168	return Address(ptr, ConvertTypeForMem(T: memberPtrType->getPointeeType()),
169	memberAlign);
170	}
171
172	CharUnits CodeGenModule::computeNonVirtualBaseClassOffset(
173	const CXXRecordDecl *DerivedClass, CastExpr::path_const_iterator Start,
174	CastExpr::path_const_iterator End) {
175	CharUnits Offset = CharUnits::Zero();
176
177	const ASTContext &Context = getContext();
178	const CXXRecordDecl *RD = DerivedClass;
179
180	for (CastExpr::path_const_iterator I = Start; I != End; ++I) {
181	const CXXBaseSpecifier *Base = *I;
182	assert(!Base->isVirtual() && "Should not see virtual bases here!");
183
184	// Get the layout.
185	const ASTRecordLayout &Layout = Context.getASTRecordLayout(RD);
186
187	const auto *BaseDecl =
188	cast<CXXRecordDecl>(Val: Base->getType()->castAs<RecordType>()->getDecl());
189
190	// Add the offset.
191	Offset += Layout.getBaseClassOffset(Base: BaseDecl);
192
193	RD = BaseDecl;
194	}
195
196	return Offset;
197	}
198
199	llvm::Constant *
200	CodeGenModule::GetNonVirtualBaseClassOffset(const CXXRecordDecl *ClassDecl,
201	CastExpr::path_const_iterator PathBegin,
202	CastExpr::path_const_iterator PathEnd) {
203	assert(PathBegin != PathEnd && "Base path should not be empty!");
204
205	CharUnits Offset =
206	computeNonVirtualBaseClassOffset(DerivedClass: ClassDecl, Start: PathBegin, End: PathEnd);
207	if (Offset.isZero())
208	return nullptr;
209
210	llvm::Type *PtrDiffTy =
211	Types.ConvertType(T: getContext().getPointerDiffType());
212
213	return llvm::ConstantInt::get(Ty: PtrDiffTy, V: Offset.getQuantity());
214	}
215
216	/// Gets the address of a direct base class within a complete object.
217	/// This should only be used for (1) non-virtual bases or (2) virtual bases
218	/// when the type is known to be complete (e.g. in complete destructors).
219	///
220	/// The object pointed to by 'This' is assumed to be non-null.
221	Address
222	CodeGenFunction::GetAddressOfDirectBaseInCompleteClass(Address This,
223	const CXXRecordDecl *Derived,
224	const CXXRecordDecl *Base,
225	bool BaseIsVirtual) {
226	// 'this' must be a pointer (in some address space) to Derived.
227	assert(This.getElementType() == ConvertType(Derived));
228
229	// Compute the offset of the virtual base.
230	CharUnits Offset;
231	const ASTRecordLayout &Layout = getContext().getASTRecordLayout(Derived);
232	if (BaseIsVirtual)
233	Offset = Layout.getVBaseClassOffset(VBase: Base);
234	else
235	Offset = Layout.getBaseClassOffset(Base);
236
237	// Shift and cast down to the base type.
238	// TODO: for complete types, this should be possible with a GEP.
239	Address V = This;
240	if (!Offset.isZero()) {
241	V = V.withElementType(ElemTy: Int8Ty);
242	V = Builder.CreateConstInBoundsByteGEP(Addr: V, Offset);
243	}
244	return V.withElementType(ElemTy: ConvertType(Base));
245	}
246
247	static Address
248	ApplyNonVirtualAndVirtualOffset(CodeGenFunction &CGF, Address addr,
249	CharUnits nonVirtualOffset,
250	llvm::Value *virtualOffset,
251	const CXXRecordDecl *derivedClass,
252	const CXXRecordDecl *nearestVBase) {
253	// Assert that we have something to do.
254	assert(!nonVirtualOffset.isZero() || virtualOffset != nullptr);
255
256	// Compute the offset from the static and dynamic components.
257	llvm::Value *baseOffset;
258	if (!nonVirtualOffset.isZero()) {
259	llvm::Type *OffsetType =
260	(CGF.CGM.getTarget().getCXXABI().isItaniumFamily() &&
261	CGF.CGM.getItaniumVTableContext().isRelativeLayout())
262	? CGF.Int32Ty
263	: CGF.PtrDiffTy;
264	baseOffset =
265	llvm::ConstantInt::get(Ty: OffsetType, V: nonVirtualOffset.getQuantity());
266	if (virtualOffset) {
267	baseOffset = CGF.Builder.CreateAdd(LHS: virtualOffset, RHS: baseOffset);
268	}
269	} else {
270	baseOffset = virtualOffset;
271	}
272
273	// Apply the base offset.
274	llvm::Value *ptr = addr.emitRawPointer(CGF);
275	ptr = CGF.Builder.CreateInBoundsGEP(Ty: CGF.Int8Ty, Ptr: ptr, IdxList: baseOffset, Name: "add.ptr");
276
277	// If we have a virtual component, the alignment of the result will
278	// be relative only to the known alignment of that vbase.
279	CharUnits alignment;
280	if (virtualOffset) {
281	assert(nearestVBase && "virtual offset without vbase?");
282	alignment = CGF.CGM.getVBaseAlignment(actualDerivedAlign: addr.getAlignment(),
283	derivedClass, vbaseClass: nearestVBase);
284	} else {
285	alignment = addr.getAlignment();
286	}
287	alignment = alignment.alignmentAtOffset(offset: nonVirtualOffset);
288
289	return Address(ptr, CGF.Int8Ty, alignment);
290	}
291
292	Address CodeGenFunction::GetAddressOfBaseClass(
293	Address Value, const CXXRecordDecl *Derived,
294	CastExpr::path_const_iterator PathBegin,
295	CastExpr::path_const_iterator PathEnd, bool NullCheckValue,
296	SourceLocation Loc) {
297	assert(PathBegin != PathEnd && "Base path should not be empty!");
298
299	CastExpr::path_const_iterator Start = PathBegin;
300	const CXXRecordDecl *VBase = nullptr;
301
302	// Sema has done some convenient canonicalization here: if the
303	// access path involved any virtual steps, the conversion path will
304	// *start* with a step down to the correct virtual base subobject,
305	// and hence will not require any further steps.
306	if ((*Start)->isVirtual()) {
307	VBase = cast<CXXRecordDecl>(
308	Val: (*Start)->getType()->castAs<RecordType>()->getDecl());
309	++Start;
310	}
311
312	// Compute the static offset of the ultimate destination within its
313	// allocating subobject (the virtual base, if there is one, or else
314	// the "complete" object that we see).
315	CharUnits NonVirtualOffset = CGM.computeNonVirtualBaseClassOffset(
316	DerivedClass: VBase ? VBase : Derived, Start, End: PathEnd);
317
318	// If there's a virtual step, we can sometimes "devirtualize" it.
319	// For now, that's limited to when the derived type is final.
320	// TODO: "devirtualize" this for accesses to known-complete objects.
321	if (VBase && Derived->hasAttr<FinalAttr>()) {
322	const ASTRecordLayout &layout = getContext().getASTRecordLayout(Derived);
323	CharUnits vBaseOffset = layout.getVBaseClassOffset(VBase);
324	NonVirtualOffset += vBaseOffset;
325	VBase = nullptr; // we no longer have a virtual step
326	}
327
328	// Get the base pointer type.
329	llvm::Type *BaseValueTy = ConvertType(T: (PathEnd[-1])->getType());
330	llvm::Type *PtrTy = llvm::PointerType::get(
331	C&: CGM.getLLVMContext(), AddressSpace: Value.getType()->getPointerAddressSpace());
332
333	QualType DerivedTy = getContext().getRecordType(Derived);
334	CharUnits DerivedAlign = CGM.getClassPointerAlignment(RD: Derived);
335
336	// If the static offset is zero and we don't have a virtual step,
337	// just do a bitcast; null checks are unnecessary.
338	if (NonVirtualOffset.isZero() && !VBase) {
339	if (sanitizePerformTypeCheck()) {
340	SanitizerSet SkippedChecks;
341	SkippedChecks.set(K: SanitizerKind::Null, Value: !NullCheckValue);
342	EmitTypeCheck(TCK: TCK_Upcast, Loc, V: Value.emitRawPointer(CGF&: *this), Type: DerivedTy,
343	Alignment: DerivedAlign, SkippedChecks);
344	}
345	return Value.withElementType(ElemTy: BaseValueTy);
346	}
347
348	llvm::BasicBlock *origBB = nullptr;
349	llvm::BasicBlock *endBB = nullptr;
350
351	// Skip over the offset (and the vtable load) if we're supposed to
352	// null-check the pointer.
353	if (NullCheckValue) {
354	origBB = Builder.GetInsertBlock();
355	llvm::BasicBlock *notNullBB = createBasicBlock(name: "cast.notnull");
356	endBB = createBasicBlock(name: "cast.end");
357
358	llvm::Value *isNull = Builder.CreateIsNull(Addr: Value);
359	Builder.CreateCondBr(Cond: isNull, True: endBB, False: notNullBB);
360	EmitBlock(BB: notNullBB);
361	}
362
363	if (sanitizePerformTypeCheck()) {
364	SanitizerSet SkippedChecks;
365	SkippedChecks.set(K: SanitizerKind::Null, Value: true);
366	EmitTypeCheck(TCK: VBase ? TCK_UpcastToVirtualBase : TCK_Upcast, Loc,
367	V: Value.emitRawPointer(CGF&: *this), Type: DerivedTy, Alignment: DerivedAlign,
368	SkippedChecks);
369	}
370
371	// Compute the virtual offset.
372	llvm::Value *VirtualOffset = nullptr;
373	if (VBase) {
374	VirtualOffset =
375	CGM.getCXXABI().GetVirtualBaseClassOffset(CGF&: *this, This: Value, ClassDecl: Derived, BaseClassDecl: VBase);
376	}
377
378	// Apply both offsets.
379	Value = ApplyNonVirtualAndVirtualOffset(CGF&: *this, addr: Value, nonVirtualOffset: NonVirtualOffset,
380	virtualOffset: VirtualOffset, derivedClass: Derived, nearestVBase: VBase);
381
382	// Cast to the destination type.
383	Value = Value.withElementType(ElemTy: BaseValueTy);
384
385	// Build a phi if we needed a null check.
386	if (NullCheckValue) {
387	llvm::BasicBlock *notNullBB = Builder.GetInsertBlock();
388	Builder.CreateBr(Dest: endBB);
389	EmitBlock(BB: endBB);
390
391	llvm::PHINode *PHI = Builder.CreatePHI(Ty: PtrTy, NumReservedValues: 2, Name: "cast.result");
392	PHI->addIncoming(V: Value.emitRawPointer(CGF&: *this), BB: notNullBB);
393	PHI->addIncoming(V: llvm::Constant::getNullValue(Ty: PtrTy), BB: origBB);
394	Value = Value.withPointer(NewPointer: PHI, IsKnownNonNull: NotKnownNonNull);
395	}
396
397	return Value;
398	}
399
400	Address
401	CodeGenFunction::GetAddressOfDerivedClass(Address BaseAddr,
402	const CXXRecordDecl *Derived,
403	CastExpr::path_const_iterator PathBegin,
404	CastExpr::path_const_iterator PathEnd,
405	bool NullCheckValue) {
406	assert(PathBegin != PathEnd && "Base path should not be empty!");
407
408	QualType DerivedTy =
409	getContext().getCanonicalType(T: getContext().getTagDeclType(Derived));
410	llvm::Type *DerivedValueTy = ConvertType(T: DerivedTy);
411
412	llvm::Value *NonVirtualOffset =
413	CGM.GetNonVirtualBaseClassOffset(ClassDecl: Derived, PathBegin, PathEnd);
414
415	if (!NonVirtualOffset) {
416	// No offset, we can just cast back.
417	return BaseAddr.withElementType(ElemTy: DerivedValueTy);
418	}
419
420	llvm::BasicBlock *CastNull = nullptr;
421	llvm::BasicBlock *CastNotNull = nullptr;
422	llvm::BasicBlock *CastEnd = nullptr;
423
424	if (NullCheckValue) {
425	CastNull = createBasicBlock(name: "cast.null");
426	CastNotNull = createBasicBlock(name: "cast.notnull");
427	CastEnd = createBasicBlock(name: "cast.end");
428
429	llvm::Value *IsNull = Builder.CreateIsNull(Addr: BaseAddr);
430	Builder.CreateCondBr(Cond: IsNull, True: CastNull, False: CastNotNull);
431	EmitBlock(BB: CastNotNull);
432	}
433
434	// Apply the offset.
435	Address Addr = BaseAddr.withElementType(ElemTy: Int8Ty);
436	Addr = Builder.CreateInBoundsGEP(
437	Addr, IdxList: Builder.CreateNeg(V: NonVirtualOffset), ElementType: Int8Ty,
438	Align: CGM.getClassPointerAlignment(RD: Derived), Name: "sub.ptr");
439
440	// Just cast.
441	Addr = Addr.withElementType(ElemTy: DerivedValueTy);
442
443	// Produce a PHI if we had a null-check.
444	if (NullCheckValue) {
445	Builder.CreateBr(Dest: CastEnd);
446	EmitBlock(BB: CastNull);
447	Builder.CreateBr(Dest: CastEnd);
448	EmitBlock(BB: CastEnd);
449
450	llvm::Value *Value = Addr.emitRawPointer(CGF&: *this);
451	llvm::PHINode *PHI = Builder.CreatePHI(Ty: Value->getType(), NumReservedValues: 2);
452	PHI->addIncoming(V: Value, BB: CastNotNull);
453	PHI->addIncoming(V: llvm::Constant::getNullValue(Ty: Value->getType()), BB: CastNull);
454	return Address(PHI, Addr.getElementType(),
455	CGM.getClassPointerAlignment(RD: Derived));
456	}
457
458	return Addr;
459	}
460
461	llvm::Value *CodeGenFunction::GetVTTParameter(GlobalDecl GD,
462	bool ForVirtualBase,
463	bool Delegating) {
464	if (!CGM.getCXXABI().NeedsVTTParameter(GD)) {
465	// This constructor/destructor does not need a VTT parameter.
466	return nullptr;
467	}
468
469	const CXXRecordDecl *RD = cast<CXXMethodDecl>(Val: CurCodeDecl)->getParent();
470	const CXXRecordDecl *Base = cast<CXXMethodDecl>(Val: GD.getDecl())->getParent();
471
472	uint64_t SubVTTIndex;
473
474	if (Delegating) {
475	// If this is a delegating constructor call, just load the VTT.
476	return LoadCXXVTT();
477	} else if (RD == Base) {
478	// If the record matches the base, this is the complete ctor/dtor
479	// variant calling the base variant in a class with virtual bases.
480	assert(!CGM.getCXXABI().NeedsVTTParameter(CurGD) &&
481	"doing no-op VTT offset in base dtor/ctor?");
482	assert(!ForVirtualBase && "Can't have same class as virtual base!");
483	SubVTTIndex = 0;
484	} else {
485	const ASTRecordLayout &Layout = getContext().getASTRecordLayout(RD);
486	CharUnits BaseOffset = ForVirtualBase ?
487	Layout.getVBaseClassOffset(VBase: Base) :
488	Layout.getBaseClassOffset(Base);
489
490	SubVTTIndex =
491	CGM.getVTables().getSubVTTIndex(RD, Base: BaseSubobject(Base, BaseOffset));
492	assert(SubVTTIndex != 0 && "Sub-VTT index must be greater than zero!");
493	}
494
495	if (CGM.getCXXABI().NeedsVTTParameter(GD: CurGD)) {
496	// A VTT parameter was passed to the constructor, use it.
497	llvm::Value *VTT = LoadCXXVTT();
498	return Builder.CreateConstInBoundsGEP1_64(Ty: VoidPtrTy, Ptr: VTT, Idx0: SubVTTIndex);
499	} else {
500	// We're the complete constructor, so get the VTT by name.
501	llvm::GlobalValue *VTT = CGM.getVTables().GetAddrOfVTT(RD);
502	return Builder.CreateConstInBoundsGEP2_64(
503	Ty: VTT->getValueType(), Ptr: VTT, Idx0: 0, Idx1: SubVTTIndex);
504	}
505	}
506
507	namespace {
508	/// Call the destructor for a direct base class.
509	struct CallBaseDtor final : EHScopeStack::Cleanup {
510	const CXXRecordDecl *BaseClass;
511	bool BaseIsVirtual;
512	CallBaseDtor(const CXXRecordDecl *Base, bool BaseIsVirtual)
513	: BaseClass(Base), BaseIsVirtual(BaseIsVirtual) {}
514
515	void Emit(CodeGenFunction &CGF, Flags flags) override {
516	const CXXRecordDecl *DerivedClass =
517	cast<CXXMethodDecl>(Val: CGF.CurCodeDecl)->getParent();
518
519	const CXXDestructorDecl *D = BaseClass->getDestructor();
520	// We are already inside a destructor, so presumably the object being
521	// destroyed should have the expected type.
522	QualType ThisTy = D->getFunctionObjectParameterType();
523	Address Addr =
524	CGF.GetAddressOfDirectBaseInCompleteClass(This: CGF.LoadCXXThisAddress(),
525	Derived: DerivedClass, Base: BaseClass,
526	BaseIsVirtual);
527	CGF.EmitCXXDestructorCall(D, Type: Dtor_Base, ForVirtualBase: BaseIsVirtual,
528	/*Delegating=*/false, This: Addr, ThisTy);
529	}
530	};
531
532	/// A visitor which checks whether an initializer uses 'this' in a
533	/// way which requires the vtable to be properly set.
534	struct DynamicThisUseChecker : ConstEvaluatedExprVisitor<DynamicThisUseChecker> {
535	typedef ConstEvaluatedExprVisitor<DynamicThisUseChecker> super;
536
537	bool UsesThis;
538
539	DynamicThisUseChecker(const ASTContext &C) : super(C), UsesThis(false) {}
540
541	// Black-list all explicit and implicit references to 'this'.
542	//
543	// Do we need to worry about external references to 'this' derived
544	// from arbitrary code? If so, then anything which runs arbitrary
545	// external code might potentially access the vtable.
546	void VisitCXXThisExpr(const CXXThisExpr *E) { UsesThis = true; }
547	};
548	} // end anonymous namespace
549
550	static bool BaseInitializerUsesThis(ASTContext &C, const Expr *Init) {
551	DynamicThisUseChecker Checker(C);
552	Checker.Visit(Init);
553	return Checker.UsesThis;
554	}
555
556	static void EmitBaseInitializer(CodeGenFunction &CGF,
557	const CXXRecordDecl *ClassDecl,
558	CXXCtorInitializer *BaseInit) {
559	assert(BaseInit->isBaseInitializer() &&
560	"Must have base initializer!");
561
562	Address ThisPtr = CGF.LoadCXXThisAddress();
563
564	const Type *BaseType = BaseInit->getBaseClass();
565	const auto *BaseClassDecl =
566	cast<CXXRecordDecl>(Val: BaseType->castAs<RecordType>()->getDecl());
567
568	bool isBaseVirtual = BaseInit->isBaseVirtual();
569
570	// If the initializer for the base (other than the constructor
571	// itself) accesses 'this' in any way, we need to initialize the
572	// vtables.
573	if (BaseInitializerUsesThis(C&: CGF.getContext(), Init: BaseInit->getInit()))
574	CGF.InitializeVTablePointers(ClassDecl);
575
576	// We can pretend to be a complete class because it only matters for
577	// virtual bases, and we only do virtual bases for complete ctors.
578	Address V =
579	CGF.GetAddressOfDirectBaseInCompleteClass(This: ThisPtr, Derived: ClassDecl,
580	Base: BaseClassDecl,
581	BaseIsVirtual: isBaseVirtual);
582	AggValueSlot AggSlot =
583	AggValueSlot::forAddr(
584	addr: V, quals: Qualifiers(),
585	isDestructed: AggValueSlot::IsDestructed,
586	needsGC: AggValueSlot::DoesNotNeedGCBarriers,
587	isAliased: AggValueSlot::IsNotAliased,
588	mayOverlap: CGF.getOverlapForBaseInit(RD: ClassDecl, BaseRD: BaseClassDecl, IsVirtual: isBaseVirtual));
589
590	CGF.EmitAggExpr(E: BaseInit->getInit(), AS: AggSlot);
591
592	if (CGF.CGM.getLangOpts().Exceptions &&
593	!BaseClassDecl->hasTrivialDestructor())
594	CGF.EHStack.pushCleanup<CallBaseDtor>(Kind: EHCleanup, A: BaseClassDecl,
595	A: isBaseVirtual);
596	}
597
598	static bool isMemcpyEquivalentSpecialMember(const CXXMethodDecl *D) {
599	auto *CD = dyn_cast<CXXConstructorDecl>(Val: D);
600	if (!(CD && CD->isCopyOrMoveConstructor()) &&
601	!D->isCopyAssignmentOperator() && !D->isMoveAssignmentOperator())
602	return false;
603
604	// We can emit a memcpy for a trivial copy or move constructor/assignment.
605	if (D->isTrivial() && !D->getParent()->mayInsertExtraPadding())
606	return true;
607
608	// We *must* emit a memcpy for a defaulted union copy or move op.
609	if (D->getParent()->isUnion() && D->isDefaulted())
610	return true;
611
612	return false;
613	}
614
615	static void EmitLValueForAnyFieldInitialization(CodeGenFunction &CGF,
616	CXXCtorInitializer *MemberInit,
617	LValue &LHS) {
618	FieldDecl *Field = MemberInit->getAnyMember();
619	if (MemberInit->isIndirectMemberInitializer()) {
620	// If we are initializing an anonymous union field, drill down to the field.
621	IndirectFieldDecl *IndirectField = MemberInit->getIndirectMember();
622	for (const auto *I : IndirectField->chain())
623	LHS = CGF.EmitLValueForFieldInitialization(Base: LHS, Field: cast<FieldDecl>(Val: I));
624	} else {
625	LHS = CGF.EmitLValueForFieldInitialization(Base: LHS, Field);
626	}
627	}
628
629	static void EmitMemberInitializer(CodeGenFunction &CGF,
630	const CXXRecordDecl *ClassDecl,
631	CXXCtorInitializer *MemberInit,
632	const CXXConstructorDecl *Constructor,
633	FunctionArgList &Args) {
634	ApplyDebugLocation Loc(CGF, MemberInit->getSourceLocation());
635	assert(MemberInit->isAnyMemberInitializer() &&
636	"Must have member initializer!");
637	assert(MemberInit->getInit() && "Must have initializer!");
638
639	// non-static data member initializers.
640	FieldDecl *Field = MemberInit->getAnyMember();
641	QualType FieldType = Field->getType();
642
643	llvm::Value *ThisPtr = CGF.LoadCXXThis();
644	QualType RecordTy = CGF.getContext().getTypeDeclType(ClassDecl);
645	LValue LHS;
646
647	// If a base constructor is being emitted, create an LValue that has the
648	// non-virtual alignment.
649	if (CGF.CurGD.getCtorType() == Ctor_Base)
650	LHS = CGF.MakeNaturalAlignPointeeAddrLValue(V: ThisPtr, T: RecordTy);
651	else
652	LHS = CGF.MakeNaturalAlignAddrLValue(V: ThisPtr, T: RecordTy);
653
654	EmitLValueForAnyFieldInitialization(CGF, MemberInit, LHS);
655
656	// Special case: if we are in a copy or move constructor, and we are copying
657	// an array of PODs or classes with trivial copy constructors, ignore the
658	// AST and perform the copy we know is equivalent.
659	// FIXME: This is hacky at best... if we had a bit more explicit information
660	// in the AST, we could generalize it more easily.
661	const ConstantArrayType *Array
662	= CGF.getContext().getAsConstantArrayType(T: FieldType);
663	if (Array && Constructor->isDefaulted() &&
664	Constructor->isCopyOrMoveConstructor()) {
665	QualType BaseElementTy = CGF.getContext().getBaseElementType(Array);
666	CXXConstructExpr *CE = dyn_cast<CXXConstructExpr>(Val: MemberInit->getInit());
667	if (BaseElementTy.isPODType(Context: CGF.getContext()) ||
668	(CE && isMemcpyEquivalentSpecialMember(CE->getConstructor()))) {
669	unsigned SrcArgIndex =
670	CGF.CGM.getCXXABI().getSrcArgforCopyCtor(Constructor, Args);
671	llvm::Value *SrcPtr
672	= CGF.Builder.CreateLoad(Addr: CGF.GetAddrOfLocalVar(VD: Args[SrcArgIndex]));
673	LValue ThisRHSLV = CGF.MakeNaturalAlignAddrLValue(V: SrcPtr, T: RecordTy);
674	LValue Src = CGF.EmitLValueForFieldInitialization(Base: ThisRHSLV, Field);
675
676	// Copy the aggregate.
677	CGF.EmitAggregateCopy(Dest: LHS, Src, EltTy: FieldType, MayOverlap: CGF.getOverlapForFieldInit(FD: Field),
678	isVolatile: LHS.isVolatileQualified());
679	// Ensure that we destroy the objects if an exception is thrown later in
680	// the constructor.
681	QualType::DestructionKind dtorKind = FieldType.isDestructedType();
682	if (CGF.needsEHCleanup(kind: dtorKind))
683	CGF.pushEHDestroy(dtorKind, addr: LHS.getAddress(CGF), type: FieldType);
684	return;
685	}
686	}
687
688	CGF.EmitInitializerForField(Field, LHS, Init: MemberInit->getInit());
689	}
690
691	void CodeGenFunction::EmitInitializerForField(FieldDecl *Field, LValue LHS,
692	Expr *Init) {
693	QualType FieldType = Field->getType();
694	switch (getEvaluationKind(T: FieldType)) {
695	case TEK_Scalar:
696	if (LHS.isSimple()) {
697	EmitExprAsInit(Init, Field, LHS, false);
698	} else {
699	RValue RHS = RValue::get(V: EmitScalarExpr(E: Init));
700	EmitStoreThroughLValue(Src: RHS, Dst: LHS);
701	}
702	break;
703	case TEK_Complex:
704	EmitComplexExprIntoLValue(E: Init, dest: LHS, /*isInit*/ true);
705	break;
706	case TEK_Aggregate: {
707	AggValueSlot Slot = AggValueSlot::forLValue(
708	LV: LHS, CGF&: *this, isDestructed: AggValueSlot::IsDestructed,
709	needsGC: AggValueSlot::DoesNotNeedGCBarriers, isAliased: AggValueSlot::IsNotAliased,
710	mayOverlap: getOverlapForFieldInit(FD: Field), isZeroed: AggValueSlot::IsNotZeroed,
711	// Checks are made by the code that calls constructor.
712	isChecked: AggValueSlot::IsSanitizerChecked);
713	EmitAggExpr(E: Init, AS: Slot);
714	break;
715	}
716	}
717
718	// Ensure that we destroy this object if an exception is thrown
719	// later in the constructor.
720	QualType::DestructionKind dtorKind = FieldType.isDestructedType();
721	if (needsEHCleanup(kind: dtorKind))
722	pushEHDestroy(dtorKind, addr: LHS.getAddress(CGF&: *this), type: FieldType);
723	}
724
725	/// Checks whether the given constructor is a valid subject for the
726	/// complete-to-base constructor delegation optimization, i.e.
727	/// emitting the complete constructor as a simple call to the base
728	/// constructor.
729	bool CodeGenFunction::IsConstructorDelegationValid(
730	const CXXConstructorDecl *Ctor) {
731
732	// Currently we disable the optimization for classes with virtual
733	// bases because (1) the addresses of parameter variables need to be
734	// consistent across all initializers but (2) the delegate function
735	// call necessarily creates a second copy of the parameter variable.
736	//
737	// The limiting example (purely theoretical AFAIK):
738	// struct A { A(int &c) { c++; } };
739	// struct B : virtual A {
740	// B(int count) : A(count) { printf("%d\n", count); }
741	// };
742	// ...although even this example could in principle be emitted as a
743	// delegation since the address of the parameter doesn't escape.
744	if (Ctor->getParent()->getNumVBases()) {
745	// TODO: white-list trivial vbase initializers. This case wouldn't
746	// be subject to the restrictions below.
747
748	// TODO: white-list cases where:
749	// - there are no non-reference parameters to the constructor
750	// - the initializers don't access any non-reference parameters
751	// - the initializers don't take the address of non-reference
752	// parameters
753	// - etc.
754	// If we ever add any of the above cases, remember that:
755	// - function-try-blocks will always exclude this optimization
756	// - we need to perform the constructor prologue and cleanup in
757	// EmitConstructorBody.
758
759	return false;
760	}
761
762	// We also disable the optimization for variadic functions because
763	// it's impossible to "re-pass" varargs.
764	if (Ctor->getType()->castAs<FunctionProtoType>()->isVariadic())
765	return false;
766
767	// FIXME: Decide if we can do a delegation of a delegating constructor.
768	if (Ctor->isDelegatingConstructor())
769	return false;
770
771	return true;
772	}
773
774	// Emit code in ctor (Prologue==true) or dtor (Prologue==false)
775	// to poison the extra field paddings inserted under
776	// -fsanitize-address-field-padding=1|2.
777	void CodeGenFunction::EmitAsanPrologueOrEpilogue(bool Prologue) {
778	ASTContext &Context = getContext();
779	const CXXRecordDecl *ClassDecl =
780	Prologue ? cast<CXXConstructorDecl>(Val: CurGD.getDecl())->getParent()
781	: cast<CXXDestructorDecl>(Val: CurGD.getDecl())->getParent();
782	if (!ClassDecl->mayInsertExtraPadding()) return;
783
784	struct SizeAndOffset {
785	uint64_t Size;
786	uint64_t Offset;
787	};
788
789	unsigned PtrSize = CGM.getDataLayout().getPointerSizeInBits();
790	const ASTRecordLayout &Info = Context.getASTRecordLayout(ClassDecl);
791
792	// Populate sizes and offsets of fields.
793	SmallVector<SizeAndOffset, 16> SSV(Info.getFieldCount());
794	for (unsigned i = 0, e = Info.getFieldCount(); i != e; ++i)
795	SSV[i].Offset =
796	Context.toCharUnitsFromBits(BitSize: Info.getFieldOffset(FieldNo: i)).getQuantity();
797
798	size_t NumFields = 0;
799	for (const auto *Field : ClassDecl->fields()) {
800	const FieldDecl *D = Field;
801	auto FieldInfo = Context.getTypeInfoInChars(D->getType());
802	CharUnits FieldSize = FieldInfo.Width;
803	assert(NumFields < SSV.size());
804	SSV[NumFields].Size = D->isBitField() ? 0 : FieldSize.getQuantity();
805	NumFields++;
806	}
807	assert(NumFields == SSV.size());
808	if (SSV.size() <= 1) return;
809
810	// We will insert calls to __asan_* run-time functions.
811	// LLVM AddressSanitizer pass may decide to inline them later.
812	llvm::Type *Args[2] = {IntPtrTy, IntPtrTy};
813	llvm::FunctionType *FTy =
814	llvm::FunctionType::get(Result: CGM.VoidTy, Params: Args, isVarArg: false);
815	llvm::FunctionCallee F = CGM.CreateRuntimeFunction(
816	Ty: FTy, Name: Prologue ? "__asan_poison_intra_object_redzone"
817	: "__asan_unpoison_intra_object_redzone");
818
819	llvm::Value *ThisPtr = LoadCXXThis();
820	ThisPtr = Builder.CreatePtrToInt(V: ThisPtr, DestTy: IntPtrTy);
821	uint64_t TypeSize = Info.getNonVirtualSize().getQuantity();
822	// For each field check if it has sufficient padding,
823	// if so (un)poison it with a call.
824	for (size_t i = 0; i < SSV.size(); i++) {
825	uint64_t AsanAlignment = 8;
826	uint64_t NextField = i == SSV.size() - 1 ? TypeSize : SSV[i + 1].Offset;
827	uint64_t PoisonSize = NextField - SSV[i].Offset - SSV[i].Size;
828	uint64_t EndOffset = SSV[i].Offset + SSV[i].Size;
829	if (PoisonSize < AsanAlignment || !SSV[i].Size ||
830	(NextField % AsanAlignment) != 0)
831	continue;
832	Builder.CreateCall(
833	Callee: F, Args: {Builder.CreateAdd(LHS: ThisPtr, RHS: Builder.getIntN(N: PtrSize, C: EndOffset)),
834	Builder.getIntN(N: PtrSize, C: PoisonSize)});
835	}
836	}
837
838	/// EmitConstructorBody - Emits the body of the current constructor.
839	void CodeGenFunction::EmitConstructorBody(FunctionArgList &Args) {
840	EmitAsanPrologueOrEpilogue(Prologue: true);
841	const CXXConstructorDecl *Ctor = cast<CXXConstructorDecl>(Val: CurGD.getDecl());
842	CXXCtorType CtorType = CurGD.getCtorType();
843
844	assert((CGM.getTarget().getCXXABI().hasConstructorVariants() ||
845	CtorType == Ctor_Complete) &&
846	"can only generate complete ctor for this ABI");
847
848	// Before we go any further, try the complete->base constructor
849	// delegation optimization.
850	if (CtorType == Ctor_Complete && IsConstructorDelegationValid(Ctor) &&
851	CGM.getTarget().getCXXABI().hasConstructorVariants()) {
852	EmitDelegateCXXConstructorCall(Ctor, CtorType: Ctor_Base, Args, Loc: Ctor->getEndLoc());
853	return;
854	}
855
856	const FunctionDecl *Definition = nullptr;
857	Stmt *Body = Ctor->getBody(Definition);
858	assert(Definition == Ctor && "emitting wrong constructor body");
859
860	// Enter the function-try-block before the constructor prologue if
861	// applicable.
862	bool IsTryBody = (Body && isa<CXXTryStmt>(Val: Body));
863	if (IsTryBody)
864	EnterCXXTryStmt(S: *cast<CXXTryStmt>(Val: Body), IsFnTryBlock: true);
865
866	incrementProfileCounter(S: Body);
867	maybeCreateMCDCCondBitmap();
868
869	RunCleanupsScope RunCleanups(*this);
870
871	// TODO: in restricted cases, we can emit the vbase initializers of
872	// a complete ctor and then delegate to the base ctor.
873
874	// Emit the constructor prologue, i.e. the base and member
875	// initializers.
876	EmitCtorPrologue(CD: Ctor, Type: CtorType, Args);
877
878	// Emit the body of the statement.
879	if (IsTryBody)
880	EmitStmt(cast<CXXTryStmt>(Val: Body)->getTryBlock());
881	else if (Body)
882	EmitStmt(S: Body);
883
884	// Emit any cleanup blocks associated with the member or base
885	// initializers, which includes (along the exceptional path) the
886	// destructors for those members and bases that were fully
887	// constructed.
888	RunCleanups.ForceCleanup();
889
890	if (IsTryBody)
891	ExitCXXTryStmt(S: *cast<CXXTryStmt>(Val: Body), IsFnTryBlock: true);
892	}
893
894	namespace {
895	/// RAII object to indicate that codegen is copying the value representation
896	/// instead of the object representation. Useful when copying a struct or
897	/// class which has uninitialized members and we're only performing
898	/// lvalue-to-rvalue conversion on the object but not its members.
899	class CopyingValueRepresentation {
900	public:
901	explicit CopyingValueRepresentation(CodeGenFunction &CGF)
902	: CGF(CGF), OldSanOpts(CGF.SanOpts) {
903	CGF.SanOpts.set(K: SanitizerKind::Bool, Value: false);
904	CGF.SanOpts.set(K: SanitizerKind::Enum, Value: false);
905	}
906	~CopyingValueRepresentation() {
907	CGF.SanOpts = OldSanOpts;
908	}
909	private:
910	CodeGenFunction &CGF;
911	SanitizerSet OldSanOpts;
912	};
913	} // end anonymous namespace
914
915	namespace {
916	class FieldMemcpyizer {
917	public:
918	FieldMemcpyizer(CodeGenFunction &CGF, const CXXRecordDecl *ClassDecl,
919	const VarDecl *SrcRec)
920	: CGF(CGF), ClassDecl(ClassDecl), SrcRec(SrcRec),
921	RecLayout(CGF.getContext().getASTRecordLayout(ClassDecl)),
922	FirstField(nullptr), LastField(nullptr), FirstFieldOffset(0),
923	LastFieldOffset(0), LastAddedFieldIndex(0) {}
924
925	bool isMemcpyableField(FieldDecl *F) const {
926	// Never memcpy fields when we are adding poisoned paddings.
927	if (CGF.getContext().getLangOpts().SanitizeAddressFieldPadding)
928	return false;
929	Qualifiers Qual = F->getType().getQualifiers();
930	if (Qual.hasVolatile() || Qual.hasObjCLifetime())
931	return false;
932	return true;
933	}
934
935	void addMemcpyableField(FieldDecl *F) {
936	if (F->isZeroSize(Ctx: CGF.getContext()))
937	return;
938	if (!FirstField)
939	addInitialField(F);
940	else
941	addNextField(F);
942	}
943
944	CharUnits getMemcpySize(uint64_t FirstByteOffset) const {
945	ASTContext &Ctx = CGF.getContext();
946	unsigned LastFieldSize =
947	LastField->isBitField()
948	? LastField->getBitWidthValue(Ctx)
949	: Ctx.toBits(
950	CharSize: Ctx.getTypeInfoDataSizeInChars(T: LastField->getType()).Width);
951	uint64_t MemcpySizeBits = LastFieldOffset + LastFieldSize -
952	FirstByteOffset + Ctx.getCharWidth() - 1;
953	CharUnits MemcpySize = Ctx.toCharUnitsFromBits(BitSize: MemcpySizeBits);
954	return MemcpySize;
955	}
956
957	void emitMemcpy() {
958	// Give the subclass a chance to bail out if it feels the memcpy isn't
959	// worth it (e.g. Hasn't aggregated enough data).
960	if (!FirstField) {
961	return;
962	}
963
964	uint64_t FirstByteOffset;
965	if (FirstField->isBitField()) {
966	const CGRecordLayout &RL =
967	CGF.getTypes().getCGRecordLayout(FirstField->getParent());
968	const CGBitFieldInfo &BFInfo = RL.getBitFieldInfo(FD: FirstField);
969	// FirstFieldOffset is not appropriate for bitfields,
970	// we need to use the storage offset instead.
971	FirstByteOffset = CGF.getContext().toBits(CharSize: BFInfo.StorageOffset);
972	} else {
973	FirstByteOffset = FirstFieldOffset;
974	}
975
976	CharUnits MemcpySize = getMemcpySize(FirstByteOffset);
977	QualType RecordTy = CGF.getContext().getTypeDeclType(ClassDecl);
978	Address ThisPtr = CGF.LoadCXXThisAddress();
979	LValue DestLV = CGF.MakeAddrLValue(Addr: ThisPtr, T: RecordTy);
980	LValue Dest = CGF.EmitLValueForFieldInitialization(Base: DestLV, Field: FirstField);
981	llvm::Value *SrcPtr = CGF.Builder.CreateLoad(Addr: CGF.GetAddrOfLocalVar(VD: SrcRec));
982	LValue SrcLV = CGF.MakeNaturalAlignAddrLValue(V: SrcPtr, T: RecordTy);
983	LValue Src = CGF.EmitLValueForFieldInitialization(Base: SrcLV, Field: FirstField);
984
985	emitMemcpyIR(
986	DestPtr: Dest.isBitField() ? Dest.getBitFieldAddress() : Dest.getAddress(CGF),
987	SrcPtr: Src.isBitField() ? Src.getBitFieldAddress() : Src.getAddress(CGF),
988	Size: MemcpySize);
989	reset();
990	}
991
992	void reset() {
993	FirstField = nullptr;
994	}
995
996	protected:
997	CodeGenFunction &CGF;
998	const CXXRecordDecl *ClassDecl;
999
1000	private:
1001	void emitMemcpyIR(Address DestPtr, Address SrcPtr, CharUnits Size) {
1002	DestPtr = DestPtr.withElementType(ElemTy: CGF.Int8Ty);
1003	SrcPtr = SrcPtr.withElementType(ElemTy: CGF.Int8Ty);
1004	CGF.Builder.CreateMemCpy(Dest: DestPtr, Src: SrcPtr, Size: Size.getQuantity());
1005	}
1006
1007	void addInitialField(FieldDecl *F) {
1008	FirstField = F;
1009	LastField = F;
1010	FirstFieldOffset = RecLayout.getFieldOffset(FieldNo: F->getFieldIndex());
1011	LastFieldOffset = FirstFieldOffset;
1012	LastAddedFieldIndex = F->getFieldIndex();
1013	}
1014
1015	void addNextField(FieldDecl *F) {
1016	// For the most part, the following invariant will hold:
1017	// F->getFieldIndex() == LastAddedFieldIndex + 1
1018	// The one exception is that Sema won't add a copy-initializer for an
1019	// unnamed bitfield, which will show up here as a gap in the sequence.
1020	assert(F->getFieldIndex() >= LastAddedFieldIndex + 1 &&
1021	"Cannot aggregate fields out of order.");
1022	LastAddedFieldIndex = F->getFieldIndex();
1023
1024	// The 'first' and 'last' fields are chosen by offset, rather than field
1025	// index. This allows the code to support bitfields, as well as regular
1026	// fields.
1027	uint64_t FOffset = RecLayout.getFieldOffset(FieldNo: F->getFieldIndex());
1028	if (FOffset < FirstFieldOffset) {
1029	FirstField = F;
1030	FirstFieldOffset = FOffset;
1031	} else if (FOffset >= LastFieldOffset) {
1032	LastField = F;
1033	LastFieldOffset = FOffset;
1034	}
1035	}
1036
1037	const VarDecl *SrcRec;
1038	const ASTRecordLayout &RecLayout;
1039	FieldDecl *FirstField;
1040	FieldDecl *LastField;
1041	uint64_t FirstFieldOffset, LastFieldOffset;
1042	unsigned LastAddedFieldIndex;
1043	};
1044
1045	class ConstructorMemcpyizer : public FieldMemcpyizer {
1046	private:
1047	/// Get source argument for copy constructor. Returns null if not a copy
1048	/// constructor.
1049	static const VarDecl *getTrivialCopySource(CodeGenFunction &CGF,
1050	const CXXConstructorDecl *CD,
1051	FunctionArgList &Args) {
1052	if (CD->isCopyOrMoveConstructor() && CD->isDefaulted())
1053	return Args[CGF.CGM.getCXXABI().getSrcArgforCopyCtor(CD, Args)];
1054	return nullptr;
1055	}
1056
1057	// Returns true if a CXXCtorInitializer represents a member initialization
1058	// that can be rolled into a memcpy.
1059	bool isMemberInitMemcpyable(CXXCtorInitializer *MemberInit) const {
1060	if (!MemcpyableCtor)
1061	return false;
1062	FieldDecl *Field = MemberInit->getMember();
1063	assert(Field && "No field for member init.");
1064	QualType FieldType = Field->getType();
1065	CXXConstructExpr *CE = dyn_cast<CXXConstructExpr>(Val: MemberInit->getInit());
1066
1067	// Bail out on non-memcpyable, not-trivially-copyable members.
1068	if (!(CE && isMemcpyEquivalentSpecialMember(CE->getConstructor())) &&
1069	!(FieldType.isTriviallyCopyableType(Context: CGF.getContext()) ||
1070	FieldType->isReferenceType()))
1071	return false;
1072
1073	// Bail out on volatile fields.
1074	if (!isMemcpyableField(F: Field))
1075	return false;
1076
1077	// Otherwise we're good.
1078	return true;
1079	}
1080
1081	public:
1082	ConstructorMemcpyizer(CodeGenFunction &CGF, const CXXConstructorDecl *CD,
1083	FunctionArgList &Args)
1084	: FieldMemcpyizer(CGF, CD->getParent(), getTrivialCopySource(CGF, CD, Args)),
1085	ConstructorDecl(CD),
1086	MemcpyableCtor(CD->isDefaulted() &&
1087	CD->isCopyOrMoveConstructor() &&
1088	CGF.getLangOpts().getGC() == LangOptions::NonGC),
1089	Args(Args) { }
1090
1091	void addMemberInitializer(CXXCtorInitializer *MemberInit) {
1092	if (isMemberInitMemcpyable(MemberInit)) {
1093	AggregatedInits.push_back(Elt: MemberInit);
1094	addMemcpyableField(F: MemberInit->getMember());
1095	} else {
1096	emitAggregatedInits();
1097	EmitMemberInitializer(CGF, ConstructorDecl->getParent(), MemberInit,
1098	ConstructorDecl, Args);
1099	}
1100	}
1101
1102	void emitAggregatedInits() {
1103	if (AggregatedInits.size() <= 1) {
1104	// This memcpy is too small to be worthwhile. Fall back on default
1105	// codegen.
1106	if (!AggregatedInits.empty()) {
1107	CopyingValueRepresentation CVR(CGF);
1108	EmitMemberInitializer(CGF, ConstructorDecl->getParent(),
1109	AggregatedInits[0], ConstructorDecl, Args);
1110	AggregatedInits.clear();
1111	}
1112	reset();
1113	return;
1114	}
1115
1116	pushEHDestructors();
1117	emitMemcpy();
1118	AggregatedInits.clear();
1119	}
1120
1121	void pushEHDestructors() {
1122	Address ThisPtr = CGF.LoadCXXThisAddress();
1123	QualType RecordTy = CGF.getContext().getTypeDeclType(ClassDecl);
1124	LValue LHS = CGF.MakeAddrLValue(Addr: ThisPtr, T: RecordTy);
1125
1126	for (unsigned i = 0; i < AggregatedInits.size(); ++i) {
1127	CXXCtorInitializer *MemberInit = AggregatedInits[i];
1128	QualType FieldType = MemberInit->getAnyMember()->getType();
1129	QualType::DestructionKind dtorKind = FieldType.isDestructedType();
1130	if (!CGF.needsEHCleanup(kind: dtorKind))
1131	continue;
1132	LValue FieldLHS = LHS;
1133	EmitLValueForAnyFieldInitialization(CGF, MemberInit, LHS&: FieldLHS);
1134	CGF.pushEHDestroy(dtorKind, addr: FieldLHS.getAddress(CGF), type: FieldType);
1135	}
1136	}
1137
1138	void finish() {
1139	emitAggregatedInits();
1140	}
1141
1142	private:
1143	const CXXConstructorDecl *ConstructorDecl;
1144	bool MemcpyableCtor;
1145	FunctionArgList &Args;
1146	SmallVector<CXXCtorInitializer*, 16> AggregatedInits;
1147	};
1148
1149	class AssignmentMemcpyizer : public FieldMemcpyizer {
1150	private:
1151	// Returns the memcpyable field copied by the given statement, if one
1152	// exists. Otherwise returns null.
1153	FieldDecl *getMemcpyableField(Stmt *S) {
1154	if (!AssignmentsMemcpyable)
1155	return nullptr;
1156	if (BinaryOperator *BO = dyn_cast<BinaryOperator>(Val: S)) {
1157	// Recognise trivial assignments.
1158	if (BO->getOpcode() != BO_Assign)
1159	return nullptr;
1160	MemberExpr *ME = dyn_cast<MemberExpr>(Val: BO->getLHS());
1161	if (!ME)
1162	return nullptr;
1163	FieldDecl *Field = dyn_cast<FieldDecl>(Val: ME->getMemberDecl());
1164	if (!Field || !isMemcpyableField(F: Field))
1165	return nullptr;
1166	Stmt *RHS = BO->getRHS();
1167	if (ImplicitCastExpr *EC = dyn_cast<ImplicitCastExpr>(Val: RHS))
1168	RHS = EC->getSubExpr();
1169	if (!RHS)
1170	return nullptr;
1171	if (MemberExpr *ME2 = dyn_cast<MemberExpr>(Val: RHS)) {
1172	if (ME2->getMemberDecl() == Field)
1173	return Field;
1174	}
1175	return nullptr;
1176	} else if (CXXMemberCallExpr *MCE = dyn_cast<CXXMemberCallExpr>(Val: S)) {
1177	CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(MCE->getCalleeDecl());
1178	if (!(MD && isMemcpyEquivalentSpecialMember(D: MD)))
1179	return nullptr;
1180	MemberExpr *IOA = dyn_cast<MemberExpr>(Val: MCE->getImplicitObjectArgument());
1181	if (!IOA)
1182	return nullptr;
1183	FieldDecl *Field = dyn_cast<FieldDecl>(Val: IOA->getMemberDecl());
1184	if (!Field || !isMemcpyableField(F: Field))
1185	return nullptr;
1186	MemberExpr *Arg0 = dyn_cast<MemberExpr>(MCE->getArg(0));
1187	if (!Arg0 || Field != dyn_cast<FieldDecl>(Val: Arg0->getMemberDecl()))
1188	return nullptr;
1189	return Field;
1190	} else if (CallExpr *CE = dyn_cast<CallExpr>(Val: S)) {
1191	FunctionDecl *FD = dyn_cast<FunctionDecl>(Val: CE->getCalleeDecl());
1192	if (!FD || FD->getBuiltinID() != Builtin::BI__builtin_memcpy)
1193	return nullptr;
1194	Expr *DstPtr = CE->getArg(Arg: 0);
1195	if (ImplicitCastExpr *DC = dyn_cast<ImplicitCastExpr>(Val: DstPtr))
1196	DstPtr = DC->getSubExpr();
1197	UnaryOperator *DUO = dyn_cast<UnaryOperator>(Val: DstPtr);
1198	if (!DUO || DUO->getOpcode() != UO_AddrOf)
1199	return nullptr;
1200	MemberExpr *ME = dyn_cast<MemberExpr>(Val: DUO->getSubExpr());
1201	if (!ME)
1202	return nullptr;
1203	FieldDecl *Field = dyn_cast<FieldDecl>(Val: ME->getMemberDecl());
1204	if (!Field || !isMemcpyableField(F: Field))
1205	return nullptr;
1206	Expr *SrcPtr = CE->getArg(Arg: 1);
1207	if (ImplicitCastExpr *SC = dyn_cast<ImplicitCastExpr>(Val: SrcPtr))
1208	SrcPtr = SC->getSubExpr();
1209	UnaryOperator *SUO = dyn_cast<UnaryOperator>(Val: SrcPtr);
1210	if (!SUO || SUO->getOpcode() != UO_AddrOf)
1211	return nullptr;
1212	MemberExpr *ME2 = dyn_cast<MemberExpr>(Val: SUO->getSubExpr());
1213	if (!ME2 || Field != dyn_cast<FieldDecl>(Val: ME2->getMemberDecl()))
1214	return nullptr;
1215	return Field;
1216	}
1217
1218	return nullptr;
1219	}
1220
1221	bool AssignmentsMemcpyable;
1222	SmallVector<Stmt*, 16> AggregatedStmts;
1223
1224	public:
1225	AssignmentMemcpyizer(CodeGenFunction &CGF, const CXXMethodDecl *AD,
1226	FunctionArgList &Args)
1227	: FieldMemcpyizer(CGF, AD->getParent(), Args[Args.size() - 1]),
1228	AssignmentsMemcpyable(CGF.getLangOpts().getGC() == LangOptions::NonGC) {
1229	assert(Args.size() == 2);
1230	}
1231
1232	void emitAssignment(Stmt *S) {
1233	FieldDecl *F = getMemcpyableField(S);
1234	if (F) {
1235	addMemcpyableField(F);
1236	AggregatedStmts.push_back(Elt: S);
1237	} else {
1238	emitAggregatedStmts();
1239	CGF.EmitStmt(S);
1240	}
1241	}
1242
1243	void emitAggregatedStmts() {
1244	if (AggregatedStmts.size() <= 1) {
1245	if (!AggregatedStmts.empty()) {
1246	CopyingValueRepresentation CVR(CGF);
1247	CGF.EmitStmt(S: AggregatedStmts[0]);
1248	}
1249	reset();
1250	}
1251
1252	emitMemcpy();
1253	AggregatedStmts.clear();
1254	}
1255
1256	void finish() {
1257	emitAggregatedStmts();
1258	}
1259	};
1260	} // end anonymous namespace
1261
1262	static bool isInitializerOfDynamicClass(const CXXCtorInitializer *BaseInit) {
1263	const Type *BaseType = BaseInit->getBaseClass();
1264	const auto *BaseClassDecl =
1265	cast<CXXRecordDecl>(Val: BaseType->castAs<RecordType>()->getDecl());
1266	return BaseClassDecl->isDynamicClass();
1267	}
1268
1269	/// EmitCtorPrologue - This routine generates necessary code to initialize
1270	/// base classes and non-static data members belonging to this constructor.
1271	void CodeGenFunction::EmitCtorPrologue(const CXXConstructorDecl *CD,
1272	CXXCtorType CtorType,
1273	FunctionArgList &Args) {
1274	if (CD->isDelegatingConstructor())
1275	return EmitDelegatingCXXConstructorCall(Ctor: CD, Args);
1276
1277	const CXXRecordDecl *ClassDecl = CD->getParent();
1278
1279	CXXConstructorDecl::init_const_iterator B = CD->init_begin(),
1280	E = CD->init_end();
1281
1282	// Virtual base initializers first, if any. They aren't needed if:
1283	// - This is a base ctor variant
1284	// - There are no vbases
1285	// - The class is abstract, so a complete object of it cannot be constructed
1286	//
1287	// The check for an abstract class is necessary because sema may not have
1288	// marked virtual base destructors referenced.
1289	bool ConstructVBases = CtorType != Ctor_Base &&
1290	ClassDecl->getNumVBases() != 0 &&
1291	!ClassDecl->isAbstract();
1292
1293	// In the Microsoft C++ ABI, there are no constructor variants. Instead, the
1294	// constructor of a class with virtual bases takes an additional parameter to
1295	// conditionally construct the virtual bases. Emit that check here.
1296	llvm::BasicBlock *BaseCtorContinueBB = nullptr;
1297	if (ConstructVBases &&
1298	!CGM.getTarget().getCXXABI().hasConstructorVariants()) {
1299	BaseCtorContinueBB =
1300	CGM.getCXXABI().EmitCtorCompleteObjectHandler(CGF&: *this, RD: ClassDecl);
1301	assert(BaseCtorContinueBB);
1302	}
1303
1304	for (; B != E && (*B)->isBaseInitializer() && (*B)->isBaseVirtual(); B++) {
1305	if (!ConstructVBases)
1306	continue;
1307	SaveAndRestore ThisRAII(CXXThisValue);
1308	if (CGM.getCodeGenOpts().StrictVTablePointers &&
1309	CGM.getCodeGenOpts().OptimizationLevel > 0 &&
1310	isInitializerOfDynamicClass(BaseInit: *B))
1311	CXXThisValue = Builder.CreateLaunderInvariantGroup(Ptr: LoadCXXThis());
1312	EmitBaseInitializer(CGF&: *this, ClassDecl, BaseInit: *B);
1313	}
1314
1315	if (BaseCtorContinueBB) {
1316	// Complete object handler should continue to the remaining initializers.
1317	Builder.CreateBr(Dest: BaseCtorContinueBB);
1318	EmitBlock(BB: BaseCtorContinueBB);
1319	}
1320
1321	// Then, non-virtual base initializers.
1322	for (; B != E && (*B)->isBaseInitializer(); B++) {
1323	assert(!(*B)->isBaseVirtual());
1324	SaveAndRestore ThisRAII(CXXThisValue);
1325	if (CGM.getCodeGenOpts().StrictVTablePointers &&
1326	CGM.getCodeGenOpts().OptimizationLevel > 0 &&
1327	isInitializerOfDynamicClass(BaseInit: *B))
1328	CXXThisValue = Builder.CreateLaunderInvariantGroup(Ptr: LoadCXXThis());
1329	EmitBaseInitializer(CGF&: *this, ClassDecl, BaseInit: *B);
1330	}
1331
1332	InitializeVTablePointers(ClassDecl);
1333
1334	// And finally, initialize class members.
1335	FieldConstructionScope FCS(*this, LoadCXXThisAddress());
1336	ConstructorMemcpyizer CM(*this, CD, Args);
1337	for (; B != E; B++) {
1338	CXXCtorInitializer *Member = (*B);
1339	assert(!Member->isBaseInitializer());
1340	assert(Member->isAnyMemberInitializer() &&
1341	"Delegating initializer on non-delegating constructor");
1342	CM.addMemberInitializer(MemberInit: Member);
1343	}
1344	CM.finish();
1345	}
1346
1347	static bool
1348	FieldHasTrivialDestructorBody(ASTContext &Context, const FieldDecl *Field);
1349
1350	static bool
1351	HasTrivialDestructorBody(ASTContext &Context,
1352	const CXXRecordDecl *BaseClassDecl,
1353	const CXXRecordDecl *MostDerivedClassDecl)
1354	{
1355	// If the destructor is trivial we don't have to check anything else.
1356	if (BaseClassDecl->hasTrivialDestructor())
1357	return true;
1358
1359	if (!BaseClassDecl->getDestructor()->hasTrivialBody())
1360	return false;
1361
1362	// Check fields.
1363	for (const auto *Field : BaseClassDecl->fields())
1364	if (!FieldHasTrivialDestructorBody(Context, Field))
1365	return false;
1366
1367	// Check non-virtual bases.
1368	for (const auto &I : BaseClassDecl->bases()) {
1369	if (I.isVirtual())
1370	continue;
1371
1372	const CXXRecordDecl *NonVirtualBase =
1373	cast<CXXRecordDecl>(Val: I.getType()->castAs<RecordType>()->getDecl());
1374	if (!HasTrivialDestructorBody(Context, BaseClassDecl: NonVirtualBase,
1375	MostDerivedClassDecl))
1376	return false;
1377	}
1378
1379	if (BaseClassDecl == MostDerivedClassDecl) {
1380	// Check virtual bases.
1381	for (const auto &I : BaseClassDecl->vbases()) {
1382	const CXXRecordDecl *VirtualBase =
1383	cast<CXXRecordDecl>(Val: I.getType()->castAs<RecordType>()->getDecl());
1384	if (!HasTrivialDestructorBody(Context, BaseClassDecl: VirtualBase,
1385	MostDerivedClassDecl))
1386	return false;
1387	}
1388	}
1389
1390	return true;
1391	}
1392
1393	static bool
1394	FieldHasTrivialDestructorBody(ASTContext &Context,
1395	const FieldDecl *Field)
1396	{
1397	QualType FieldBaseElementType = Context.getBaseElementType(Field->getType());
1398
1399	const RecordType *RT = FieldBaseElementType->getAs<RecordType>();
1400	if (!RT)
1401	return true;
1402
1403	CXXRecordDecl *FieldClassDecl = cast<CXXRecordDecl>(Val: RT->getDecl());
1404
1405	// The destructor for an implicit anonymous union member is never invoked.
1406	if (FieldClassDecl->isUnion() && FieldClassDecl->isAnonymousStructOrUnion())
1407	return true;
1408
1409	return HasTrivialDestructorBody(Context, BaseClassDecl: FieldClassDecl, MostDerivedClassDecl: FieldClassDecl);
1410	}
1411
1412	/// CanSkipVTablePointerInitialization - Check whether we need to initialize
1413	/// any vtable pointers before calling this destructor.
1414	static bool CanSkipVTablePointerInitialization(CodeGenFunction &CGF,
1415	const CXXDestructorDecl *Dtor) {
1416	const CXXRecordDecl *ClassDecl = Dtor->getParent();
1417	if (!ClassDecl->isDynamicClass())
1418	return true;
1419
1420	// For a final class, the vtable pointer is known to already point to the
1421	// class's vtable.
1422	if (ClassDecl->isEffectivelyFinal())
1423	return true;
1424
1425	if (!Dtor->hasTrivialBody())
1426	return false;
1427
1428	// Check the fields.
1429	for (const auto *Field : ClassDecl->fields())
1430	if (!FieldHasTrivialDestructorBody(CGF.getContext(), Field))
1431	return false;
1432
1433	return true;
1434	}
1435
1436	/// EmitDestructorBody - Emits the body of the current destructor.
1437	void CodeGenFunction::EmitDestructorBody(FunctionArgList &Args) {
1438	const CXXDestructorDecl *Dtor = cast<CXXDestructorDecl>(Val: CurGD.getDecl());
1439	CXXDtorType DtorType = CurGD.getDtorType();
1440
1441	// For an abstract class, non-base destructors are never used (and can't
1442	// be emitted in general, because vbase dtors may not have been validated
1443	// by Sema), but the Itanium ABI doesn't make them optional and Clang may
1444	// in fact emit references to them from other compilations, so emit them
1445	// as functions containing a trap instruction.
1446	if (DtorType != Dtor_Base && Dtor->getParent()->isAbstract()) {
1447	llvm::CallInst *TrapCall = EmitTrapCall(llvm::Intrinsic::trap);
1448	TrapCall->setDoesNotReturn();
1449	TrapCall->setDoesNotThrow();
1450	Builder.CreateUnreachable();
1451	Builder.ClearInsertionPoint();
1452	return;
1453	}
1454
1455	Stmt *Body = Dtor->getBody();
1456	if (Body) {
1457	incrementProfileCounter(S: Body);
1458	maybeCreateMCDCCondBitmap();
1459	}
1460
1461	// The call to operator delete in a deleting destructor happens
1462	// outside of the function-try-block, which means it's always
1463	// possible to delegate the destructor body to the complete
1464	// destructor. Do so.
1465	if (DtorType == Dtor_Deleting) {
1466	RunCleanupsScope DtorEpilogue(*this);
1467	EnterDtorCleanups(Dtor, Type: Dtor_Deleting);
1468	if (HaveInsertPoint()) {
1469	QualType ThisTy = Dtor->getFunctionObjectParameterType();
1470	EmitCXXDestructorCall(D: Dtor, Type: Dtor_Complete, /*ForVirtualBase=*/false,
1471	/*Delegating=*/false, This: LoadCXXThisAddress(), ThisTy);
1472	}
1473	return;
1474	}
1475
1476	// If the body is a function-try-block, enter the try before
1477	// anything else.
1478	bool isTryBody = (Body && isa<CXXTryStmt>(Val: Body));
1479	if (isTryBody)
1480	EnterCXXTryStmt(S: *cast<CXXTryStmt>(Val: Body), IsFnTryBlock: true);
1481	EmitAsanPrologueOrEpilogue(Prologue: false);
1482
1483	// Enter the epilogue cleanups.
1484	RunCleanupsScope DtorEpilogue(*this);
1485
1486	// If this is the complete variant, just invoke the base variant;
1487	// the epilogue will destruct the virtual bases. But we can't do
1488	// this optimization if the body is a function-try-block, because
1489	// we'd introduce *two* handler blocks. In the Microsoft ABI, we
1490	// always delegate because we might not have a definition in this TU.
1491	switch (DtorType) {
1492	case Dtor_Comdat: llvm_unreachable("not expecting a COMDAT");
1493	case Dtor_Deleting: llvm_unreachable("already handled deleting case");
1494
1495	case Dtor_Complete:
1496	assert((Body || getTarget().getCXXABI().isMicrosoft()) &&
1497	"can't emit a dtor without a body for non-Microsoft ABIs");
1498
1499	// Enter the cleanup scopes for virtual bases.
1500	EnterDtorCleanups(Dtor, Type: Dtor_Complete);
1501
1502	if (!isTryBody) {
1503	QualType ThisTy = Dtor->getFunctionObjectParameterType();
1504	EmitCXXDestructorCall(D: Dtor, Type: Dtor_Base, /*ForVirtualBase=*/false,
1505	/*Delegating=*/false, This: LoadCXXThisAddress(), ThisTy);
1506	break;
1507	}
1508
1509	// Fallthrough: act like we're in the base variant.
1510	[[fallthrough]];
1511
1512	case Dtor_Base:
1513	assert(Body);
1514
1515	// Enter the cleanup scopes for fields and non-virtual bases.
1516	EnterDtorCleanups(Dtor, Type: Dtor_Base);
1517
1518	// Initialize the vtable pointers before entering the body.
1519	if (!CanSkipVTablePointerInitialization(CGF&: *this, Dtor)) {
1520	// Insert the llvm.launder.invariant.group intrinsic before initializing
1521	// the vptrs to cancel any previous assumptions we might have made.
1522	if (CGM.getCodeGenOpts().StrictVTablePointers &&
1523	CGM.getCodeGenOpts().OptimizationLevel > 0)
1524	CXXThisValue = Builder.CreateLaunderInvariantGroup(Ptr: LoadCXXThis());
1525	InitializeVTablePointers(ClassDecl: Dtor->getParent());
1526	}
1527
1528	if (isTryBody)
1529	EmitStmt(cast<CXXTryStmt>(Val: Body)->getTryBlock());
1530	else if (Body)
1531	EmitStmt(S: Body);
1532	else {
1533	assert(Dtor->isImplicit() && "bodyless dtor not implicit");
1534	// nothing to do besides what's in the epilogue
1535	}
1536	// -fapple-kext must inline any call to this dtor into
1537	// the caller's body.
1538	if (getLangOpts().AppleKext)
1539	CurFn->addFnAttr(llvm::Attribute::AlwaysInline);
1540
1541	break;
1542	}
1543
1544	// Jump out through the epilogue cleanups.
1545	DtorEpilogue.ForceCleanup();
1546
1547	// Exit the try if applicable.
1548	if (isTryBody)
1549	ExitCXXTryStmt(S: *cast<CXXTryStmt>(Val: Body), IsFnTryBlock: true);
1550	}
1551
1552	void CodeGenFunction::emitImplicitAssignmentOperatorBody(FunctionArgList &Args) {
1553	const CXXMethodDecl *AssignOp = cast<CXXMethodDecl>(Val: CurGD.getDecl());
1554	const Stmt *RootS = AssignOp->getBody();
1555	assert(isa<CompoundStmt>(RootS) &&
1556	"Body of an implicit assignment operator should be compound stmt.");
1557	const CompoundStmt *RootCS = cast<CompoundStmt>(Val: RootS);
1558
1559	LexicalScope Scope(*this, RootCS->getSourceRange());
1560
1561	incrementProfileCounter(RootCS);
1562	maybeCreateMCDCCondBitmap();
1563	AssignmentMemcpyizer AM(*this, AssignOp, Args);
1564	for (auto *I : RootCS->body())
1565	AM.emitAssignment(I);
1566	AM.finish();
1567	}
1568
1569	namespace {
1570	llvm::Value *LoadThisForDtorDelete(CodeGenFunction &CGF,
1571	const CXXDestructorDecl *DD) {
1572	if (Expr *ThisArg = DD->getOperatorDeleteThisArg())
1573	return CGF.EmitScalarExpr(E: ThisArg);
1574	return CGF.LoadCXXThis();
1575	}
1576
1577	/// Call the operator delete associated with the current destructor.
1578	struct CallDtorDelete final : EHScopeStack::Cleanup {
1579	CallDtorDelete() {}
1580
1581	void Emit(CodeGenFunction &CGF, Flags flags) override {
1582	const CXXDestructorDecl *Dtor = cast<CXXDestructorDecl>(Val: CGF.CurCodeDecl);
1583	const CXXRecordDecl *ClassDecl = Dtor->getParent();
1584	CGF.EmitDeleteCall(DeleteFD: Dtor->getOperatorDelete(),
1585	Ptr: LoadThisForDtorDelete(CGF, DD: Dtor),
1586	DeleteTy: CGF.getContext().getTagDeclType(ClassDecl));
1587	}
1588	};
1589
1590	void EmitConditionalDtorDeleteCall(CodeGenFunction &CGF,
1591	llvm::Value *ShouldDeleteCondition,
1592	bool ReturnAfterDelete) {
1593	llvm::BasicBlock *callDeleteBB = CGF.createBasicBlock(name: "dtor.call_delete");
1594	llvm::BasicBlock *continueBB = CGF.createBasicBlock(name: "dtor.continue");
1595	llvm::Value *ShouldCallDelete
1596	= CGF.Builder.CreateIsNull(Arg: ShouldDeleteCondition);
1597	CGF.Builder.CreateCondBr(Cond: ShouldCallDelete, True: continueBB, False: callDeleteBB);
1598
1599	CGF.EmitBlock(BB: callDeleteBB);
1600	const CXXDestructorDecl *Dtor = cast<CXXDestructorDecl>(Val: CGF.CurCodeDecl);
1601	const CXXRecordDecl *ClassDecl = Dtor->getParent();
1602	CGF.EmitDeleteCall(DeleteFD: Dtor->getOperatorDelete(),
1603	Ptr: LoadThisForDtorDelete(CGF, DD: Dtor),
1604	DeleteTy: CGF.getContext().getTagDeclType(ClassDecl));
1605	assert(Dtor->getOperatorDelete()->isDestroyingOperatorDelete() ==
1606	ReturnAfterDelete &&
1607	"unexpected value for ReturnAfterDelete");
1608	if (ReturnAfterDelete)
1609	CGF.EmitBranchThroughCleanup(Dest: CGF.ReturnBlock);
1610	else
1611	CGF.Builder.CreateBr(Dest: continueBB);
1612
1613	CGF.EmitBlock(BB: continueBB);
1614	}
1615
1616	struct CallDtorDeleteConditional final : EHScopeStack::Cleanup {
1617	llvm::Value *ShouldDeleteCondition;
1618
1619	public:
1620	CallDtorDeleteConditional(llvm::Value *ShouldDeleteCondition)
1621	: ShouldDeleteCondition(ShouldDeleteCondition) {
1622	assert(ShouldDeleteCondition != nullptr);
1623	}
1624
1625	void Emit(CodeGenFunction &CGF, Flags flags) override {
1626	EmitConditionalDtorDeleteCall(CGF, ShouldDeleteCondition,
1627	/*ReturnAfterDelete*/false);
1628	}
1629	};
1630
1631	class DestroyField final : public EHScopeStack::Cleanup {
1632	const FieldDecl *field;
1633	CodeGenFunction::Destroyer *destroyer;
1634	bool useEHCleanupForArray;
1635
1636	public:
1637	DestroyField(const FieldDecl *field, CodeGenFunction::Destroyer *destroyer,
1638	bool useEHCleanupForArray)
1639	: field(field), destroyer(destroyer),
1640	useEHCleanupForArray(useEHCleanupForArray) {}
1641
1642	void Emit(CodeGenFunction &CGF, Flags flags) override {
1643	// Find the address of the field.
1644	Address thisValue = CGF.LoadCXXThisAddress();
1645	QualType RecordTy = CGF.getContext().getTagDeclType(field->getParent());
1646	LValue ThisLV = CGF.MakeAddrLValue(Addr: thisValue, T: RecordTy);
1647	LValue LV = CGF.EmitLValueForField(Base: ThisLV, Field: field);
1648	assert(LV.isSimple());
1649
1650	CGF.emitDestroy(addr: LV.getAddress(CGF), type: field->getType(), destroyer,
1651	useEHCleanupForArray: flags.isForNormalCleanup() && useEHCleanupForArray);
1652	}
1653	};
1654
1655	class DeclAsInlineDebugLocation {
1656	CGDebugInfo *DI;
1657	llvm::MDNode *InlinedAt;
1658	std::optional<ApplyDebugLocation> Location;
1659
1660	public:
1661	DeclAsInlineDebugLocation(CodeGenFunction &CGF, const NamedDecl &Decl)
1662	: DI(CGF.getDebugInfo()) {
1663	if (!DI)
1664	return;
1665	InlinedAt = DI->getInlinedAt();
1666	DI->setInlinedAt(CGF.Builder.getCurrentDebugLocation());
1667	Location.emplace(CGF, Decl.getLocation());
1668	}
1669
1670	~DeclAsInlineDebugLocation() {
1671	if (!DI)
1672	return;
1673	Location.reset();
1674	DI->setInlinedAt(InlinedAt);
1675	}
1676	};
1677
1678	static void EmitSanitizerDtorCallback(
1679	CodeGenFunction &CGF, StringRef Name, llvm::Value *Ptr,
1680	std::optional<CharUnits::QuantityType> PoisonSize = {}) {
1681	CodeGenFunction::SanitizerScope SanScope(&CGF);
1682	// Pass in void pointer and size of region as arguments to runtime
1683	// function
1684	SmallVector<llvm::Value *, 2> Args = {Ptr};
1685	SmallVector<llvm::Type *, 2> ArgTypes = {CGF.VoidPtrTy};
1686
1687	if (PoisonSize.has_value()) {
1688	Args.emplace_back(Args: llvm::ConstantInt::get(Ty: CGF.SizeTy, V: *PoisonSize));
1689	ArgTypes.emplace_back(Args&: CGF.SizeTy);
1690	}
1691
1692	llvm::FunctionType *FnType =
1693	llvm::FunctionType::get(Result: CGF.VoidTy, Params: ArgTypes, isVarArg: false);
1694	llvm::FunctionCallee Fn = CGF.CGM.CreateRuntimeFunction(Ty: FnType, Name);
1695
1696	CGF.EmitNounwindRuntimeCall(callee: Fn, args: Args);
1697	}
1698
1699	static void
1700	EmitSanitizerDtorFieldsCallback(CodeGenFunction &CGF, llvm::Value *Ptr,
1701	CharUnits::QuantityType PoisonSize) {
1702	EmitSanitizerDtorCallback(CGF, Name: "__sanitizer_dtor_callback_fields", Ptr,
1703	PoisonSize);
1704	}
1705
1706	/// Poison base class with a trivial destructor.
1707	struct SanitizeDtorTrivialBase final : EHScopeStack::Cleanup {
1708	const CXXRecordDecl *BaseClass;
1709	bool BaseIsVirtual;
1710	SanitizeDtorTrivialBase(const CXXRecordDecl *Base, bool BaseIsVirtual)
1711	: BaseClass(Base), BaseIsVirtual(BaseIsVirtual) {}
1712
1713	void Emit(CodeGenFunction &CGF, Flags flags) override {
1714	const CXXRecordDecl *DerivedClass =
1715	cast<CXXMethodDecl>(Val: CGF.CurCodeDecl)->getParent();
1716
1717	Address Addr = CGF.GetAddressOfDirectBaseInCompleteClass(
1718	This: CGF.LoadCXXThisAddress(), Derived: DerivedClass, Base: BaseClass, BaseIsVirtual);
1719
1720	const ASTRecordLayout &BaseLayout =
1721	CGF.getContext().getASTRecordLayout(BaseClass);
1722	CharUnits BaseSize = BaseLayout.getSize();
1723
1724	if (!BaseSize.isPositive())
1725	return;
1726
1727	// Use the base class declaration location as inline DebugLocation. All
1728	// fields of the class are destroyed.
1729	DeclAsInlineDebugLocation InlineHere(CGF, *BaseClass);
1730	EmitSanitizerDtorFieldsCallback(CGF, Ptr: Addr.emitRawPointer(CGF),
1731	PoisonSize: BaseSize.getQuantity());
1732
1733	// Prevent the current stack frame from disappearing from the stack trace.
1734	CGF.CurFn->addFnAttr(Kind: "disable-tail-calls", Val: "true");
1735	}
1736	};
1737
1738	class SanitizeDtorFieldRange final : public EHScopeStack::Cleanup {
1739	const CXXDestructorDecl *Dtor;
1740	unsigned StartIndex;
1741	unsigned EndIndex;
1742
1743	public:
1744	SanitizeDtorFieldRange(const CXXDestructorDecl *Dtor, unsigned StartIndex,
1745	unsigned EndIndex)
1746	: Dtor(Dtor), StartIndex(StartIndex), EndIndex(EndIndex) {}
1747
1748	// Generate function call for handling object poisoning.
1749	// Disables tail call elimination, to prevent the current stack frame
1750	// from disappearing from the stack trace.
1751	void Emit(CodeGenFunction &CGF, Flags flags) override {
1752	const ASTContext &Context = CGF.getContext();
1753	const ASTRecordLayout &Layout =
1754	Context.getASTRecordLayout(D: Dtor->getParent());
1755
1756	// It's a first trivial field so it should be at the begining of a char,
1757	// still round up start offset just in case.
1758	CharUnits PoisonStart = Context.toCharUnitsFromBits(
1759	BitSize: Layout.getFieldOffset(FieldNo: StartIndex) + Context.getCharWidth() - 1);
1760	llvm::ConstantInt *OffsetSizePtr =
1761	llvm::ConstantInt::get(Ty: CGF.SizeTy, V: PoisonStart.getQuantity());
1762
1763	llvm::Value *OffsetPtr =
1764	CGF.Builder.CreateGEP(Ty: CGF.Int8Ty, Ptr: CGF.LoadCXXThis(), IdxList: OffsetSizePtr);
1765
1766	CharUnits PoisonEnd;
1767	if (EndIndex >= Layout.getFieldCount()) {
1768	PoisonEnd = Layout.getNonVirtualSize();
1769	} else {
1770	PoisonEnd =
1771	Context.toCharUnitsFromBits(BitSize: Layout.getFieldOffset(FieldNo: EndIndex));
1772	}
1773	CharUnits PoisonSize = PoisonEnd - PoisonStart;
1774	if (!PoisonSize.isPositive())
1775	return;
1776
1777	// Use the top field declaration location as inline DebugLocation.
1778	DeclAsInlineDebugLocation InlineHere(
1779	CGF, **std::next(Dtor->getParent()->field_begin(), StartIndex));
1780	EmitSanitizerDtorFieldsCallback(CGF, Ptr: OffsetPtr, PoisonSize: PoisonSize.getQuantity());
1781
1782	// Prevent the current stack frame from disappearing from the stack trace.
1783	CGF.CurFn->addFnAttr(Kind: "disable-tail-calls", Val: "true");
1784	}
1785	};
1786
1787	class SanitizeDtorVTable final : public EHScopeStack::Cleanup {
1788	const CXXDestructorDecl *Dtor;
1789
1790	public:
1791	SanitizeDtorVTable(const CXXDestructorDecl *Dtor) : Dtor(Dtor) {}
1792
1793	// Generate function call for handling vtable pointer poisoning.
1794	void Emit(CodeGenFunction &CGF, Flags flags) override {
1795	assert(Dtor->getParent()->isDynamicClass());
1796	(void)Dtor;
1797	// Poison vtable and vtable ptr if they exist for this class.
1798	llvm::Value *VTablePtr = CGF.LoadCXXThis();
1799
1800	// Pass in void pointer and size of region as arguments to runtime
1801	// function
1802	EmitSanitizerDtorCallback(CGF, Name: "__sanitizer_dtor_callback_vptr",
1803	Ptr: VTablePtr);
1804	}
1805	};
1806
1807	class SanitizeDtorCleanupBuilder {
1808	ASTContext &Context;
1809	EHScopeStack &EHStack;
1810	const CXXDestructorDecl *DD;
1811	std::optional<unsigned> StartIndex;
1812
1813	public:
1814	SanitizeDtorCleanupBuilder(ASTContext &Context, EHScopeStack &EHStack,
1815	const CXXDestructorDecl *DD)
1816	: Context(Context), EHStack(EHStack), DD(DD), StartIndex(std::nullopt) {}
1817	void PushCleanupForField(const FieldDecl *Field) {
1818	if (Field->isZeroSize(Ctx: Context))
1819	return;
1820	unsigned FieldIndex = Field->getFieldIndex();
1821	if (FieldHasTrivialDestructorBody(Context, Field)) {
1822	if (!StartIndex)
1823	StartIndex = FieldIndex;
1824	} else if (StartIndex) {
1825	EHStack.pushCleanup<SanitizeDtorFieldRange>(Kind: NormalAndEHCleanup, A: DD,
1826	A: *StartIndex, A: FieldIndex);
1827	StartIndex = std::nullopt;
1828	}
1829	}
1830	void End() {
1831	if (StartIndex)
1832	EHStack.pushCleanup<SanitizeDtorFieldRange>(Kind: NormalAndEHCleanup, A: DD,
1833	A: *StartIndex, A: -1);
1834	}
1835	};
1836	} // end anonymous namespace
1837
1838	/// Emit all code that comes at the end of class's
1839	/// destructor. This is to call destructors on members and base classes
1840	/// in reverse order of their construction.
1841	///
1842	/// For a deleting destructor, this also handles the case where a destroying
1843	/// operator delete completely overrides the definition.
1844	void CodeGenFunction::EnterDtorCleanups(const CXXDestructorDecl *DD,
1845	CXXDtorType DtorType) {
1846	assert((!DD->isTrivial() || DD->hasAttr<DLLExportAttr>()) &&
1847	"Should not emit dtor epilogue for non-exported trivial dtor!");
1848
1849	// The deleting-destructor phase just needs to call the appropriate
1850	// operator delete that Sema picked up.
1851	if (DtorType == Dtor_Deleting) {
1852	assert(DD->getOperatorDelete() &&
1853	"operator delete missing - EnterDtorCleanups");
1854	if (CXXStructorImplicitParamValue) {
1855	// If there is an implicit param to the deleting dtor, it's a boolean
1856	// telling whether this is a deleting destructor.
1857	if (DD->getOperatorDelete()->isDestroyingOperatorDelete())
1858	EmitConditionalDtorDeleteCall(CGF&: *this, ShouldDeleteCondition: CXXStructorImplicitParamValue,
1859	/*ReturnAfterDelete*/true);
1860	else
1861	EHStack.pushCleanup<CallDtorDeleteConditional>(
1862	Kind: NormalAndEHCleanup, A: CXXStructorImplicitParamValue);
1863	} else {
1864	if (DD->getOperatorDelete()->isDestroyingOperatorDelete()) {
1865	const CXXRecordDecl *ClassDecl = DD->getParent();
1866	EmitDeleteCall(DeleteFD: DD->getOperatorDelete(),
1867	Ptr: LoadThisForDtorDelete(CGF&: *this, DD),
1868	DeleteTy: getContext().getTagDeclType(ClassDecl));
1869	EmitBranchThroughCleanup(Dest: ReturnBlock);
1870	} else {
1871	EHStack.pushCleanup<CallDtorDelete>(Kind: NormalAndEHCleanup);
1872	}
1873	}
1874	return;
1875	}
1876
1877	const CXXRecordDecl *ClassDecl = DD->getParent();
1878
1879	// Unions have no bases and do not call field destructors.
1880	if (ClassDecl->isUnion())
1881	return;
1882
1883	// The complete-destructor phase just destructs all the virtual bases.
1884	if (DtorType == Dtor_Complete) {
1885	// Poison the vtable pointer such that access after the base
1886	// and member destructors are invoked is invalid.
1887	if (CGM.getCodeGenOpts().SanitizeMemoryUseAfterDtor &&
1888	SanOpts.has(K: SanitizerKind::Memory) && ClassDecl->getNumVBases() &&
1889	ClassDecl->isPolymorphic())
1890	EHStack.pushCleanup<SanitizeDtorVTable>(Kind: NormalAndEHCleanup, A: DD);
1891
1892	// We push them in the forward order so that they'll be popped in
1893	// the reverse order.
1894	for (const auto &Base : ClassDecl->vbases()) {
1895	auto *BaseClassDecl =
1896	cast<CXXRecordDecl>(Base.getType()->castAs<RecordType>()->getDecl());
1897
1898	if (BaseClassDecl->hasTrivialDestructor()) {
1899	// Under SanitizeMemoryUseAfterDtor, poison the trivial base class
1900	// memory. For non-trival base classes the same is done in the class
1901	// destructor.
1902	if (CGM.getCodeGenOpts().SanitizeMemoryUseAfterDtor &&
1903	SanOpts.has(SanitizerKind::Memory) && !BaseClassDecl->isEmpty())
1904	EHStack.pushCleanup<SanitizeDtorTrivialBase>(NormalAndEHCleanup,
1905	BaseClassDecl,
1906	/*BaseIsVirtual*/ true);
1907	} else {
1908	EHStack.pushCleanup<CallBaseDtor>(NormalAndEHCleanup, BaseClassDecl,
1909	/*BaseIsVirtual*/ true);
1910	}
1911	}
1912
1913	return;
1914	}
1915
1916	assert(DtorType == Dtor_Base);
1917	// Poison the vtable pointer if it has no virtual bases, but inherits
1918	// virtual functions.
1919	if (CGM.getCodeGenOpts().SanitizeMemoryUseAfterDtor &&
1920	SanOpts.has(K: SanitizerKind::Memory) && !ClassDecl->getNumVBases() &&
1921	ClassDecl->isPolymorphic())
1922	EHStack.pushCleanup<SanitizeDtorVTable>(Kind: NormalAndEHCleanup, A: DD);
1923
1924	// Destroy non-virtual bases.
1925	for (const auto &Base : ClassDecl->bases()) {
1926	// Ignore virtual bases.
1927	if (Base.isVirtual())
1928	continue;
1929
1930	CXXRecordDecl *BaseClassDecl = Base.getType()->getAsCXXRecordDecl();
1931
1932	if (BaseClassDecl->hasTrivialDestructor()) {
1933	if (CGM.getCodeGenOpts().SanitizeMemoryUseAfterDtor &&
1934	SanOpts.has(SanitizerKind::Memory) && !BaseClassDecl->isEmpty())
1935	EHStack.pushCleanup<SanitizeDtorTrivialBase>(NormalAndEHCleanup,
1936	BaseClassDecl,
1937	/*BaseIsVirtual*/ false);
1938	} else {
1939	EHStack.pushCleanup<CallBaseDtor>(NormalAndEHCleanup, BaseClassDecl,
1940	/*BaseIsVirtual*/ false);
1941	}
1942	}
1943
1944	// Poison fields such that access after their destructors are
1945	// invoked, and before the base class destructor runs, is invalid.
1946	bool SanitizeFields = CGM.getCodeGenOpts().SanitizeMemoryUseAfterDtor &&
1947	SanOpts.has(K: SanitizerKind::Memory);
1948	SanitizeDtorCleanupBuilder SanitizeBuilder(getContext(), EHStack, DD);
1949
1950	// Destroy direct fields.
1951	for (const auto *Field : ClassDecl->fields()) {
1952	if (SanitizeFields)
1953	SanitizeBuilder.PushCleanupForField(Field);
1954
1955	QualType type = Field->getType();
1956	QualType::DestructionKind dtorKind = type.isDestructedType();
1957	if (!dtorKind)
1958	continue;
1959
1960	// Anonymous union members do not have their destructors called.
1961	const RecordType *RT = type->getAsUnionType();
1962	if (RT && RT->getDecl()->isAnonymousStructOrUnion())
1963	continue;
1964
1965	CleanupKind cleanupKind = getCleanupKind(dtorKind);
1966	EHStack.pushCleanup<DestroyField>(
1967	cleanupKind, Field, getDestroyer(dtorKind), cleanupKind & EHCleanup);
1968	}
1969
1970	if (SanitizeFields)
1971	SanitizeBuilder.End();
1972	}
1973
1974	/// EmitCXXAggrConstructorCall - Emit a loop to call a particular
1975	/// constructor for each of several members of an array.
1976	///
1977	/// \param ctor the constructor to call for each element
1978	/// \param arrayType the type of the array to initialize
1979	/// \param arrayBegin an arrayType*
1980	/// \param zeroInitialize true if each element should be
1981	/// zero-initialized before it is constructed
1982	void CodeGenFunction::EmitCXXAggrConstructorCall(
1983	const CXXConstructorDecl *ctor, const ArrayType *arrayType,
1984	Address arrayBegin, const CXXConstructExpr *E, bool NewPointerIsChecked,
1985	bool zeroInitialize) {
1986	QualType elementType;
1987	llvm::Value *numElements =
1988	emitArrayLength(arrayType, baseType&: elementType, addr&: arrayBegin);
1989
1990	EmitCXXAggrConstructorCall(D: ctor, NumElements: numElements, ArrayPtr: arrayBegin, E,
1991	NewPointerIsChecked, ZeroInitialization: zeroInitialize);
1992	}
1993
1994	/// EmitCXXAggrConstructorCall - Emit a loop to call a particular
1995	/// constructor for each of several members of an array.
1996	///
1997	/// \param ctor the constructor to call for each element
1998	/// \param numElements the number of elements in the array;
1999	/// may be zero
2000	/// \param arrayBase a T*, where T is the type constructed by ctor
2001	/// \param zeroInitialize true if each element should be
2002	/// zero-initialized before it is constructed
2003	void CodeGenFunction::EmitCXXAggrConstructorCall(const CXXConstructorDecl *ctor,
2004	llvm::Value *numElements,
2005	Address arrayBase,
2006	const CXXConstructExpr *E,
2007	bool NewPointerIsChecked,
2008	bool zeroInitialize) {
2009	// It's legal for numElements to be zero. This can happen both
2010	// dynamically, because x can be zero in 'new A[x]', and statically,
2011	// because of GCC extensions that permit zero-length arrays. There
2012	// are probably legitimate places where we could assume that this
2013	// doesn't happen, but it's not clear that it's worth it.
2014	llvm::BranchInst *zeroCheckBranch = nullptr;
2015
2016	// Optimize for a constant count.
2017	llvm::ConstantInt *constantCount
2018	= dyn_cast<llvm::ConstantInt>(Val: numElements);
2019	if (constantCount) {
2020	// Just skip out if the constant count is zero.
2021	if (constantCount->isZero()) return;
2022
2023	// Otherwise, emit the check.
2024	} else {
2025	llvm::BasicBlock *loopBB = createBasicBlock(name: "new.ctorloop");
2026	llvm::Value *iszero = Builder.CreateIsNull(Arg: numElements, Name: "isempty");
2027	zeroCheckBranch = Builder.CreateCondBr(Cond: iszero, True: loopBB, False: loopBB);
2028	EmitBlock(BB: loopBB);
2029	}
2030
2031	// Find the end of the array.
2032	llvm::Type *elementType = arrayBase.getElementType();
2033	llvm::Value *arrayBegin = arrayBase.emitRawPointer(CGF&: *this);
2034	llvm::Value *arrayEnd = Builder.CreateInBoundsGEP(
2035	Ty: elementType, Ptr: arrayBegin, IdxList: numElements, Name: "arrayctor.end");
2036
2037	// Enter the loop, setting up a phi for the current location to initialize.
2038	llvm::BasicBlock *entryBB = Builder.GetInsertBlock();
2039	llvm::BasicBlock *loopBB = createBasicBlock(name: "arrayctor.loop");
2040	EmitBlock(BB: loopBB);
2041	llvm::PHINode *cur = Builder.CreatePHI(Ty: arrayBegin->getType(), NumReservedValues: 2,
2042	Name: "arrayctor.cur");
2043	cur->addIncoming(V: arrayBegin, BB: entryBB);
2044
2045	// Inside the loop body, emit the constructor call on the array element.
2046
2047	// The alignment of the base, adjusted by the size of a single element,
2048	// provides a conservative estimate of the alignment of every element.
2049	// (This assumes we never start tracking offsetted alignments.)
2050	//
2051	// Note that these are complete objects and so we don't need to
2052	// use the non-virtual size or alignment.
2053	QualType type = getContext().getTypeDeclType(Decl: ctor->getParent());
2054	CharUnits eltAlignment =
2055	arrayBase.getAlignment()
2056	.alignmentOfArrayElement(elementSize: getContext().getTypeSizeInChars(T: type));
2057	Address curAddr = Address(cur, elementType, eltAlignment);
2058
2059	// Zero initialize the storage, if requested.
2060	if (zeroInitialize)
2061	EmitNullInitialization(DestPtr: curAddr, Ty: type);
2062
2063	// C++ [class.temporary]p4:
2064	// There are two contexts in which temporaries are destroyed at a different
2065	// point than the end of the full-expression. The first context is when a
2066	// default constructor is called to initialize an element of an array.
2067	// If the constructor has one or more default arguments, the destruction of
2068	// every temporary created in a default argument expression is sequenced
2069	// before the construction of the next array element, if any.
2070
2071	{
2072	RunCleanupsScope Scope(*this);
2073
2074	// Evaluate the constructor and its arguments in a regular
2075	// partial-destroy cleanup.
2076	if (getLangOpts().Exceptions &&
2077	!ctor->getParent()->hasTrivialDestructor()) {
2078	Destroyer *destroyer = destroyCXXObject;
2079	pushRegularPartialArrayCleanup(arrayBegin, arrayEnd: cur, elementType: type, elementAlignment: eltAlignment,
2080	destroyer: *destroyer);
2081	}
2082	auto currAVS = AggValueSlot::forAddr(
2083	addr: curAddr, quals: type.getQualifiers(), isDestructed: AggValueSlot::IsDestructed,
2084	needsGC: AggValueSlot::DoesNotNeedGCBarriers, isAliased: AggValueSlot::IsNotAliased,
2085	mayOverlap: AggValueSlot::DoesNotOverlap, isZeroed: AggValueSlot::IsNotZeroed,
2086	isChecked: NewPointerIsChecked ? AggValueSlot::IsSanitizerChecked
2087	: AggValueSlot::IsNotSanitizerChecked);
2088	EmitCXXConstructorCall(ctor, Ctor_Complete, /*ForVirtualBase=*/false,
2089	/*Delegating=*/false, currAVS, E);
2090	}
2091
2092	// Go to the next element.
2093	llvm::Value *next = Builder.CreateInBoundsGEP(
2094	Ty: elementType, Ptr: cur, IdxList: llvm::ConstantInt::get(Ty: SizeTy, V: 1), Name: "arrayctor.next");
2095	cur->addIncoming(V: next, BB: Builder.GetInsertBlock());
2096
2097	// Check whether that's the end of the loop.
2098	llvm::Value *done = Builder.CreateICmpEQ(LHS: next, RHS: arrayEnd, Name: "arrayctor.done");
2099	llvm::BasicBlock *contBB = createBasicBlock(name: "arrayctor.cont");
2100	Builder.CreateCondBr(Cond: done, True: contBB, False: loopBB);
2101
2102	// Patch the earlier check to skip over the loop.
2103	if (zeroCheckBranch) zeroCheckBranch->setSuccessor(idx: 0, NewSucc: contBB);
2104
2105	EmitBlock(BB: contBB);
2106	}
2107
2108	void CodeGenFunction::destroyCXXObject(CodeGenFunction &CGF,
2109	Address addr,
2110	QualType type) {
2111	const RecordType *rtype = type->castAs<RecordType>();
2112	const CXXRecordDecl *record = cast<CXXRecordDecl>(Val: rtype->getDecl());
2113	const CXXDestructorDecl *dtor = record->getDestructor();
2114	assert(!dtor->isTrivial());
2115	CGF.EmitCXXDestructorCall(D: dtor, Type: Dtor_Complete, /*for vbase*/ ForVirtualBase: false,
2116	/*Delegating=*/false, This: addr, ThisTy: type);
2117	}
2118
2119	void CodeGenFunction::EmitCXXConstructorCall(const CXXConstructorDecl *D,
2120	CXXCtorType Type,
2121	bool ForVirtualBase,
2122	bool Delegating,
2123	AggValueSlot ThisAVS,
2124	const CXXConstructExpr *E) {
2125	CallArgList Args;
2126	Address This = ThisAVS.getAddress();
2127	LangAS SlotAS = ThisAVS.getQualifiers().getAddressSpace();
2128	LangAS ThisAS = D->getFunctionObjectParameterType().getAddressSpace();
2129	llvm::Value *ThisPtr =
2130	getAsNaturalPointerTo(Addr: This, PointeeType: D->getThisType()->getPointeeType());
2131
2132	if (SlotAS != ThisAS) {
2133	unsigned TargetThisAS = getContext().getTargetAddressSpace(AS: ThisAS);
2134	llvm::Type *NewType =
2135	llvm::PointerType::get(C&: getLLVMContext(), AddressSpace: TargetThisAS);
2136	ThisPtr = getTargetHooks().performAddrSpaceCast(CGF&: *this, V: ThisPtr, SrcAddr: ThisAS,
2137	DestAddr: SlotAS, DestTy: NewType);
2138	}
2139
2140	// Push the this ptr.
2141	Args.add(rvalue: RValue::get(V: ThisPtr), type: D->getThisType());
2142
2143	// If this is a trivial constructor, emit a memcpy now before we lose
2144	// the alignment information on the argument.
2145	// FIXME: It would be better to preserve alignment information into CallArg.
2146	if (isMemcpyEquivalentSpecialMember(D)) {
2147	assert(E->getNumArgs() == 1 && "unexpected argcount for trivial ctor");
2148
2149	const Expr *Arg = E->getArg(Arg: 0);
2150	LValue Src = EmitLValue(E: Arg);
2151	QualType DestTy = getContext().getTypeDeclType(Decl: D->getParent());
2152	LValue Dest = MakeAddrLValue(Addr: This, T: DestTy);
2153	EmitAggregateCopyCtor(Dest, Src, MayOverlap: ThisAVS.mayOverlap());
2154	return;
2155	}
2156
2157	// Add the rest of the user-supplied arguments.
2158	const FunctionProtoType *FPT = D->getType()->castAs<FunctionProtoType>();
2159	EvaluationOrder Order = E->isListInitialization()
2160	? EvaluationOrder::ForceLeftToRight
2161	: EvaluationOrder::Default;
2162	EmitCallArgs(Args, Prototype: FPT, ArgRange: E->arguments(), AC: E->getConstructor(),
2163	/*ParamsToSkip*/ 0, Order);
2164
2165	EmitCXXConstructorCall(D, Type, ForVirtualBase, Delegating, This, Args,
2166	ThisAVS.mayOverlap(), E->getExprLoc(),
2167	ThisAVS.isSanitizerChecked());
2168	}
2169
2170	static bool canEmitDelegateCallArgs(CodeGenFunction &CGF,
2171	const CXXConstructorDecl *Ctor,
2172	CXXCtorType Type, CallArgList &Args) {
2173	// We can't forward a variadic call.
2174	if (Ctor->isVariadic())
2175	return false;
2176
2177	if (CGF.getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee()) {
2178	// If the parameters are callee-cleanup, it's not safe to forward.
2179	for (auto *P : Ctor->parameters())
2180	if (P->needsDestruction(CGF.getContext()))
2181	return false;
2182
2183	// Likewise if they're inalloca.
2184	const CGFunctionInfo &Info =
2185	CGF.CGM.getTypes().arrangeCXXConstructorCall(Args, D: Ctor, CtorKind: Type, ExtraPrefixArgs: 0, ExtraSuffixArgs: 0);
2186	if (Info.usesInAlloca())
2187	return false;
2188	}
2189
2190	// Anything else should be OK.
2191	return true;
2192	}
2193
2194	void CodeGenFunction::EmitCXXConstructorCall(const CXXConstructorDecl *D,
2195	CXXCtorType Type,
2196	bool ForVirtualBase,
2197	bool Delegating,
2198	Address This,
2199	CallArgList &Args,
2200	AggValueSlot::Overlap_t Overlap,
2201	SourceLocation Loc,
2202	bool NewPointerIsChecked) {
2203	const CXXRecordDecl *ClassDecl = D->getParent();
2204
2205	if (!NewPointerIsChecked)
2206	EmitTypeCheck(TCK: CodeGenFunction::TCK_ConstructorCall, Loc, Addr: This,
2207	Type: getContext().getRecordType(ClassDecl), Alignment: CharUnits::Zero());
2208
2209	if (D->isTrivial() && D->isDefaultConstructor()) {
2210	assert(Args.size() == 1 && "trivial default ctor with args");
2211	return;
2212	}
2213
2214	// If this is a trivial constructor, just emit what's needed. If this is a
2215	// union copy constructor, we must emit a memcpy, because the AST does not
2216	// model that copy.
2217	if (isMemcpyEquivalentSpecialMember(D)) {
2218	assert(Args.size() == 2 && "unexpected argcount for trivial ctor");
2219	QualType SrcTy = D->getParamDecl(0)->getType().getNonReferenceType();
2220	Address Src = makeNaturalAddressForPointer(
2221	Ptr: Args[1].getRValue(CGF&: *this).getScalarVal(), T: SrcTy);
2222	LValue SrcLVal = MakeAddrLValue(Addr: Src, T: SrcTy);
2223	QualType DestTy = getContext().getTypeDeclType(ClassDecl);
2224	LValue DestLVal = MakeAddrLValue(Addr: This, T: DestTy);
2225	EmitAggregateCopyCtor(Dest: DestLVal, Src: SrcLVal, MayOverlap: Overlap);
2226	return;
2227	}
2228
2229	bool PassPrototypeArgs = true;
2230	// Check whether we can actually emit the constructor before trying to do so.
2231	if (auto Inherited = D->getInheritedConstructor()) {
2232	PassPrototypeArgs = getTypes().inheritingCtorHasParams(Inherited, Type);
2233	if (PassPrototypeArgs && !canEmitDelegateCallArgs(CGF&: *this, Ctor: D, Type, Args)) {
2234	EmitInlinedInheritingCXXConstructorCall(Ctor: D, CtorType: Type, ForVirtualBase,
2235	Delegating, Args);
2236	return;
2237	}
2238	}
2239
2240	// Insert any ABI-specific implicit constructor arguments.
2241	CGCXXABI::AddedStructorArgCounts ExtraArgs =
2242	CGM.getCXXABI().addImplicitConstructorArgs(CGF&: *this, D, Type, ForVirtualBase,
2243	Delegating, Args);
2244
2245	// Emit the call.
2246	llvm::Constant *CalleePtr = CGM.getAddrOfCXXStructor(GD: GlobalDecl(D, Type));
2247	const CGFunctionInfo &Info = CGM.getTypes().arrangeCXXConstructorCall(
2248	Args, D, CtorKind: Type, ExtraPrefixArgs: ExtraArgs.Prefix, ExtraSuffixArgs: ExtraArgs.Suffix, PassProtoArgs: PassPrototypeArgs);
2249	CGCallee Callee = CGCallee::forDirect(functionPtr: CalleePtr, abstractInfo: GlobalDecl(D, Type));
2250	EmitCall(CallInfo: Info, Callee, ReturnValue: ReturnValueSlot(), Args, callOrInvoke: nullptr, IsMustTail: false, Loc);
2251
2252	// Generate vtable assumptions if we're constructing a complete object
2253	// with a vtable. We don't do this for base subobjects for two reasons:
2254	// first, it's incorrect for classes with virtual bases, and second, we're
2255	// about to overwrite the vptrs anyway.
2256	// We also have to make sure if we can refer to vtable:
2257	// - Otherwise we can refer to vtable if it's safe to speculatively emit.
2258	// FIXME: If vtable is used by ctor/dtor, or if vtable is external and we are
2259	// sure that definition of vtable is not hidden,
2260	// then we are always safe to refer to it.
2261	// FIXME: It looks like InstCombine is very inefficient on dealing with
2262	// assumes. Make assumption loads require -fstrict-vtable-pointers temporarily.
2263	if (CGM.getCodeGenOpts().OptimizationLevel > 0 &&
2264	ClassDecl->isDynamicClass() && Type != Ctor_Base &&
2265	CGM.getCXXABI().canSpeculativelyEmitVTable(RD: ClassDecl) &&
2266	CGM.getCodeGenOpts().StrictVTablePointers)
2267	EmitVTableAssumptionLoads(ClassDecl, This);
2268	}
2269
2270	void CodeGenFunction::EmitInheritedCXXConstructorCall(
2271	const CXXConstructorDecl *D, bool ForVirtualBase, Address This,
2272	bool InheritedFromVBase, const CXXInheritedCtorInitExpr *E) {
2273	CallArgList Args;
2274	CallArg ThisArg(RValue::get(V: getAsNaturalPointerTo(
2275	Addr: This, PointeeType: D->getThisType()->getPointeeType())),
2276	D->getThisType());
2277
2278	// Forward the parameters.
2279	if (InheritedFromVBase &&
2280	CGM.getTarget().getCXXABI().hasConstructorVariants()) {
2281	// Nothing to do; this construction is not responsible for constructing
2282	// the base class containing the inherited constructor.
2283	// FIXME: Can we just pass undef's for the remaining arguments if we don't
2284	// have constructor variants?
2285	Args.push_back(Elt: ThisArg);
2286	} else if (!CXXInheritedCtorInitExprArgs.empty()) {
2287	// The inheriting constructor was inlined; just inject its arguments.
2288	assert(CXXInheritedCtorInitExprArgs.size() >= D->getNumParams() &&
2289	"wrong number of parameters for inherited constructor call");
2290	Args = CXXInheritedCtorInitExprArgs;
2291	Args[0] = ThisArg;
2292	} else {
2293	// The inheriting constructor was not inlined. Emit delegating arguments.
2294	Args.push_back(Elt: ThisArg);
2295	const auto *OuterCtor = cast<CXXConstructorDecl>(Val: CurCodeDecl);
2296	assert(OuterCtor->getNumParams() == D->getNumParams());
2297	assert(!OuterCtor->isVariadic() && "should have been inlined");
2298
2299	for (const auto *Param : OuterCtor->parameters()) {
2300	assert(getContext().hasSameUnqualifiedType(
2301	OuterCtor->getParamDecl(Param->getFunctionScopeIndex())->getType(),
2302	Param->getType()));
2303	EmitDelegateCallArg(Args, Param, E->getLocation());
2304
2305	// Forward __attribute__(pass_object_size).
2306	if (Param->hasAttr<PassObjectSizeAttr>()) {
2307	auto *POSParam = SizeArguments[Param];
2308	assert(POSParam && "missing pass_object_size value for forwarding");
2309	EmitDelegateCallArg(Args, POSParam, E->getLocation());
2310	}
2311	}
2312	}
2313
2314	EmitCXXConstructorCall(D, Type: Ctor_Base, ForVirtualBase, /*Delegating*/false,
2315	This, Args, Overlap: AggValueSlot::MayOverlap,
2316	Loc: E->getLocation(), /*NewPointerIsChecked*/true);
2317	}
2318
2319	void CodeGenFunction::EmitInlinedInheritingCXXConstructorCall(
2320	const CXXConstructorDecl *Ctor, CXXCtorType CtorType, bool ForVirtualBase,
2321	bool Delegating, CallArgList &Args) {
2322	GlobalDecl GD(Ctor, CtorType);
2323	InlinedInheritingConstructorScope Scope(*this, GD);
2324	ApplyInlineDebugLocation DebugScope(*this, GD);
2325	RunCleanupsScope RunCleanups(*this);
2326
2327	// Save the arguments to be passed to the inherited constructor.
2328	CXXInheritedCtorInitExprArgs = Args;
2329
2330	FunctionArgList Params;
2331	QualType RetType = BuildFunctionArgList(GD: CurGD, Args&: Params);
2332	FnRetTy = RetType;
2333
2334	// Insert any ABI-specific implicit constructor arguments.
2335	CGM.getCXXABI().addImplicitConstructorArgs(CGF&: *this, D: Ctor, Type: CtorType,
2336	ForVirtualBase, Delegating, Args);
2337
2338	// Emit a simplified prolog. We only need to emit the implicit params.
2339	assert(Args.size() >= Params.size() && "too few arguments for call");
2340	for (unsigned I = 0, N = Args.size(); I != N; ++I) {
2341	if (I < Params.size() && isa<ImplicitParamDecl>(Val: Params[I])) {
2342	const RValue &RV = Args[I].getRValue(CGF&: *this);
2343	assert(!RV.isComplex() && "complex indirect params not supported");
2344	ParamValue Val = RV.isScalar()
2345	? ParamValue::forDirect(value: RV.getScalarVal())
2346	: ParamValue::forIndirect(addr: RV.getAggregateAddress());
2347	EmitParmDecl(D: *Params[I], Arg: Val, ArgNo: I + 1);
2348	}
2349	}
2350
2351	// Create a return value slot if the ABI implementation wants one.
2352	// FIXME: This is dumb, we should ask the ABI not to try to set the return
2353	// value instead.
2354	if (!RetType->isVoidType())
2355	ReturnValue = CreateIRTemp(T: RetType, Name: "retval.inhctor");
2356
2357	CGM.getCXXABI().EmitInstanceFunctionProlog(CGF&: *this);
2358	CXXThisValue = CXXABIThisValue;
2359
2360	// Directly emit the constructor initializers.
2361	EmitCtorPrologue(CD: Ctor, CtorType, Args&: Params);
2362	}
2363
2364	void CodeGenFunction::EmitVTableAssumptionLoad(const VPtr &Vptr, Address This) {
2365	llvm::Value *VTableGlobal =
2366	CGM.getCXXABI().getVTableAddressPoint(Base: Vptr.Base, VTableClass: Vptr.VTableClass);
2367	if (!VTableGlobal)
2368	return;
2369
2370	// We can just use the base offset in the complete class.
2371	CharUnits NonVirtualOffset = Vptr.Base.getBaseOffset();
2372
2373	if (!NonVirtualOffset.isZero())
2374	This =
2375	ApplyNonVirtualAndVirtualOffset(CGF&: *this, addr: This, nonVirtualOffset: NonVirtualOffset, virtualOffset: nullptr,
2376	derivedClass: Vptr.VTableClass, nearestVBase: Vptr.NearestVBase);
2377
2378	llvm::Value *VPtrValue =
2379	GetVTablePtr(This, VTableTy: VTableGlobal->getType(), VTableClass: Vptr.VTableClass);
2380	llvm::Value *Cmp =
2381	Builder.CreateICmpEQ(LHS: VPtrValue, RHS: VTableGlobal, Name: "cmp.vtables");
2382	Builder.CreateAssumption(Cond: Cmp);
2383	}
2384
2385	void CodeGenFunction::EmitVTableAssumptionLoads(const CXXRecordDecl *ClassDecl,
2386	Address This) {
2387	if (CGM.getCXXABI().doStructorsInitializeVPtrs(VTableClass: ClassDecl))
2388	for (const VPtr &Vptr : getVTablePointers(VTableClass: ClassDecl))
2389	EmitVTableAssumptionLoad(Vptr, This);
2390	}
2391
2392	void
2393	CodeGenFunction::EmitSynthesizedCXXCopyCtorCall(const CXXConstructorDecl *D,
2394	Address This, Address Src,
2395	const CXXConstructExpr *E) {
2396	const FunctionProtoType *FPT = D->getType()->castAs<FunctionProtoType>();
2397
2398	CallArgList Args;
2399
2400	// Push the this ptr.
2401	Args.add(rvalue: RValue::get(V: getAsNaturalPointerTo(Addr: This, PointeeType: D->getThisType())),
2402	type: D->getThisType());
2403
2404	// Push the src ptr.
2405	QualType QT = *(FPT->param_type_begin());
2406	llvm::Type *t = CGM.getTypes().ConvertType(T: QT);
2407	llvm::Value *Val = getAsNaturalPointerTo(Addr: Src, PointeeType: D->getThisType());
2408	llvm::Value *SrcVal = Builder.CreateBitCast(V: Val, DestTy: t);
2409	Args.add(rvalue: RValue::get(V: SrcVal), type: QT);
2410
2411	// Skip over first argument (Src).
2412	EmitCallArgs(Args, Prototype: FPT, ArgRange: drop_begin(E->arguments(), 1), AC: E->getConstructor(),
2413	/*ParamsToSkip*/ 1);
2414
2415	EmitCXXConstructorCall(D, Ctor_Complete, /*ForVirtualBase*/false,
2416	/*Delegating*/false, This, Args,
2417	AggValueSlot::MayOverlap, E->getExprLoc(),
2418	/*NewPointerIsChecked*/false);
2419	}
2420
2421	void
2422	CodeGenFunction::EmitDelegateCXXConstructorCall(const CXXConstructorDecl *Ctor,
2423	CXXCtorType CtorType,
2424	const FunctionArgList &Args,
2425	SourceLocation Loc) {
2426	CallArgList DelegateArgs;
2427
2428	FunctionArgList::const_iterator I = Args.begin(), E = Args.end();
2429	assert(I != E && "no parameters to constructor");
2430
2431	// this
2432	Address This = LoadCXXThisAddress();
2433	DelegateArgs.add(rvalue: RValue::get(getAsNaturalPointerTo(
2434	Addr: This, PointeeType: (*I)->getType()->getPointeeType())),
2435	type: (*I)->getType());
2436	++I;
2437
2438	// FIXME: The location of the VTT parameter in the parameter list is
2439	// specific to the Itanium ABI and shouldn't be hardcoded here.
2440	if (CGM.getCXXABI().NeedsVTTParameter(GD: CurGD)) {
2441	assert(I != E && "cannot skip vtt parameter, already done with args");
2442	assert((*I)->getType()->isPointerType() &&
2443	"skipping parameter not of vtt type");
2444	++I;
2445	}
2446
2447	// Explicit arguments.
2448	for (; I != E; ++I) {
2449	const VarDecl *param = *I;
2450	// FIXME: per-argument source location
2451	EmitDelegateCallArg(args&: DelegateArgs, param, loc: Loc);
2452	}
2453
2454	EmitCXXConstructorCall(D: Ctor, Type: CtorType, /*ForVirtualBase=*/false,
2455	/*Delegating=*/true, This, Args&: DelegateArgs,
2456	Overlap: AggValueSlot::MayOverlap, Loc,
2457	/*NewPointerIsChecked=*/true);
2458	}
2459
2460	namespace {
2461	struct CallDelegatingCtorDtor final : EHScopeStack::Cleanup {
2462	const CXXDestructorDecl *Dtor;
2463	Address Addr;
2464	CXXDtorType Type;
2465
2466	CallDelegatingCtorDtor(const CXXDestructorDecl *D, Address Addr,
2467	CXXDtorType Type)
2468	: Dtor(D), Addr(Addr), Type(Type) {}
2469
2470	void Emit(CodeGenFunction &CGF, Flags flags) override {
2471	// We are calling the destructor from within the constructor.
2472	// Therefore, "this" should have the expected type.
2473	QualType ThisTy = Dtor->getFunctionObjectParameterType();
2474	CGF.EmitCXXDestructorCall(D: Dtor, Type, /*ForVirtualBase=*/false,
2475	/*Delegating=*/true, This: Addr, ThisTy);
2476	}
2477	};
2478	} // end anonymous namespace
2479
2480	void
2481	CodeGenFunction::EmitDelegatingCXXConstructorCall(const CXXConstructorDecl *Ctor,
2482	const FunctionArgList &Args) {
2483	assert(Ctor->isDelegatingConstructor());
2484
2485	Address ThisPtr = LoadCXXThisAddress();
2486
2487	AggValueSlot AggSlot =
2488	AggValueSlot::forAddr(addr: ThisPtr, quals: Qualifiers(),
2489	isDestructed: AggValueSlot::IsDestructed,
2490	needsGC: AggValueSlot::DoesNotNeedGCBarriers,
2491	isAliased: AggValueSlot::IsNotAliased,
2492	mayOverlap: AggValueSlot::MayOverlap,
2493	isZeroed: AggValueSlot::IsNotZeroed,
2494	// Checks are made by the code that calls constructor.
2495	isChecked: AggValueSlot::IsSanitizerChecked);
2496
2497	EmitAggExpr(E: Ctor->init_begin()[0]->getInit(), AS: AggSlot);
2498
2499	const CXXRecordDecl *ClassDecl = Ctor->getParent();
2500	if (CGM.getLangOpts().Exceptions && !ClassDecl->hasTrivialDestructor()) {
2501	CXXDtorType Type =
2502	CurGD.getCtorType() == Ctor_Complete ? Dtor_Complete : Dtor_Base;
2503
2504	EHStack.pushCleanup<CallDelegatingCtorDtor>(Kind: EHCleanup,
2505	A: ClassDecl->getDestructor(),
2506	A: ThisPtr, A: Type);
2507	}
2508	}
2509
2510	void CodeGenFunction::EmitCXXDestructorCall(const CXXDestructorDecl *DD,
2511	CXXDtorType Type,
2512	bool ForVirtualBase,
2513	bool Delegating, Address This,
2514	QualType ThisTy) {
2515	CGM.getCXXABI().EmitDestructorCall(CGF&: *this, DD, Type, ForVirtualBase,
2516	Delegating, This, ThisTy);
2517	}
2518
2519	namespace {
2520	struct CallLocalDtor final : EHScopeStack::Cleanup {
2521	const CXXDestructorDecl *Dtor;
2522	Address Addr;
2523	QualType Ty;
2524
2525	CallLocalDtor(const CXXDestructorDecl *D, Address Addr, QualType Ty)
2526	: Dtor(D), Addr(Addr), Ty(Ty) {}
2527
2528	void Emit(CodeGenFunction &CGF, Flags flags) override {
2529	CGF.EmitCXXDestructorCall(Dtor, Dtor_Complete,
2530	/*ForVirtualBase=*/false,
2531	/*Delegating=*/false, Addr, Ty);
2532	}
2533	};
2534	} // end anonymous namespace
2535
2536	void CodeGenFunction::PushDestructorCleanup(const CXXDestructorDecl *D,
2537	QualType T, Address Addr) {
2538	EHStack.pushCleanup<CallLocalDtor>(Kind: NormalAndEHCleanup, A: D, A: Addr, A: T);
2539	}
2540
2541	void CodeGenFunction::PushDestructorCleanup(QualType T, Address Addr) {
2542	CXXRecordDecl *ClassDecl = T->getAsCXXRecordDecl();
2543	if (!ClassDecl) return;
2544	if (ClassDecl->hasTrivialDestructor()) return;
2545
2546	const CXXDestructorDecl *D = ClassDecl->getDestructor();
2547	assert(D && D->isUsed() && "destructor not marked as used!");
2548	PushDestructorCleanup(D, T, Addr);
2549	}
2550
2551	void CodeGenFunction::InitializeVTablePointer(const VPtr &Vptr) {
2552	// Compute the address point.
2553	llvm::Value *VTableAddressPoint =
2554	CGM.getCXXABI().getVTableAddressPointInStructor(
2555	CGF&: *this, RD: Vptr.VTableClass, Base: Vptr.Base, NearestVBase: Vptr.NearestVBase);
2556
2557	if (!VTableAddressPoint)
2558	return;
2559
2560	// Compute where to store the address point.
2561	llvm::Value *VirtualOffset = nullptr;
2562	CharUnits NonVirtualOffset = CharUnits::Zero();
2563
2564	if (CGM.getCXXABI().isVirtualOffsetNeededForVTableField(CGF&: *this, Vptr)) {
2565	// We need to use the virtual base offset offset because the virtual base
2566	// might have a different offset in the most derived class.
2567
2568	VirtualOffset = CGM.getCXXABI().GetVirtualBaseClassOffset(
2569	CGF&: *this, This: LoadCXXThisAddress(), ClassDecl: Vptr.VTableClass, BaseClassDecl: Vptr.NearestVBase);
2570	NonVirtualOffset = Vptr.OffsetFromNearestVBase;
2571	} else {
2572	// We can just use the base offset in the complete class.
2573	NonVirtualOffset = Vptr.Base.getBaseOffset();
2574	}
2575
2576	// Apply the offsets.
2577	Address VTableField = LoadCXXThisAddress();
2578	if (!NonVirtualOffset.isZero() || VirtualOffset)
2579	VTableField = ApplyNonVirtualAndVirtualOffset(
2580	CGF&: *this, addr: VTableField, nonVirtualOffset: NonVirtualOffset, virtualOffset: VirtualOffset, derivedClass: Vptr.VTableClass,
2581	nearestVBase: Vptr.NearestVBase);
2582
2583	// Finally, store the address point. Use the same LLVM types as the field to
2584	// support optimization.
2585	unsigned GlobalsAS = CGM.getDataLayout().getDefaultGlobalsAddressSpace();
2586	llvm::Type *PtrTy = llvm::PointerType::get(C&: CGM.getLLVMContext(), AddressSpace: GlobalsAS);
2587	// vtable field is derived from `this` pointer, therefore they should be in
2588	// the same addr space. Note that this might not be LLVM address space 0.
2589	VTableField = VTableField.withElementType(ElemTy: PtrTy);
2590
2591	llvm::StoreInst *Store = Builder.CreateStore(Val: VTableAddressPoint, Addr: VTableField);
2592	TBAAAccessInfo TBAAInfo = CGM.getTBAAVTablePtrAccessInfo(VTablePtrType: PtrTy);
2593	CGM.DecorateInstructionWithTBAA(Inst: Store, TBAAInfo);
2594	if (CGM.getCodeGenOpts().OptimizationLevel > 0 &&
2595	CGM.getCodeGenOpts().StrictVTablePointers)
2596	CGM.DecorateInstructionWithInvariantGroup(I: Store, RD: Vptr.VTableClass);
2597	}
2598
2599	CodeGenFunction::VPtrsVector
2600	CodeGenFunction::getVTablePointers(const CXXRecordDecl *VTableClass) {
2601	CodeGenFunction::VPtrsVector VPtrsResult;
2602	VisitedVirtualBasesSetTy VBases;
2603	getVTablePointers(Base: BaseSubobject(VTableClass, CharUnits::Zero()),
2604	/*NearestVBase=*/nullptr,
2605	/*OffsetFromNearestVBase=*/CharUnits::Zero(),
2606	/*BaseIsNonVirtualPrimaryBase=*/false, VTableClass, VBases,
2607	vptrs&: VPtrsResult);
2608	return VPtrsResult;
2609	}
2610
2611	void CodeGenFunction::getVTablePointers(BaseSubobject Base,
2612	const CXXRecordDecl *NearestVBase,
2613	CharUnits OffsetFromNearestVBase,
2614	bool BaseIsNonVirtualPrimaryBase,
2615	const CXXRecordDecl *VTableClass,
2616	VisitedVirtualBasesSetTy &VBases,
2617	VPtrsVector &Vptrs) {
2618	// If this base is a non-virtual primary base the address point has already
2619	// been set.
2620	if (!BaseIsNonVirtualPrimaryBase) {
2621	// Initialize the vtable pointer for this base.
2622	VPtr Vptr = {.Base: Base, .NearestVBase: NearestVBase, .OffsetFromNearestVBase: OffsetFromNearestVBase, .VTableClass: VTableClass};
2623	Vptrs.push_back(Elt: Vptr);
2624	}
2625
2626	const CXXRecordDecl *RD = Base.getBase();
2627
2628	// Traverse bases.
2629	for (const auto &I : RD->bases()) {
2630	auto *BaseDecl =
2631	cast<CXXRecordDecl>(Val: I.getType()->castAs<RecordType>()->getDecl());
2632
2633	// Ignore classes without a vtable.
2634	if (!BaseDecl->isDynamicClass())
2635	continue;
2636
2637	CharUnits BaseOffset;
2638	CharUnits BaseOffsetFromNearestVBase;
2639	bool BaseDeclIsNonVirtualPrimaryBase;
2640
2641	if (I.isVirtual()) {
2642	// Check if we've visited this virtual base before.
2643	if (!VBases.insert(Ptr: BaseDecl).second)
2644	continue;
2645
2646	const ASTRecordLayout &Layout =
2647	getContext().getASTRecordLayout(VTableClass);
2648
2649	BaseOffset = Layout.getVBaseClassOffset(VBase: BaseDecl);
2650	BaseOffsetFromNearestVBase = CharUnits::Zero();
2651	BaseDeclIsNonVirtualPrimaryBase = false;
2652	} else {
2653	const ASTRecordLayout &Layout = getContext().getASTRecordLayout(RD);
2654
2655	BaseOffset = Base.getBaseOffset() + Layout.getBaseClassOffset(Base: BaseDecl);
2656	BaseOffsetFromNearestVBase =
2657	OffsetFromNearestVBase + Layout.getBaseClassOffset(Base: BaseDecl);
2658	BaseDeclIsNonVirtualPrimaryBase = Layout.getPrimaryBase() == BaseDecl;
2659	}
2660
2661	getVTablePointers(
2662	Base: BaseSubobject(BaseDecl, BaseOffset),
2663	NearestVBase: I.isVirtual() ? BaseDecl : NearestVBase, OffsetFromNearestVBase: BaseOffsetFromNearestVBase,
2664	BaseIsNonVirtualPrimaryBase: BaseDeclIsNonVirtualPrimaryBase, VTableClass, VBases, Vptrs);
2665	}
2666	}
2667
2668	void CodeGenFunction::InitializeVTablePointers(const CXXRecordDecl *RD) {
2669	// Ignore classes without a vtable.
2670	if (!RD->isDynamicClass())
2671	return;
2672
2673	// Initialize the vtable pointers for this class and all of its bases.
2674	if (CGM.getCXXABI().doStructorsInitializeVPtrs(VTableClass: RD))
2675	for (const VPtr &Vptr : getVTablePointers(VTableClass: RD))
2676	InitializeVTablePointer(Vptr);
2677
2678	if (RD->getNumVBases())
2679	CGM.getCXXABI().initializeHiddenVirtualInheritanceMembers(CGF&: *this, RD);
2680	}
2681
2682	llvm::Value *CodeGenFunction::GetVTablePtr(Address This,
2683	llvm::Type *VTableTy,
2684	const CXXRecordDecl *RD) {
2685	Address VTablePtrSrc = This.withElementType(ElemTy: VTableTy);
2686	llvm::Instruction *VTable = Builder.CreateLoad(Addr: VTablePtrSrc, Name: "vtable");
2687	TBAAAccessInfo TBAAInfo = CGM.getTBAAVTablePtrAccessInfo(VTablePtrType: VTableTy);
2688	CGM.DecorateInstructionWithTBAA(Inst: VTable, TBAAInfo);
2689
2690	if (CGM.getCodeGenOpts().OptimizationLevel > 0 &&
2691	CGM.getCodeGenOpts().StrictVTablePointers)
2692	CGM.DecorateInstructionWithInvariantGroup(I: VTable, RD);
2693
2694	return VTable;
2695	}
2696
2697	// If a class has a single non-virtual base and does not introduce or override
2698	// virtual member functions or fields, it will have the same layout as its base.
2699	// This function returns the least derived such class.
2700	//
2701	// Casting an instance of a base class to such a derived class is technically
2702	// undefined behavior, but it is a relatively common hack for introducing member
2703	// functions on class instances with specific properties (e.g. llvm::Operator)
2704	// that works under most compilers and should not have security implications, so
2705	// we allow it by default. It can be disabled with -fsanitize=cfi-cast-strict.
2706	static const CXXRecordDecl *
2707	LeastDerivedClassWithSameLayout(const CXXRecordDecl *RD) {
2708	if (!RD->field_empty())
2709	return RD;
2710
2711	if (RD->getNumVBases() != 0)
2712	return RD;
2713
2714	if (RD->getNumBases() != 1)
2715	return RD;
2716
2717	for (const CXXMethodDecl *MD : RD->methods()) {
2718	if (MD->isVirtual()) {
2719	// Virtual member functions are only ok if they are implicit destructors
2720	// because the implicit destructor will have the same semantics as the
2721	// base class's destructor if no fields are added.
2722	if (isa<CXXDestructorDecl>(Val: MD) && MD->isImplicit())
2723	continue;
2724	return RD;
2725	}
2726	}
2727
2728	return LeastDerivedClassWithSameLayout(
2729	RD: RD->bases_begin()->getType()->getAsCXXRecordDecl());
2730	}
2731
2732	void CodeGenFunction::EmitTypeMetadataCodeForVCall(const CXXRecordDecl *RD,
2733	llvm::Value *VTable,
2734	SourceLocation Loc) {
2735	if (SanOpts.has(K: SanitizerKind::CFIVCall))
2736	EmitVTablePtrCheckForCall(RD, VTable, TCK: CodeGenFunction::CFITCK_VCall, Loc);
2737	else if (CGM.getCodeGenOpts().WholeProgramVTables &&
2738	// Don't insert type test assumes if we are forcing public
2739	// visibility.
2740	!CGM.AlwaysHasLTOVisibilityPublic(RD)) {
2741	QualType Ty = QualType(RD->getTypeForDecl(), 0);
2742	llvm::Metadata *MD = CGM.CreateMetadataIdentifierForType(T: Ty);
2743	llvm::Value *TypeId =
2744	llvm::MetadataAsValue::get(Context&: CGM.getLLVMContext(), MD);
2745
2746	// If we already know that the call has hidden LTO visibility, emit
2747	// @llvm.type.test(). Otherwise emit @llvm.public.type.test(), which WPD
2748	// will convert to @llvm.type.test() if we assert at link time that we have
2749	// whole program visibility.
2750	llvm::Intrinsic::ID IID = CGM.HasHiddenLTOVisibility(RD)
2751	? llvm::Intrinsic::type_test
2752	: llvm::Intrinsic::public_type_test;
2753	llvm::Value *TypeTest =
2754	Builder.CreateCall(Callee: CGM.getIntrinsic(IID), Args: {VTable, TypeId});
2755	Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::assume), TypeTest);
2756	}
2757	}
2758
2759	void CodeGenFunction::EmitVTablePtrCheckForCall(const CXXRecordDecl *RD,
2760	llvm::Value *VTable,
2761	CFITypeCheckKind TCK,
2762	SourceLocation Loc) {
2763	if (!SanOpts.has(K: SanitizerKind::CFICastStrict))
2764	RD = LeastDerivedClassWithSameLayout(RD);
2765
2766	EmitVTablePtrCheck(RD, VTable, TCK, Loc);
2767	}
2768
2769	void CodeGenFunction::EmitVTablePtrCheckForCast(QualType T, Address Derived,
2770	bool MayBeNull,
2771	CFITypeCheckKind TCK,
2772	SourceLocation Loc) {
2773	if (!getLangOpts().CPlusPlus)
2774	return;
2775
2776	auto *ClassTy = T->getAs<RecordType>();
2777	if (!ClassTy)
2778	return;
2779
2780	const CXXRecordDecl *ClassDecl = cast<CXXRecordDecl>(Val: ClassTy->getDecl());
2781
2782	if (!ClassDecl->isCompleteDefinition() || !ClassDecl->isDynamicClass())
2783	return;
2784
2785	if (!SanOpts.has(K: SanitizerKind::CFICastStrict))
2786	ClassDecl = LeastDerivedClassWithSameLayout(RD: ClassDecl);
2787
2788	llvm::BasicBlock *ContBlock = nullptr;
2789
2790	if (MayBeNull) {
2791	llvm::Value *DerivedNotNull =
2792	Builder.CreateIsNotNull(Arg: Derived.emitRawPointer(CGF&: *this), Name: "cast.nonnull");
2793
2794	llvm::BasicBlock *CheckBlock = createBasicBlock(name: "cast.check");
2795	ContBlock = createBasicBlock(name: "cast.cont");
2796
2797	Builder.CreateCondBr(Cond: DerivedNotNull, True: CheckBlock, False: ContBlock);
2798
2799	EmitBlock(BB: CheckBlock);
2800	}
2801
2802	llvm::Value *VTable;
2803	std::tie(args&: VTable, args&: ClassDecl) =
2804	CGM.getCXXABI().LoadVTablePtr(CGF&: *this, This: Derived, RD: ClassDecl);
2805
2806	EmitVTablePtrCheck(RD: ClassDecl, VTable, TCK, Loc);
2807
2808	if (MayBeNull) {
2809	Builder.CreateBr(Dest: ContBlock);
2810	EmitBlock(BB: ContBlock);
2811	}
2812	}
2813
2814	void CodeGenFunction::EmitVTablePtrCheck(const CXXRecordDecl *RD,
2815	llvm::Value *VTable,
2816	CFITypeCheckKind TCK,
2817	SourceLocation Loc) {
2818	if (!CGM.getCodeGenOpts().SanitizeCfiCrossDso &&
2819	!CGM.HasHiddenLTOVisibility(RD))
2820	return;
2821
2822	SanitizerMask M;
2823	llvm::SanitizerStatKind SSK;
2824	switch (TCK) {
2825	case CFITCK_VCall:
2826	M = SanitizerKind::CFIVCall;
2827	SSK = llvm::SanStat_CFI_VCall;
2828	break;
2829	case CFITCK_NVCall:
2830	M = SanitizerKind::CFINVCall;
2831	SSK = llvm::SanStat_CFI_NVCall;
2832	break;
2833	case CFITCK_DerivedCast:
2834	M = SanitizerKind::CFIDerivedCast;
2835	SSK = llvm::SanStat_CFI_DerivedCast;
2836	break;
2837	case CFITCK_UnrelatedCast:
2838	M = SanitizerKind::CFIUnrelatedCast;
2839	SSK = llvm::SanStat_CFI_UnrelatedCast;
2840	break;
2841	case CFITCK_ICall:
2842	case CFITCK_NVMFCall:
2843	case CFITCK_VMFCall:
2844	llvm_unreachable("unexpected sanitizer kind");
2845	}
2846
2847	std::string TypeName = RD->getQualifiedNameAsString();
2848	if (getContext().getNoSanitizeList().containsType(Mask: M, MangledTypeName: TypeName))
2849	return;
2850
2851	SanitizerScope SanScope(this);
2852	EmitSanitizerStatReport(SSK);
2853
2854	llvm::Metadata *MD =
2855	CGM.CreateMetadataIdentifierForType(T: QualType(RD->getTypeForDecl(), 0));
2856	llvm::Value *TypeId = llvm::MetadataAsValue::get(Context&: getLLVMContext(), MD);
2857
2858	llvm::Value *TypeTest = Builder.CreateCall(
2859	CGM.getIntrinsic(llvm::Intrinsic::type_test), {VTable, TypeId});
2860
2861	llvm::Constant *StaticData[] = {
2862	llvm::ConstantInt::get(Ty: Int8Ty, V: TCK),
2863	EmitCheckSourceLocation(Loc),
2864	EmitCheckTypeDescriptor(T: QualType(RD->getTypeForDecl(), 0)),
2865	};
2866
2867	auto CrossDsoTypeId = CGM.CreateCrossDsoCfiTypeId(MD);
2868	if (CGM.getCodeGenOpts().SanitizeCfiCrossDso && CrossDsoTypeId) {
2869	EmitCfiSlowPathCheck(Kind: M, Cond: TypeTest, TypeId: CrossDsoTypeId, Ptr: VTable, StaticArgs: StaticData);
2870	return;
2871	}
2872
2873	if (CGM.getCodeGenOpts().SanitizeTrap.has(K: M)) {
2874	EmitTrapCheck(Checked: TypeTest, CheckHandlerID: SanitizerHandler::CFICheckFail);
2875	return;
2876	}
2877
2878	llvm::Value *AllVtables = llvm::MetadataAsValue::get(
2879	Context&: CGM.getLLVMContext(),
2880	MD: llvm::MDString::get(Context&: CGM.getLLVMContext(), Str: "all-vtables"));
2881	llvm::Value *ValidVtable = Builder.CreateCall(
2882	CGM.getIntrinsic(llvm::Intrinsic::type_test), {VTable, AllVtables});
2883	EmitCheck(Checked: std::make_pair(x&: TypeTest, y&: M), Check: SanitizerHandler::CFICheckFail,
2884	StaticArgs: StaticData, DynamicArgs: {VTable, ValidVtable});
2885	}
2886
2887	bool CodeGenFunction::ShouldEmitVTableTypeCheckedLoad(const CXXRecordDecl *RD) {
2888	if (!CGM.getCodeGenOpts().WholeProgramVTables ||
2889	!CGM.HasHiddenLTOVisibility(RD))
2890	return false;
2891
2892	if (CGM.getCodeGenOpts().VirtualFunctionElimination)
2893	return true;
2894
2895	if (!SanOpts.has(K: SanitizerKind::CFIVCall) ||
2896	!CGM.getCodeGenOpts().SanitizeTrap.has(K: SanitizerKind::CFIVCall))
2897	return false;
2898
2899	std::string TypeName = RD->getQualifiedNameAsString();
2900	return !getContext().getNoSanitizeList().containsType(Mask: SanitizerKind::CFIVCall,
2901	MangledTypeName: TypeName);
2902	}
2903
2904	llvm::Value *CodeGenFunction::EmitVTableTypeCheckedLoad(
2905	const CXXRecordDecl *RD, llvm::Value *VTable, llvm::Type *VTableTy,
2906	uint64_t VTableByteOffset) {
2907	SanitizerScope SanScope(this);
2908
2909	EmitSanitizerStatReport(SSK: llvm::SanStat_CFI_VCall);
2910
2911	llvm::Metadata *MD =
2912	CGM.CreateMetadataIdentifierForType(T: QualType(RD->getTypeForDecl(), 0));
2913	llvm::Value *TypeId = llvm::MetadataAsValue::get(Context&: CGM.getLLVMContext(), MD);
2914
2915	llvm::Value *CheckedLoad = Builder.CreateCall(
2916	CGM.getIntrinsic(llvm::Intrinsic::type_checked_load),
2917	{VTable, llvm::ConstantInt::get(Int32Ty, VTableByteOffset), TypeId});
2918	llvm::Value *CheckResult = Builder.CreateExtractValue(Agg: CheckedLoad, Idxs: 1);
2919
2920	std::string TypeName = RD->getQualifiedNameAsString();
2921	if (SanOpts.has(K: SanitizerKind::CFIVCall) &&
2922	!getContext().getNoSanitizeList().containsType(Mask: SanitizerKind::CFIVCall,
2923	MangledTypeName: TypeName)) {
2924	EmitCheck(Checked: std::make_pair(x&: CheckResult, y: SanitizerKind::CFIVCall),
2925	Check: SanitizerHandler::CFICheckFail, StaticArgs: {}, DynamicArgs: {});
2926	}
2927
2928	return Builder.CreateBitCast(V: Builder.CreateExtractValue(Agg: CheckedLoad, Idxs: 0),
2929	DestTy: VTableTy);
2930	}
2931
2932	void CodeGenFunction::EmitForwardingCallToLambda(
2933	const CXXMethodDecl *callOperator, CallArgList &callArgs,
2934	const CGFunctionInfo *calleeFnInfo, llvm::Constant *calleePtr) {
2935	// Get the address of the call operator.
2936	if (!calleeFnInfo)
2937	calleeFnInfo = &CGM.getTypes().arrangeCXXMethodDeclaration(MD: callOperator);
2938
2939	if (!calleePtr)
2940	calleePtr =
2941	CGM.GetAddrOfFunction(GD: GlobalDecl(callOperator),
2942	Ty: CGM.getTypes().GetFunctionType(Info: *calleeFnInfo));
2943
2944	// Prepare the return slot.
2945	const FunctionProtoType *FPT =
2946	callOperator->getType()->castAs<FunctionProtoType>();
2947	QualType resultType = FPT->getReturnType();
2948	ReturnValueSlot returnSlot;
2949	if (!resultType->isVoidType() &&
2950	calleeFnInfo->getReturnInfo().getKind() == ABIArgInfo::Indirect &&
2951	!hasScalarEvaluationKind(T: calleeFnInfo->getReturnType()))
2952	returnSlot =
2953	ReturnValueSlot(ReturnValue, resultType.isVolatileQualified(),
2954	/*IsUnused=*/false, /*IsExternallyDestructed=*/true);
2955
2956	// We don't need to separately arrange the call arguments because
2957	// the call can't be variadic anyway --- it's impossible to forward
2958	// variadic arguments.
2959
2960	// Now emit our call.
2961	auto callee = CGCallee::forDirect(functionPtr: calleePtr, abstractInfo: GlobalDecl(callOperator));
2962	RValue RV = EmitCall(*calleeFnInfo, callee, returnSlot, callArgs);
2963
2964	// If necessary, copy the returned value into the slot.
2965	if (!resultType->isVoidType() && returnSlot.isNull()) {
2966	if (getLangOpts().ObjCAutoRefCount && resultType->isObjCRetainableType()) {
2967	RV = RValue::get(V: EmitARCRetainAutoreleasedReturnValue(value: RV.getScalarVal()));
2968	}
2969	EmitReturnOfRValue(RV, Ty: resultType);
2970	} else
2971	EmitBranchThroughCleanup(Dest: ReturnBlock);
2972	}
2973
2974	void CodeGenFunction::EmitLambdaBlockInvokeBody() {
2975	const BlockDecl *BD = BlockInfo->getBlockDecl();
2976	const VarDecl *variable = BD->capture_begin()->getVariable();
2977	const CXXRecordDecl *Lambda = variable->getType()->getAsCXXRecordDecl();
2978	const CXXMethodDecl *CallOp = Lambda->getLambdaCallOperator();
2979
2980	if (CallOp->isVariadic()) {
2981	// FIXME: Making this work correctly is nasty because it requires either
2982	// cloning the body of the call operator or making the call operator
2983	// forward.
2984	CGM.ErrorUnsupported(D: CurCodeDecl, Type: "lambda conversion to variadic function");
2985	return;
2986	}
2987
2988	// Start building arguments for forwarding call
2989	CallArgList CallArgs;
2990
2991	QualType ThisType = getContext().getPointerType(T: getContext().getRecordType(Lambda));
2992	Address ThisPtr = GetAddrOfBlockDecl(var: variable);
2993	CallArgs.add(rvalue: RValue::get(V: getAsNaturalPointerTo(Addr: ThisPtr, PointeeType: ThisType)), type: ThisType);
2994
2995	// Add the rest of the parameters.
2996	for (auto *param : BD->parameters())
2997	EmitDelegateCallArg(args&: CallArgs, param, loc: param->getBeginLoc());
2998
2999	assert(!Lambda->isGenericLambda() &&
3000	"generic lambda interconversion to block not implemented");
3001	EmitForwardingCallToLambda(callOperator: CallOp, callArgs&: CallArgs);
3002	}
3003
3004	void CodeGenFunction::EmitLambdaStaticInvokeBody(const CXXMethodDecl *MD) {
3005	if (MD->isVariadic()) {
3006	// FIXME: Making this work correctly is nasty because it requires either
3007	// cloning the body of the call operator or making the call operator
3008	// forward.
3009	CGM.ErrorUnsupported(MD, "lambda conversion to variadic function");
3010	return;
3011	}
3012
3013	const CXXRecordDecl *Lambda = MD->getParent();
3014
3015	// Start building arguments for forwarding call
3016	CallArgList CallArgs;
3017
3018	QualType LambdaType = getContext().getRecordType(Lambda);
3019	QualType ThisType = getContext().getPointerType(T: LambdaType);
3020	Address ThisPtr = CreateMemTemp(T: LambdaType, Name: "unused.capture");
3021	CallArgs.add(rvalue: RValue::get(V: ThisPtr.emitRawPointer(CGF&: *this)), type: ThisType);
3022
3023	EmitLambdaDelegatingInvokeBody(MD, CallArgs);
3024	}
3025
3026	void CodeGenFunction::EmitLambdaDelegatingInvokeBody(const CXXMethodDecl *MD,
3027	CallArgList &CallArgs) {
3028	// Add the rest of the forwarded parameters.
3029	for (auto *Param : MD->parameters())
3030	EmitDelegateCallArg(CallArgs, Param, Param->getBeginLoc());
3031
3032	const CXXRecordDecl *Lambda = MD->getParent();
3033	const CXXMethodDecl *CallOp = Lambda->getLambdaCallOperator();
3034	// For a generic lambda, find the corresponding call operator specialization
3035	// to which the call to the static-invoker shall be forwarded.
3036	if (Lambda->isGenericLambda()) {
3037	assert(MD->isFunctionTemplateSpecialization());
3038	const TemplateArgumentList *TAL = MD->getTemplateSpecializationArgs();
3039	FunctionTemplateDecl *CallOpTemplate = CallOp->getDescribedFunctionTemplate();
3040	void *InsertPos = nullptr;
3041	FunctionDecl *CorrespondingCallOpSpecialization =
3042	CallOpTemplate->findSpecialization(Args: TAL->asArray(), InsertPos);
3043	assert(CorrespondingCallOpSpecialization);
3044	CallOp = cast<CXXMethodDecl>(Val: CorrespondingCallOpSpecialization);
3045	}
3046
3047	// Special lambda forwarding when there are inalloca parameters.
3048	if (hasInAllocaArg(MD)) {
3049	const CGFunctionInfo *ImplFnInfo = nullptr;
3050	llvm::Function *ImplFn = nullptr;
3051	EmitLambdaInAllocaImplFn(CallOp, ImplFnInfo: &ImplFnInfo, ImplFn: &ImplFn);
3052
3053	EmitForwardingCallToLambda(callOperator: CallOp, callArgs&: CallArgs, calleeFnInfo: ImplFnInfo, calleePtr: ImplFn);
3054	return;
3055	}
3056
3057	EmitForwardingCallToLambda(callOperator: CallOp, callArgs&: CallArgs);
3058	}
3059
3060	void CodeGenFunction::EmitLambdaInAllocaCallOpBody(const CXXMethodDecl *MD) {
3061	if (MD->isVariadic()) {
3062	// FIXME: Making this work correctly is nasty because it requires either
3063	// cloning the body of the call operator or making the call operator forward.
3064	CGM.ErrorUnsupported(MD, "lambda conversion to variadic function");
3065	return;
3066	}
3067
3068	// Forward %this argument.
3069	CallArgList CallArgs;
3070	QualType LambdaType = getContext().getRecordType(MD->getParent());
3071	QualType ThisType = getContext().getPointerType(T: LambdaType);
3072	llvm::Value *ThisArg = CurFn->getArg(i: 0);
3073	CallArgs.add(rvalue: RValue::get(V: ThisArg), type: ThisType);
3074
3075	EmitLambdaDelegatingInvokeBody(MD, CallArgs);
3076	}
3077
3078	void CodeGenFunction::EmitLambdaInAllocaImplFn(
3079	const CXXMethodDecl *CallOp, const CGFunctionInfo **ImplFnInfo,
3080	llvm::Function **ImplFn) {
3081	const CGFunctionInfo &FnInfo =
3082	CGM.getTypes().arrangeCXXMethodDeclaration(MD: CallOp);
3083	llvm::Function *CallOpFn =
3084	cast<llvm::Function>(Val: CGM.GetAddrOfFunction(GD: GlobalDecl(CallOp)));
3085
3086	// Emit function containing the original call op body. __invoke will delegate
3087	// to this function.
3088	SmallVector<CanQualType, 4> ArgTypes;
3089	for (auto I = FnInfo.arg_begin(); I != FnInfo.arg_end(); ++I)
3090	ArgTypes.push_back(Elt: I->type);
3091	*ImplFnInfo = &CGM.getTypes().arrangeLLVMFunctionInfo(
3092	returnType: FnInfo.getReturnType(), opts: FnInfoOpts::IsDelegateCall, argTypes: ArgTypes,
3093	info: FnInfo.getExtInfo(), paramInfos: {}, args: FnInfo.getRequiredArgs());
3094
3095	// Create mangled name as if this was a method named __impl. If for some
3096	// reason the name doesn't look as expected then just tack __impl to the
3097	// front.
3098	// TODO: Use the name mangler to produce the right name instead of using
3099	// string replacement.
3100	StringRef CallOpName = CallOpFn->getName();
3101	std::string ImplName;
3102	if (size_t Pos = CallOpName.find_first_of(Chars: "<lambda"))
3103	ImplName = ("?__impl@" + CallOpName.drop_front(N: Pos)).str();
3104	else
3105	ImplName = ("__impl" + CallOpName).str();
3106
3107	llvm::Function *Fn = CallOpFn->getParent()->getFunction(Name: ImplName);
3108	if (!Fn) {
3109	Fn = llvm::Function::Create(Ty: CGM.getTypes().GetFunctionType(Info: **ImplFnInfo),
3110	Linkage: llvm::GlobalValue::InternalLinkage, N: ImplName,
3111	M&: CGM.getModule());
3112	CGM.SetInternalFunctionAttributes(CallOp, Fn, **ImplFnInfo);
3113
3114	const GlobalDecl &GD = GlobalDecl(CallOp);
3115	const auto *D = cast<FunctionDecl>(Val: GD.getDecl());
3116	CodeGenFunction(CGM).GenerateCode(GD, Fn, FnInfo: **ImplFnInfo);
3117	CGM.SetLLVMFunctionAttributesForDefinition(D: D, F: Fn);
3118	}
3119	*ImplFn = Fn;
3120	}
3121