1	//===--- CGClass.cpp - Emit LLVM Code for C++ classes -----------*- C++ -*-===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This contains code dealing with C++ code generation of classes
10	//
11	//===----------------------------------------------------------------------===//
12
13	#include "CGBlocks.h"
14	#include "CGCXXABI.h"
15	#include "CGDebugInfo.h"
16	#include "CGRecordLayout.h"
17	#include "CodeGenFunction.h"
18	#include "TargetInfo.h"
19	#include "clang/AST/Attr.h"
20	#include "clang/AST/CXXInheritance.h"
21	#include "clang/AST/CharUnits.h"
22	#include "clang/AST/DeclTemplate.h"
23	#include "clang/AST/EvaluatedExprVisitor.h"
24	#include "clang/AST/RecordLayout.h"
25	#include "clang/AST/StmtCXX.h"
26	#include "clang/Basic/CodeGenOptions.h"
27	#include "clang/Basic/TargetBuiltins.h"
28	#include "clang/CodeGen/CGFunctionInfo.h"
29	#include "llvm/IR/Intrinsics.h"
30	#include "llvm/IR/Metadata.h"
31	#include "llvm/Support/SaveAndRestore.h"
32	#include "llvm/Transforms/Utils/SanitizerStats.h"
33	#include <optional>
34
35	using namespace clang;
36	using namespace CodeGen;
37
38	/// Return the best known alignment for an unknown pointer to a
39	/// particular class.
40	CharUnits CodeGenModule::getClassPointerAlignment(const CXXRecordDecl *RD) {
41	if (!RD->hasDefinition())
42	return CharUnits::One(); // Hopefully won't be used anywhere.
43
44	auto &layout = getContext().getASTRecordLayout(RD);
45
46	// If the class is final, then we know that the pointer points to an
47	// object of that type and can use the full alignment.
48	if (RD->isEffectivelyFinal())
49	return layout.getAlignment();
50
51	// Otherwise, we have to assume it could be a subclass.
52	return layout.getNonVirtualAlignment();
53	}
54
55	/// Return the smallest possible amount of storage that might be allocated
56	/// starting from the beginning of an object of a particular class.
57	///
58	/// This may be smaller than sizeof(RD) if RD has virtual base classes.
59	CharUnits CodeGenModule::getMinimumClassObjectSize(const CXXRecordDecl *RD) {
60	if (!RD->hasDefinition())
61	return CharUnits::One();
62
63	auto &layout = getContext().getASTRecordLayout(RD);
64
65	// If the class is final, then we know that the pointer points to an
66	// object of that type and can use the full alignment.
67	if (RD->isEffectivelyFinal())
68	return layout.getSize();
69
70	// Otherwise, we have to assume it could be a subclass.
71	return std::max(layout.getNonVirtualSize(), CharUnits::One());
72	}
73
74	/// Return the best known alignment for a pointer to a virtual base,
75	/// given the alignment of a pointer to the derived class.
76	CharUnits CodeGenModule::getVBaseAlignment(CharUnits actualDerivedAlign,
77	const CXXRecordDecl *derivedClass,
78	const CXXRecordDecl *vbaseClass) {
79	// The basic idea here is that an underaligned derived pointer might
80	// indicate an underaligned base pointer.
81
82	assert(vbaseClass->isCompleteDefinition());
83	auto &baseLayout = getContext().getASTRecordLayout(vbaseClass);
84	CharUnits expectedVBaseAlign = baseLayout.getNonVirtualAlignment();
85
86	return getDynamicOffsetAlignment(ActualAlign: actualDerivedAlign, Class: derivedClass,
87	ExpectedTargetAlign: expectedVBaseAlign);
88	}
89
90	CharUnits
91	CodeGenModule::getDynamicOffsetAlignment(CharUnits actualBaseAlign,
92	const CXXRecordDecl *baseDecl,
93	CharUnits expectedTargetAlign) {
94	// If the base is an incomplete type (which is, alas, possible with
95	// member pointers), be pessimistic.
96	if (!baseDecl->isCompleteDefinition())
97	return std::min(a: actualBaseAlign, b: expectedTargetAlign);
98
99	auto &baseLayout = getContext().getASTRecordLayout(baseDecl);
100	CharUnits expectedBaseAlign = baseLayout.getNonVirtualAlignment();
101
102	// If the class is properly aligned, assume the target offset is, too.
103	//
104	// This actually isn't necessarily the right thing to do --- if the
105	// class is a complete object, but it's only properly aligned for a
106	// base subobject, then the alignments of things relative to it are
107	// probably off as well. (Note that this requires the alignment of
108	// the target to be greater than the NV alignment of the derived
109	// class.)
110	//
111	// However, our approach to this kind of under-alignment can only
112	// ever be best effort; after all, we're never going to propagate
113	// alignments through variables or parameters. Note, in particular,
114	// that constructing a polymorphic type in an address that's less
115	// than pointer-aligned will generally trap in the constructor,
116	// unless we someday add some sort of attribute to change the
117	// assumed alignment of 'this'. So our goal here is pretty much
118	// just to allow the user to explicitly say that a pointer is
119	// under-aligned and then safely access its fields and vtables.
120	if (actualBaseAlign >= expectedBaseAlign) {
121	return expectedTargetAlign;
122	}
123
124	// Otherwise, we might be offset by an arbitrary multiple of the
125	// actual alignment. The correct adjustment is to take the min of
126	// the two alignments.
127	return std::min(a: actualBaseAlign, b: expectedTargetAlign);
128	}
129
130	Address CodeGenFunction::LoadCXXThisAddress() {
131	assert(CurFuncDecl && "loading 'this' without a func declaration?");
132	auto *MD = cast<CXXMethodDecl>(Val: CurFuncDecl);
133
134	// Lazily compute CXXThisAlignment.
135	if (CXXThisAlignment.isZero()) {
136	// Just use the best known alignment for the parent.
137	// TODO: if we're currently emitting a complete-object ctor/dtor,
138	// we can always use the complete-object alignment.
139	CXXThisAlignment = CGM.getClassPointerAlignment(RD: MD->getParent());
140	}
141
142	llvm::Type *Ty = ConvertType(T: MD->getFunctionObjectParameterType());
143	return Address(LoadCXXThis(), Ty, CXXThisAlignment, KnownNonNull);
144	}
145
146	/// Emit the address of a field using a member data pointer.
147	///
148	/// \param E Only used for emergency diagnostics
149	Address
150	CodeGenFunction::EmitCXXMemberDataPointerAddress(const Expr *E, Address base,
151	llvm::Value *memberPtr,
152	const MemberPointerType *memberPtrType,
153	LValueBaseInfo *BaseInfo,
154	TBAAAccessInfo *TBAAInfo) {
155	// Ask the ABI to compute the actual address.
156	llvm::Value *ptr =
157	CGM.getCXXABI().EmitMemberDataPointerAddress(CGF&: *this, E, Base: base,
158	MemPtr: memberPtr, MPT: memberPtrType);
159
160	QualType memberType = memberPtrType->getPointeeType();
161	CharUnits memberAlign =
162	CGM.getNaturalTypeAlignment(T: memberType, BaseInfo, TBAAInfo);
163	memberAlign =
164	CGM.getDynamicOffsetAlignment(actualBaseAlign: base.getAlignment(),
165	baseDecl: memberPtrType->getClass()->getAsCXXRecordDecl(),
166	expectedTargetAlign: memberAlign);
167	return Address(ptr, ConvertTypeForMem(T: memberPtrType->getPointeeType()),
168	memberAlign);
169	}
170
171	CharUnits CodeGenModule::computeNonVirtualBaseClassOffset(
172	const CXXRecordDecl *DerivedClass, CastExpr::path_const_iterator Start,
173	CastExpr::path_const_iterator End) {
174	CharUnits Offset = CharUnits::Zero();
175
176	const ASTContext &Context = getContext();
177	const CXXRecordDecl *RD = DerivedClass;
178
179	for (CastExpr::path_const_iterator I = Start; I != End; ++I) {
180	const CXXBaseSpecifier *Base = *I;
181	assert(!Base->isVirtual() && "Should not see virtual bases here!");
182
183	// Get the layout.
184	const ASTRecordLayout &Layout = Context.getASTRecordLayout(RD);
185
186	const auto *BaseDecl =
187	cast<CXXRecordDecl>(Val: Base->getType()->castAs<RecordType>()->getDecl());
188
189	// Add the offset.
190	Offset += Layout.getBaseClassOffset(Base: BaseDecl);
191
192	RD = BaseDecl;
193	}
194
195	return Offset;
196	}
197
198	llvm::Constant *
199	CodeGenModule::GetNonVirtualBaseClassOffset(const CXXRecordDecl *ClassDecl,
200	CastExpr::path_const_iterator PathBegin,
201	CastExpr::path_const_iterator PathEnd) {
202	assert(PathBegin != PathEnd && "Base path should not be empty!");
203
204	CharUnits Offset =
205	computeNonVirtualBaseClassOffset(DerivedClass: ClassDecl, Start: PathBegin, End: PathEnd);
206	if (Offset.isZero())
207	return nullptr;
208
209	llvm::Type *PtrDiffTy =
210	Types.ConvertType(T: getContext().getPointerDiffType());
211
212	return llvm::ConstantInt::get(Ty: PtrDiffTy, V: Offset.getQuantity());
213	}
214
215	/// Gets the address of a direct base class within a complete object.
216	/// This should only be used for (1) non-virtual bases or (2) virtual bases
217	/// when the type is known to be complete (e.g. in complete destructors).
218	///
219	/// The object pointed to by 'This' is assumed to be non-null.
220	Address
221	CodeGenFunction::GetAddressOfDirectBaseInCompleteClass(Address This,
222	const CXXRecordDecl *Derived,
223	const CXXRecordDecl *Base,
224	bool BaseIsVirtual) {
225	// 'this' must be a pointer (in some address space) to Derived.
226	assert(This.getElementType() == ConvertType(Derived));
227
228	// Compute the offset of the virtual base.
229	CharUnits Offset;
230	const ASTRecordLayout &Layout = getContext().getASTRecordLayout(Derived);
231	if (BaseIsVirtual)
232	Offset = Layout.getVBaseClassOffset(VBase: Base);
233	else
234	Offset = Layout.getBaseClassOffset(Base);
235
236	// Shift and cast down to the base type.
237	// TODO: for complete types, this should be possible with a GEP.
238	Address V = This;
239	if (!Offset.isZero()) {
240	V = V.withElementType(ElemTy: Int8Ty);
241	V = Builder.CreateConstInBoundsByteGEP(Addr: V, Offset);
242	}
243	return V.withElementType(ElemTy: ConvertType(Base));
244	}
245
246	static Address
247	ApplyNonVirtualAndVirtualOffset(CodeGenFunction &CGF, Address addr,
248	CharUnits nonVirtualOffset,
249	llvm::Value *virtualOffset,
250	const CXXRecordDecl *derivedClass,
251	const CXXRecordDecl *nearestVBase) {
252	// Assert that we have something to do.
253	assert(!nonVirtualOffset.isZero() || virtualOffset != nullptr);
254
255	// Compute the offset from the static and dynamic components.
256	llvm::Value *baseOffset;
257	if (!nonVirtualOffset.isZero()) {
258	llvm::Type *OffsetType =
259	(CGF.CGM.getTarget().getCXXABI().isItaniumFamily() &&
260	CGF.CGM.getItaniumVTableContext().isRelativeLayout())
261	? CGF.Int32Ty
262	: CGF.PtrDiffTy;
263	baseOffset =
264	llvm::ConstantInt::get(Ty: OffsetType, V: nonVirtualOffset.getQuantity());
265	if (virtualOffset) {
266	baseOffset = CGF.Builder.CreateAdd(LHS: virtualOffset, RHS: baseOffset);
267	}
268	} else {
269	baseOffset = virtualOffset;
270	}
271
272	// Apply the base offset.
273	llvm::Value *ptr = addr.getPointer();
274	ptr = CGF.Builder.CreateInBoundsGEP(Ty: CGF.Int8Ty, Ptr: ptr, IdxList: baseOffset, Name: "add.ptr");
275
276	// If we have a virtual component, the alignment of the result will
277	// be relative only to the known alignment of that vbase.
278	CharUnits alignment;
279	if (virtualOffset) {
280	assert(nearestVBase && "virtual offset without vbase?");
281	alignment = CGF.CGM.getVBaseAlignment(actualDerivedAlign: addr.getAlignment(),
282	derivedClass, vbaseClass: nearestVBase);
283	} else {
284	alignment = addr.getAlignment();
285	}
286	alignment = alignment.alignmentAtOffset(offset: nonVirtualOffset);
287
288	return Address(ptr, CGF.Int8Ty, alignment);
289	}
290
291	Address CodeGenFunction::GetAddressOfBaseClass(
292	Address Value, const CXXRecordDecl *Derived,
293	CastExpr::path_const_iterator PathBegin,
294	CastExpr::path_const_iterator PathEnd, bool NullCheckValue,
295	SourceLocation Loc) {
296	assert(PathBegin != PathEnd && "Base path should not be empty!");
297
298	CastExpr::path_const_iterator Start = PathBegin;
299	const CXXRecordDecl *VBase = nullptr;
300
301	// Sema has done some convenient canonicalization here: if the
302	// access path involved any virtual steps, the conversion path will
303	// *start* with a step down to the correct virtual base subobject,
304	// and hence will not require any further steps.
305	if ((*Start)->isVirtual()) {
306	VBase = cast<CXXRecordDecl>(
307	Val: (*Start)->getType()->castAs<RecordType>()->getDecl());
308	++Start;
309	}
310
311	// Compute the static offset of the ultimate destination within its
312	// allocating subobject (the virtual base, if there is one, or else
313	// the "complete" object that we see).
314	CharUnits NonVirtualOffset = CGM.computeNonVirtualBaseClassOffset(
315	DerivedClass: VBase ? VBase : Derived, Start, End: PathEnd);
316
317	// If there's a virtual step, we can sometimes "devirtualize" it.
318	// For now, that's limited to when the derived type is final.
319	// TODO: "devirtualize" this for accesses to known-complete objects.
320	if (VBase && Derived->hasAttr<FinalAttr>()) {
321	const ASTRecordLayout &layout = getContext().getASTRecordLayout(Derived);
322	CharUnits vBaseOffset = layout.getVBaseClassOffset(VBase);
323	NonVirtualOffset += vBaseOffset;
324	VBase = nullptr; // we no longer have a virtual step
325	}
326
327	// Get the base pointer type.
328	llvm::Type *BaseValueTy = ConvertType(T: (PathEnd[-1])->getType());
329	llvm::Type *PtrTy = llvm::PointerType::get(
330	C&: CGM.getLLVMContext(), AddressSpace: Value.getType()->getPointerAddressSpace());
331
332	QualType DerivedTy = getContext().getRecordType(Derived);
333	CharUnits DerivedAlign = CGM.getClassPointerAlignment(RD: Derived);
334
335	// If the static offset is zero and we don't have a virtual step,
336	// just do a bitcast; null checks are unnecessary.
337	if (NonVirtualOffset.isZero() && !VBase) {
338	if (sanitizePerformTypeCheck()) {
339	SanitizerSet SkippedChecks;
340	SkippedChecks.set(K: SanitizerKind::Null, Value: !NullCheckValue);
341	EmitTypeCheck(TCK: TCK_Upcast, Loc, V: Value.getPointer(),
342	Type: DerivedTy, Alignment: DerivedAlign, SkippedChecks);
343	}
344	return Value.withElementType(ElemTy: BaseValueTy);
345	}
346
347	llvm::BasicBlock *origBB = nullptr;
348	llvm::BasicBlock *endBB = nullptr;
349
350	// Skip over the offset (and the vtable load) if we're supposed to
351	// null-check the pointer.
352	if (NullCheckValue) {
353	origBB = Builder.GetInsertBlock();
354	llvm::BasicBlock *notNullBB = createBasicBlock(name: "cast.notnull");
355	endBB = createBasicBlock(name: "cast.end");
356
357	llvm::Value *isNull = Builder.CreateIsNull(Arg: Value.getPointer());
358	Builder.CreateCondBr(Cond: isNull, True: endBB, False: notNullBB);
359	EmitBlock(BB: notNullBB);
360	}
361
362	if (sanitizePerformTypeCheck()) {
363	SanitizerSet SkippedChecks;
364	SkippedChecks.set(K: SanitizerKind::Null, Value: true);
365	EmitTypeCheck(TCK: VBase ? TCK_UpcastToVirtualBase : TCK_Upcast, Loc,
366	V: Value.getPointer(), Type: DerivedTy, Alignment: DerivedAlign, SkippedChecks);
367	}
368
369	// Compute the virtual offset.
370	llvm::Value *VirtualOffset = nullptr;
371	if (VBase) {
372	VirtualOffset =
373	CGM.getCXXABI().GetVirtualBaseClassOffset(CGF&: *this, This: Value, ClassDecl: Derived, BaseClassDecl: VBase);
374	}
375
376	// Apply both offsets.
377	Value = ApplyNonVirtualAndVirtualOffset(CGF&: *this, addr: Value, nonVirtualOffset: NonVirtualOffset,
378	virtualOffset: VirtualOffset, derivedClass: Derived, nearestVBase: VBase);
379
380	// Cast to the destination type.
381	Value = Value.withElementType(ElemTy: BaseValueTy);
382
383	// Build a phi if we needed a null check.
384	if (NullCheckValue) {
385	llvm::BasicBlock *notNullBB = Builder.GetInsertBlock();
386	Builder.CreateBr(Dest: endBB);
387	EmitBlock(BB: endBB);
388
389	llvm::PHINode *PHI = Builder.CreatePHI(Ty: PtrTy, NumReservedValues: 2, Name: "cast.result");
390	PHI->addIncoming(V: Value.getPointer(), BB: notNullBB);
391	PHI->addIncoming(V: llvm::Constant::getNullValue(Ty: PtrTy), BB: origBB);
392	Value = Value.withPointer(NewPointer: PHI, IsKnownNonNull: NotKnownNonNull);
393	}
394
395	return Value;
396	}
397
398	Address
399	CodeGenFunction::GetAddressOfDerivedClass(Address BaseAddr,
400	const CXXRecordDecl *Derived,
401	CastExpr::path_const_iterator PathBegin,
402	CastExpr::path_const_iterator PathEnd,
403	bool NullCheckValue) {
404	assert(PathBegin != PathEnd && "Base path should not be empty!");
405
406	QualType DerivedTy =
407	getContext().getCanonicalType(T: getContext().getTagDeclType(Derived));
408	llvm::Type *DerivedValueTy = ConvertType(T: DerivedTy);
409
410	llvm::Value *NonVirtualOffset =
411	CGM.GetNonVirtualBaseClassOffset(ClassDecl: Derived, PathBegin, PathEnd);
412
413	if (!NonVirtualOffset) {
414	// No offset, we can just cast back.
415	return BaseAddr.withElementType(ElemTy: DerivedValueTy);
416	}
417
418	llvm::BasicBlock *CastNull = nullptr;
419	llvm::BasicBlock *CastNotNull = nullptr;
420	llvm::BasicBlock *CastEnd = nullptr;
421
422	if (NullCheckValue) {
423	CastNull = createBasicBlock(name: "cast.null");
424	CastNotNull = createBasicBlock(name: "cast.notnull");
425	CastEnd = createBasicBlock(name: "cast.end");
426
427	llvm::Value *IsNull = Builder.CreateIsNull(Arg: BaseAddr.getPointer());
428	Builder.CreateCondBr(Cond: IsNull, True: CastNull, False: CastNotNull);
429	EmitBlock(BB: CastNotNull);
430	}
431
432	// Apply the offset.
433	llvm::Value *Value = BaseAddr.getPointer();
434	Value = Builder.CreateInBoundsGEP(
435	Ty: Int8Ty, Ptr: Value, IdxList: Builder.CreateNeg(V: NonVirtualOffset), Name: "sub.ptr");
436
437	// Produce a PHI if we had a null-check.
438	if (NullCheckValue) {
439	Builder.CreateBr(Dest: CastEnd);
440	EmitBlock(BB: CastNull);
441	Builder.CreateBr(Dest: CastEnd);
442	EmitBlock(BB: CastEnd);
443
444	llvm::PHINode *PHI = Builder.CreatePHI(Ty: Value->getType(), NumReservedValues: 2);
445	PHI->addIncoming(V: Value, BB: CastNotNull);
446	PHI->addIncoming(V: llvm::Constant::getNullValue(Ty: Value->getType()), BB: CastNull);
447	Value = PHI;
448	}
449
450	return Address(Value, DerivedValueTy, CGM.getClassPointerAlignment(RD: Derived));
451	}
452
453	llvm::Value *CodeGenFunction::GetVTTParameter(GlobalDecl GD,
454	bool ForVirtualBase,
455	bool Delegating) {
456	if (!CGM.getCXXABI().NeedsVTTParameter(GD)) {
457	// This constructor/destructor does not need a VTT parameter.
458	return nullptr;
459	}
460
461	const CXXRecordDecl *RD = cast<CXXMethodDecl>(Val: CurCodeDecl)->getParent();
462	const CXXRecordDecl *Base = cast<CXXMethodDecl>(Val: GD.getDecl())->getParent();
463
464	uint64_t SubVTTIndex;
465
466	if (Delegating) {
467	// If this is a delegating constructor call, just load the VTT.
468	return LoadCXXVTT();
469	} else if (RD == Base) {
470	// If the record matches the base, this is the complete ctor/dtor
471	// variant calling the base variant in a class with virtual bases.
472	assert(!CGM.getCXXABI().NeedsVTTParameter(CurGD) &&
473	"doing no-op VTT offset in base dtor/ctor?");
474	assert(!ForVirtualBase && "Can't have same class as virtual base!");
475	SubVTTIndex = 0;
476	} else {
477	const ASTRecordLayout &Layout = getContext().getASTRecordLayout(RD);
478	CharUnits BaseOffset = ForVirtualBase ?
479	Layout.getVBaseClassOffset(VBase: Base) :
480	Layout.getBaseClassOffset(Base);
481
482	SubVTTIndex =
483	CGM.getVTables().getSubVTTIndex(RD, Base: BaseSubobject(Base, BaseOffset));
484	assert(SubVTTIndex != 0 && "Sub-VTT index must be greater than zero!");
485	}
486
487	if (CGM.getCXXABI().NeedsVTTParameter(GD: CurGD)) {
488	// A VTT parameter was passed to the constructor, use it.
489	llvm::Value *VTT = LoadCXXVTT();
490	return Builder.CreateConstInBoundsGEP1_64(Ty: VoidPtrTy, Ptr: VTT, Idx0: SubVTTIndex);
491	} else {
492	// We're the complete constructor, so get the VTT by name.
493	llvm::GlobalValue *VTT = CGM.getVTables().GetAddrOfVTT(RD);
494	return Builder.CreateConstInBoundsGEP2_64(
495	Ty: VTT->getValueType(), Ptr: VTT, Idx0: 0, Idx1: SubVTTIndex);
496	}
497	}
498
499	namespace {
500	/// Call the destructor for a direct base class.
501	struct CallBaseDtor final : EHScopeStack::Cleanup {
502	const CXXRecordDecl *BaseClass;
503	bool BaseIsVirtual;
504	CallBaseDtor(const CXXRecordDecl *Base, bool BaseIsVirtual)
505	: BaseClass(Base), BaseIsVirtual(BaseIsVirtual) {}
506
507	void Emit(CodeGenFunction &CGF, Flags flags) override {
508	const CXXRecordDecl *DerivedClass =
509	cast<CXXMethodDecl>(Val: CGF.CurCodeDecl)->getParent();
510
511	const CXXDestructorDecl *D = BaseClass->getDestructor();
512	// We are already inside a destructor, so presumably the object being
513	// destroyed should have the expected type.
514	QualType ThisTy = D->getFunctionObjectParameterType();
515	Address Addr =
516	CGF.GetAddressOfDirectBaseInCompleteClass(This: CGF.LoadCXXThisAddress(),
517	Derived: DerivedClass, Base: BaseClass,
518	BaseIsVirtual);
519	CGF.EmitCXXDestructorCall(D, Type: Dtor_Base, ForVirtualBase: BaseIsVirtual,
520	/*Delegating=*/false, This: Addr, ThisTy);
521	}
522	};
523
524	/// A visitor which checks whether an initializer uses 'this' in a
525	/// way which requires the vtable to be properly set.
526	struct DynamicThisUseChecker : ConstEvaluatedExprVisitor<DynamicThisUseChecker> {
527	typedef ConstEvaluatedExprVisitor<DynamicThisUseChecker> super;
528
529	bool UsesThis;
530
531	DynamicThisUseChecker(const ASTContext &C) : super(C), UsesThis(false) {}
532
533	// Black-list all explicit and implicit references to 'this'.
534	//
535	// Do we need to worry about external references to 'this' derived
536	// from arbitrary code? If so, then anything which runs arbitrary
537	// external code might potentially access the vtable.
538	void VisitCXXThisExpr(const CXXThisExpr *E) { UsesThis = true; }
539	};
540	} // end anonymous namespace
541
542	static bool BaseInitializerUsesThis(ASTContext &C, const Expr *Init) {
543	DynamicThisUseChecker Checker(C);
544	Checker.Visit(Init);
545	return Checker.UsesThis;
546	}
547
548	static void EmitBaseInitializer(CodeGenFunction &CGF,
549	const CXXRecordDecl *ClassDecl,
550	CXXCtorInitializer *BaseInit) {
551	assert(BaseInit->isBaseInitializer() &&
552	"Must have base initializer!");
553
554	Address ThisPtr = CGF.LoadCXXThisAddress();
555
556	const Type *BaseType = BaseInit->getBaseClass();
557	const auto *BaseClassDecl =
558	cast<CXXRecordDecl>(Val: BaseType->castAs<RecordType>()->getDecl());
559
560	bool isBaseVirtual = BaseInit->isBaseVirtual();
561
562	// If the initializer for the base (other than the constructor
563	// itself) accesses 'this' in any way, we need to initialize the
564	// vtables.
565	if (BaseInitializerUsesThis(C&: CGF.getContext(), Init: BaseInit->getInit()))
566	CGF.InitializeVTablePointers(ClassDecl);
567
568	// We can pretend to be a complete class because it only matters for
569	// virtual bases, and we only do virtual bases for complete ctors.
570	Address V =
571	CGF.GetAddressOfDirectBaseInCompleteClass(This: ThisPtr, Derived: ClassDecl,
572	Base: BaseClassDecl,
573	BaseIsVirtual: isBaseVirtual);
574	AggValueSlot AggSlot =
575	AggValueSlot::forAddr(
576	addr: V, quals: Qualifiers(),
577	isDestructed: AggValueSlot::IsDestructed,
578	needsGC: AggValueSlot::DoesNotNeedGCBarriers,
579	isAliased: AggValueSlot::IsNotAliased,
580	mayOverlap: CGF.getOverlapForBaseInit(RD: ClassDecl, BaseRD: BaseClassDecl, IsVirtual: isBaseVirtual));
581
582	CGF.EmitAggExpr(E: BaseInit->getInit(), AS: AggSlot);
583
584	if (CGF.CGM.getLangOpts().Exceptions &&
585	!BaseClassDecl->hasTrivialDestructor())
586	CGF.EHStack.pushCleanup<CallBaseDtor>(Kind: EHCleanup, A: BaseClassDecl,
587	A: isBaseVirtual);
588	}
589
590	static bool isMemcpyEquivalentSpecialMember(const CXXMethodDecl *D) {
591	auto *CD = dyn_cast<CXXConstructorDecl>(Val: D);
592	if (!(CD && CD->isCopyOrMoveConstructor()) &&
593	!D->isCopyAssignmentOperator() && !D->isMoveAssignmentOperator())
594	return false;
595
596	// We can emit a memcpy for a trivial copy or move constructor/assignment.
597	if (D->isTrivial() && !D->getParent()->mayInsertExtraPadding())
598	return true;
599
600	// We *must* emit a memcpy for a defaulted union copy or move op.
601	if (D->getParent()->isUnion() && D->isDefaulted())
602	return true;
603
604	return false;
605	}
606
607	static void EmitLValueForAnyFieldInitialization(CodeGenFunction &CGF,
608	CXXCtorInitializer *MemberInit,
609	LValue &LHS) {
610	FieldDecl *Field = MemberInit->getAnyMember();
611	if (MemberInit->isIndirectMemberInitializer()) {
612	// If we are initializing an anonymous union field, drill down to the field.
613	IndirectFieldDecl *IndirectField = MemberInit->getIndirectMember();
614	for (const auto *I : IndirectField->chain())
615	LHS = CGF.EmitLValueForFieldInitialization(Base: LHS, Field: cast<FieldDecl>(Val: I));
616	} else {
617	LHS = CGF.EmitLValueForFieldInitialization(Base: LHS, Field);
618	}
619	}
620
621	static void EmitMemberInitializer(CodeGenFunction &CGF,
622	const CXXRecordDecl *ClassDecl,
623	CXXCtorInitializer *MemberInit,
624	const CXXConstructorDecl *Constructor,
625	FunctionArgList &Args) {
626	ApplyDebugLocation Loc(CGF, MemberInit->getSourceLocation());
627	assert(MemberInit->isAnyMemberInitializer() &&
628	"Must have member initializer!");
629	assert(MemberInit->getInit() && "Must have initializer!");
630
631	// non-static data member initializers.
632	FieldDecl *Field = MemberInit->getAnyMember();
633	QualType FieldType = Field->getType();
634
635	llvm::Value *ThisPtr = CGF.LoadCXXThis();
636	QualType RecordTy = CGF.getContext().getTypeDeclType(ClassDecl);
637	LValue LHS;
638
639	// If a base constructor is being emitted, create an LValue that has the
640	// non-virtual alignment.
641	if (CGF.CurGD.getCtorType() == Ctor_Base)
642	LHS = CGF.MakeNaturalAlignPointeeAddrLValue(V: ThisPtr, T: RecordTy);
643	else
644	LHS = CGF.MakeNaturalAlignAddrLValue(V: ThisPtr, T: RecordTy);
645
646	EmitLValueForAnyFieldInitialization(CGF, MemberInit, LHS);
647
648	// Special case: if we are in a copy or move constructor, and we are copying
649	// an array of PODs or classes with trivial copy constructors, ignore the
650	// AST and perform the copy we know is equivalent.
651	// FIXME: This is hacky at best... if we had a bit more explicit information
652	// in the AST, we could generalize it more easily.
653	const ConstantArrayType *Array
654	= CGF.getContext().getAsConstantArrayType(T: FieldType);
655	if (Array && Constructor->isDefaulted() &&
656	Constructor->isCopyOrMoveConstructor()) {
657	QualType BaseElementTy = CGF.getContext().getBaseElementType(Array);
658	CXXConstructExpr *CE = dyn_cast<CXXConstructExpr>(Val: MemberInit->getInit());
659	if (BaseElementTy.isPODType(Context: CGF.getContext()) ||
660	(CE && isMemcpyEquivalentSpecialMember(CE->getConstructor()))) {
661	unsigned SrcArgIndex =
662	CGF.CGM.getCXXABI().getSrcArgforCopyCtor(Constructor, Args);
663	llvm::Value *SrcPtr
664	= CGF.Builder.CreateLoad(Addr: CGF.GetAddrOfLocalVar(VD: Args[SrcArgIndex]));
665	LValue ThisRHSLV = CGF.MakeNaturalAlignAddrLValue(V: SrcPtr, T: RecordTy);
666	LValue Src = CGF.EmitLValueForFieldInitialization(Base: ThisRHSLV, Field);
667
668	// Copy the aggregate.
669	CGF.EmitAggregateCopy(Dest: LHS, Src, EltTy: FieldType, MayOverlap: CGF.getOverlapForFieldInit(FD: Field),
670	isVolatile: LHS.isVolatileQualified());
671	// Ensure that we destroy the objects if an exception is thrown later in
672	// the constructor.
673	QualType::DestructionKind dtorKind = FieldType.isDestructedType();
674	if (CGF.needsEHCleanup(kind: dtorKind))
675	CGF.pushEHDestroy(dtorKind, addr: LHS.getAddress(CGF), type: FieldType);
676	return;
677	}
678	}
679
680	CGF.EmitInitializerForField(Field, LHS, Init: MemberInit->getInit());
681	}
682
683	void CodeGenFunction::EmitInitializerForField(FieldDecl *Field, LValue LHS,
684	Expr *Init) {
685	QualType FieldType = Field->getType();
686	switch (getEvaluationKind(T: FieldType)) {
687	case TEK_Scalar:
688	if (LHS.isSimple()) {
689	EmitExprAsInit(Init, Field, LHS, false);
690	} else {
691	RValue RHS = RValue::get(V: EmitScalarExpr(E: Init));
692	EmitStoreThroughLValue(Src: RHS, Dst: LHS);
693	}
694	break;
695	case TEK_Complex:
696	EmitComplexExprIntoLValue(E: Init, dest: LHS, /*isInit*/ true);
697	break;
698	case TEK_Aggregate: {
699	AggValueSlot Slot = AggValueSlot::forLValue(
700	LV: LHS, CGF&: *this, isDestructed: AggValueSlot::IsDestructed,
701	needsGC: AggValueSlot::DoesNotNeedGCBarriers, isAliased: AggValueSlot::IsNotAliased,
702	mayOverlap: getOverlapForFieldInit(FD: Field), isZeroed: AggValueSlot::IsNotZeroed,
703	// Checks are made by the code that calls constructor.
704	isChecked: AggValueSlot::IsSanitizerChecked);
705	EmitAggExpr(E: Init, AS: Slot);
706	break;
707	}
708	}
709
710	// Ensure that we destroy this object if an exception is thrown
711	// later in the constructor.
712	QualType::DestructionKind dtorKind = FieldType.isDestructedType();
713	if (needsEHCleanup(kind: dtorKind))
714	pushEHDestroy(dtorKind, addr: LHS.getAddress(CGF&: *this), type: FieldType);
715	}
716
717	/// Checks whether the given constructor is a valid subject for the
718	/// complete-to-base constructor delegation optimization, i.e.
719	/// emitting the complete constructor as a simple call to the base
720	/// constructor.
721	bool CodeGenFunction::IsConstructorDelegationValid(
722	const CXXConstructorDecl *Ctor) {
723
724	// Currently we disable the optimization for classes with virtual
725	// bases because (1) the addresses of parameter variables need to be
726	// consistent across all initializers but (2) the delegate function
727	// call necessarily creates a second copy of the parameter variable.
728	//
729	// The limiting example (purely theoretical AFAIK):
730	// struct A { A(int &c) { c++; } };
731	// struct B : virtual A {
732	// B(int count) : A(count) { printf("%d\n", count); }
733	// };
734	// ...although even this example could in principle be emitted as a
735	// delegation since the address of the parameter doesn't escape.
736	if (Ctor->getParent()->getNumVBases()) {
737	// TODO: white-list trivial vbase initializers. This case wouldn't
738	// be subject to the restrictions below.
739
740	// TODO: white-list cases where:
741	// - there are no non-reference parameters to the constructor
742	// - the initializers don't access any non-reference parameters
743	// - the initializers don't take the address of non-reference
744	// parameters
745	// - etc.
746	// If we ever add any of the above cases, remember that:
747	// - function-try-blocks will always exclude this optimization
748	// - we need to perform the constructor prologue and cleanup in
749	// EmitConstructorBody.
750
751	return false;
752	}
753
754	// We also disable the optimization for variadic functions because
755	// it's impossible to "re-pass" varargs.
756	if (Ctor->getType()->castAs<FunctionProtoType>()->isVariadic())
757	return false;
758
759	// FIXME: Decide if we can do a delegation of a delegating constructor.
760	if (Ctor->isDelegatingConstructor())
761	return false;
762
763	return true;
764	}
765
766	// Emit code in ctor (Prologue==true) or dtor (Prologue==false)
767	// to poison the extra field paddings inserted under
768	// -fsanitize-address-field-padding=1|2.
769	void CodeGenFunction::EmitAsanPrologueOrEpilogue(bool Prologue) {
770	ASTContext &Context = getContext();
771	const CXXRecordDecl *ClassDecl =
772	Prologue ? cast<CXXConstructorDecl>(Val: CurGD.getDecl())->getParent()
773	: cast<CXXDestructorDecl>(Val: CurGD.getDecl())->getParent();
774	if (!ClassDecl->mayInsertExtraPadding()) return;
775
776	struct SizeAndOffset {
777	uint64_t Size;
778	uint64_t Offset;
779	};
780
781	unsigned PtrSize = CGM.getDataLayout().getPointerSizeInBits();
782	const ASTRecordLayout &Info = Context.getASTRecordLayout(ClassDecl);
783
784	// Populate sizes and offsets of fields.
785	SmallVector<SizeAndOffset, 16> SSV(Info.getFieldCount());
786	for (unsigned i = 0, e = Info.getFieldCount(); i != e; ++i)
787	SSV[i].Offset =
788	Context.toCharUnitsFromBits(BitSize: Info.getFieldOffset(FieldNo: i)).getQuantity();
789
790	size_t NumFields = 0;
791	for (const auto *Field : ClassDecl->fields()) {
792	const FieldDecl *D = Field;
793	auto FieldInfo = Context.getTypeInfoInChars(D->getType());
794	CharUnits FieldSize = FieldInfo.Width;
795	assert(NumFields < SSV.size());
796	SSV[NumFields].Size = D->isBitField() ? 0 : FieldSize.getQuantity();
797	NumFields++;
798	}
799	assert(NumFields == SSV.size());
800	if (SSV.size() <= 1) return;
801
802	// We will insert calls to __asan_* run-time functions.
803	// LLVM AddressSanitizer pass may decide to inline them later.
804	llvm::Type *Args[2] = {IntPtrTy, IntPtrTy};
805	llvm::FunctionType *FTy =
806	llvm::FunctionType::get(Result: CGM.VoidTy, Params: Args, isVarArg: false);
807	llvm::FunctionCallee F = CGM.CreateRuntimeFunction(
808	Ty: FTy, Name: Prologue ? "__asan_poison_intra_object_redzone"
809	: "__asan_unpoison_intra_object_redzone");
810
811	llvm::Value *ThisPtr = LoadCXXThis();
812	ThisPtr = Builder.CreatePtrToInt(V: ThisPtr, DestTy: IntPtrTy);
813	uint64_t TypeSize = Info.getNonVirtualSize().getQuantity();
814	// For each field check if it has sufficient padding,
815	// if so (un)poison it with a call.
816	for (size_t i = 0; i < SSV.size(); i++) {
817	uint64_t AsanAlignment = 8;
818	uint64_t NextField = i == SSV.size() - 1 ? TypeSize : SSV[i + 1].Offset;
819	uint64_t PoisonSize = NextField - SSV[i].Offset - SSV[i].Size;
820	uint64_t EndOffset = SSV[i].Offset + SSV[i].Size;
821	if (PoisonSize < AsanAlignment || !SSV[i].Size ||
822	(NextField % AsanAlignment) != 0)
823	continue;
824	Builder.CreateCall(
825	Callee: F, Args: {Builder.CreateAdd(LHS: ThisPtr, RHS: Builder.getIntN(N: PtrSize, C: EndOffset)),
826	Builder.getIntN(N: PtrSize, C: PoisonSize)});
827	}
828	}
829
830	/// EmitConstructorBody - Emits the body of the current constructor.
831	void CodeGenFunction::EmitConstructorBody(FunctionArgList &Args) {
832	EmitAsanPrologueOrEpilogue(Prologue: true);
833	const CXXConstructorDecl *Ctor = cast<CXXConstructorDecl>(Val: CurGD.getDecl());
834	CXXCtorType CtorType = CurGD.getCtorType();
835
836	assert((CGM.getTarget().getCXXABI().hasConstructorVariants() ||
837	CtorType == Ctor_Complete) &&
838	"can only generate complete ctor for this ABI");
839
840	// Before we go any further, try the complete->base constructor
841	// delegation optimization.
842	if (CtorType == Ctor_Complete && IsConstructorDelegationValid(Ctor) &&
843	CGM.getTarget().getCXXABI().hasConstructorVariants()) {
844	EmitDelegateCXXConstructorCall(Ctor, CtorType: Ctor_Base, Args, Loc: Ctor->getEndLoc());
845	return;
846	}
847
848	const FunctionDecl *Definition = nullptr;
849	Stmt *Body = Ctor->getBody(Definition);
850	assert(Definition == Ctor && "emitting wrong constructor body");
851
852	// Enter the function-try-block before the constructor prologue if
853	// applicable.
854	bool IsTryBody = (Body && isa<CXXTryStmt>(Val: Body));
855	if (IsTryBody)
856	EnterCXXTryStmt(S: *cast<CXXTryStmt>(Val: Body), IsFnTryBlock: true);
857
858	incrementProfileCounter(S: Body);
859	maybeCreateMCDCCondBitmap();
860
861	RunCleanupsScope RunCleanups(*this);
862
863	// TODO: in restricted cases, we can emit the vbase initializers of
864	// a complete ctor and then delegate to the base ctor.
865
866	// Emit the constructor prologue, i.e. the base and member
867	// initializers.
868	EmitCtorPrologue(CD: Ctor, Type: CtorType, Args);
869
870	// Emit the body of the statement.
871	if (IsTryBody)
872	EmitStmt(cast<CXXTryStmt>(Val: Body)->getTryBlock());
873	else if (Body)
874	EmitStmt(S: Body);
875
876	// Emit any cleanup blocks associated with the member or base
877	// initializers, which includes (along the exceptional path) the
878	// destructors for those members and bases that were fully
879	// constructed.
880	RunCleanups.ForceCleanup();
881
882	if (IsTryBody)
883	ExitCXXTryStmt(S: *cast<CXXTryStmt>(Val: Body), IsFnTryBlock: true);
884	}
885
886	namespace {
887	/// RAII object to indicate that codegen is copying the value representation
888	/// instead of the object representation. Useful when copying a struct or
889	/// class which has uninitialized members and we're only performing
890	/// lvalue-to-rvalue conversion on the object but not its members.
891	class CopyingValueRepresentation {
892	public:
893	explicit CopyingValueRepresentation(CodeGenFunction &CGF)
894	: CGF(CGF), OldSanOpts(CGF.SanOpts) {
895	CGF.SanOpts.set(K: SanitizerKind::Bool, Value: false);
896	CGF.SanOpts.set(K: SanitizerKind::Enum, Value: false);
897	}
898	~CopyingValueRepresentation() {
899	CGF.SanOpts = OldSanOpts;
900	}
901	private:
902	CodeGenFunction &CGF;
903	SanitizerSet OldSanOpts;
904	};
905	} // end anonymous namespace
906
907	namespace {
908	class FieldMemcpyizer {
909	public:
910	FieldMemcpyizer(CodeGenFunction &CGF, const CXXRecordDecl *ClassDecl,
911	const VarDecl *SrcRec)
912	: CGF(CGF), ClassDecl(ClassDecl), SrcRec(SrcRec),
913	RecLayout(CGF.getContext().getASTRecordLayout(ClassDecl)),
914	FirstField(nullptr), LastField(nullptr), FirstFieldOffset(0),
915	LastFieldOffset(0), LastAddedFieldIndex(0) {}
916
917	bool isMemcpyableField(FieldDecl *F) const {
918	// Never memcpy fields when we are adding poisoned paddings.
919	if (CGF.getContext().getLangOpts().SanitizeAddressFieldPadding)
920	return false;
921	Qualifiers Qual = F->getType().getQualifiers();
922	if (Qual.hasVolatile() || Qual.hasObjCLifetime())
923	return false;
924	return true;
925	}
926
927	void addMemcpyableField(FieldDecl *F) {
928	if (F->isZeroSize(Ctx: CGF.getContext()))
929	return;
930	if (!FirstField)
931	addInitialField(F);
932	else
933	addNextField(F);
934	}
935
936	CharUnits getMemcpySize(uint64_t FirstByteOffset) const {
937	ASTContext &Ctx = CGF.getContext();
938	unsigned LastFieldSize =
939	LastField->isBitField()
940	? LastField->getBitWidthValue(Ctx)
941	: Ctx.toBits(
942	CharSize: Ctx.getTypeInfoDataSizeInChars(T: LastField->getType()).Width);
943	uint64_t MemcpySizeBits = LastFieldOffset + LastFieldSize -
944	FirstByteOffset + Ctx.getCharWidth() - 1;
945	CharUnits MemcpySize = Ctx.toCharUnitsFromBits(BitSize: MemcpySizeBits);
946	return MemcpySize;
947	}
948
949	void emitMemcpy() {
950	// Give the subclass a chance to bail out if it feels the memcpy isn't
951	// worth it (e.g. Hasn't aggregated enough data).
952	if (!FirstField) {
953	return;
954	}
955
956	uint64_t FirstByteOffset;
957	if (FirstField->isBitField()) {
958	const CGRecordLayout &RL =
959	CGF.getTypes().getCGRecordLayout(FirstField->getParent());
960	const CGBitFieldInfo &BFInfo = RL.getBitFieldInfo(FD: FirstField);
961	// FirstFieldOffset is not appropriate for bitfields,
962	// we need to use the storage offset instead.
963	FirstByteOffset = CGF.getContext().toBits(CharSize: BFInfo.StorageOffset);
964	} else {
965	FirstByteOffset = FirstFieldOffset;
966	}
967
968	CharUnits MemcpySize = getMemcpySize(FirstByteOffset);
969	QualType RecordTy = CGF.getContext().getTypeDeclType(ClassDecl);
970	Address ThisPtr = CGF.LoadCXXThisAddress();
971	LValue DestLV = CGF.MakeAddrLValue(Addr: ThisPtr, T: RecordTy);
972	LValue Dest = CGF.EmitLValueForFieldInitialization(Base: DestLV, Field: FirstField);
973	llvm::Value *SrcPtr = CGF.Builder.CreateLoad(Addr: CGF.GetAddrOfLocalVar(VD: SrcRec));
974	LValue SrcLV = CGF.MakeNaturalAlignAddrLValue(V: SrcPtr, T: RecordTy);
975	LValue Src = CGF.EmitLValueForFieldInitialization(Base: SrcLV, Field: FirstField);
976
977	emitMemcpyIR(
978	DestPtr: Dest.isBitField() ? Dest.getBitFieldAddress() : Dest.getAddress(CGF),
979	SrcPtr: Src.isBitField() ? Src.getBitFieldAddress() : Src.getAddress(CGF),
980	Size: MemcpySize);
981	reset();
982	}
983
984	void reset() {
985	FirstField = nullptr;
986	}
987
988	protected:
989	CodeGenFunction &CGF;
990	const CXXRecordDecl *ClassDecl;
991
992	private:
993	void emitMemcpyIR(Address DestPtr, Address SrcPtr, CharUnits Size) {
994	DestPtr = DestPtr.withElementType(ElemTy: CGF.Int8Ty);
995	SrcPtr = SrcPtr.withElementType(ElemTy: CGF.Int8Ty);
996	CGF.Builder.CreateMemCpy(Dest: DestPtr, Src: SrcPtr, Size: Size.getQuantity());
997	}
998
999	void addInitialField(FieldDecl *F) {
1000	FirstField = F;
1001	LastField = F;
1002	FirstFieldOffset = RecLayout.getFieldOffset(FieldNo: F->getFieldIndex());
1003	LastFieldOffset = FirstFieldOffset;
1004	LastAddedFieldIndex = F->getFieldIndex();
1005	}
1006
1007	void addNextField(FieldDecl *F) {
1008	// For the most part, the following invariant will hold:
1009	// F->getFieldIndex() == LastAddedFieldIndex + 1
1010	// The one exception is that Sema won't add a copy-initializer for an
1011	// unnamed bitfield, which will show up here as a gap in the sequence.
1012	assert(F->getFieldIndex() >= LastAddedFieldIndex + 1 &&
1013	"Cannot aggregate fields out of order.");
1014	LastAddedFieldIndex = F->getFieldIndex();
1015
1016	// The 'first' and 'last' fields are chosen by offset, rather than field
1017	// index. This allows the code to support bitfields, as well as regular
1018	// fields.
1019	uint64_t FOffset = RecLayout.getFieldOffset(FieldNo: F->getFieldIndex());
1020	if (FOffset < FirstFieldOffset) {
1021	FirstField = F;
1022	FirstFieldOffset = FOffset;
1023	} else if (FOffset >= LastFieldOffset) {
1024	LastField = F;
1025	LastFieldOffset = FOffset;
1026	}
1027	}
1028
1029	const VarDecl *SrcRec;
1030	const ASTRecordLayout &RecLayout;
1031	FieldDecl *FirstField;
1032	FieldDecl *LastField;
1033	uint64_t FirstFieldOffset, LastFieldOffset;
1034	unsigned LastAddedFieldIndex;
1035	};
1036
1037	class ConstructorMemcpyizer : public FieldMemcpyizer {
1038	private:
1039	/// Get source argument for copy constructor. Returns null if not a copy
1040	/// constructor.
1041	static const VarDecl *getTrivialCopySource(CodeGenFunction &CGF,
1042	const CXXConstructorDecl *CD,
1043	FunctionArgList &Args) {
1044	if (CD->isCopyOrMoveConstructor() && CD->isDefaulted())
1045	return Args[CGF.CGM.getCXXABI().getSrcArgforCopyCtor(CD, Args)];
1046	return nullptr;
1047	}
1048
1049	// Returns true if a CXXCtorInitializer represents a member initialization
1050	// that can be rolled into a memcpy.
1051	bool isMemberInitMemcpyable(CXXCtorInitializer *MemberInit) const {
1052	if (!MemcpyableCtor)
1053	return false;
1054	FieldDecl *Field = MemberInit->getMember();
1055	assert(Field && "No field for member init.");
1056	QualType FieldType = Field->getType();
1057	CXXConstructExpr *CE = dyn_cast<CXXConstructExpr>(Val: MemberInit->getInit());
1058
1059	// Bail out on non-memcpyable, not-trivially-copyable members.
1060	if (!(CE && isMemcpyEquivalentSpecialMember(CE->getConstructor())) &&
1061	!(FieldType.isTriviallyCopyableType(Context: CGF.getContext()) ||
1062	FieldType->isReferenceType()))
1063	return false;
1064
1065	// Bail out on volatile fields.
1066	if (!isMemcpyableField(F: Field))
1067	return false;
1068
1069	// Otherwise we're good.
1070	return true;
1071	}
1072
1073	public:
1074	ConstructorMemcpyizer(CodeGenFunction &CGF, const CXXConstructorDecl *CD,
1075	FunctionArgList &Args)
1076	: FieldMemcpyizer(CGF, CD->getParent(), getTrivialCopySource(CGF, CD, Args)),
1077	ConstructorDecl(CD),
1078	MemcpyableCtor(CD->isDefaulted() &&
1079	CD->isCopyOrMoveConstructor() &&
1080	CGF.getLangOpts().getGC() == LangOptions::NonGC),
1081	Args(Args) { }
1082
1083	void addMemberInitializer(CXXCtorInitializer *MemberInit) {
1084	if (isMemberInitMemcpyable(MemberInit)) {
1085	AggregatedInits.push_back(Elt: MemberInit);
1086	addMemcpyableField(F: MemberInit->getMember());
1087	} else {
1088	emitAggregatedInits();
1089	EmitMemberInitializer(CGF, ConstructorDecl->getParent(), MemberInit,
1090	ConstructorDecl, Args);
1091	}
1092	}
1093
1094	void emitAggregatedInits() {
1095	if (AggregatedInits.size() <= 1) {
1096	// This memcpy is too small to be worthwhile. Fall back on default
1097	// codegen.
1098	if (!AggregatedInits.empty()) {
1099	CopyingValueRepresentation CVR(CGF);
1100	EmitMemberInitializer(CGF, ConstructorDecl->getParent(),
1101	AggregatedInits[0], ConstructorDecl, Args);
1102	AggregatedInits.clear();
1103	}
1104	reset();
1105	return;
1106	}
1107
1108	pushEHDestructors();
1109	emitMemcpy();
1110	AggregatedInits.clear();
1111	}
1112
1113	void pushEHDestructors() {
1114	Address ThisPtr = CGF.LoadCXXThisAddress();
1115	QualType RecordTy = CGF.getContext().getTypeDeclType(ClassDecl);
1116	LValue LHS = CGF.MakeAddrLValue(Addr: ThisPtr, T: RecordTy);
1117
1118	for (unsigned i = 0; i < AggregatedInits.size(); ++i) {
1119	CXXCtorInitializer *MemberInit = AggregatedInits[i];
1120	QualType FieldType = MemberInit->getAnyMember()->getType();
1121	QualType::DestructionKind dtorKind = FieldType.isDestructedType();
1122	if (!CGF.needsEHCleanup(kind: dtorKind))
1123	continue;
1124	LValue FieldLHS = LHS;
1125	EmitLValueForAnyFieldInitialization(CGF, MemberInit, LHS&: FieldLHS);
1126	CGF.pushEHDestroy(dtorKind, addr: FieldLHS.getAddress(CGF), type: FieldType);
1127	}
1128	}
1129
1130	void finish() {
1131	emitAggregatedInits();
1132	}
1133
1134	private:
1135	const CXXConstructorDecl *ConstructorDecl;
1136	bool MemcpyableCtor;
1137	FunctionArgList &Args;
1138	SmallVector<CXXCtorInitializer*, 16> AggregatedInits;
1139	};
1140
1141	class AssignmentMemcpyizer : public FieldMemcpyizer {
1142	private:
1143	// Returns the memcpyable field copied by the given statement, if one
1144	// exists. Otherwise returns null.
1145	FieldDecl *getMemcpyableField(Stmt *S) {
1146	if (!AssignmentsMemcpyable)
1147	return nullptr;
1148	if (BinaryOperator *BO = dyn_cast<BinaryOperator>(Val: S)) {
1149	// Recognise trivial assignments.
1150	if (BO->getOpcode() != BO_Assign)
1151	return nullptr;
1152	MemberExpr *ME = dyn_cast<MemberExpr>(Val: BO->getLHS());
1153	if (!ME)
1154	return nullptr;
1155	FieldDecl *Field = dyn_cast<FieldDecl>(Val: ME->getMemberDecl());
1156	if (!Field || !isMemcpyableField(F: Field))
1157	return nullptr;
1158	Stmt *RHS = BO->getRHS();
1159	if (ImplicitCastExpr *EC = dyn_cast<ImplicitCastExpr>(Val: RHS))
1160	RHS = EC->getSubExpr();
1161	if (!RHS)
1162	return nullptr;
1163	if (MemberExpr *ME2 = dyn_cast<MemberExpr>(Val: RHS)) {
1164	if (ME2->getMemberDecl() == Field)
1165	return Field;
1166	}
1167	return nullptr;
1168	} else if (CXXMemberCallExpr *MCE = dyn_cast<CXXMemberCallExpr>(Val: S)) {
1169	CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(MCE->getCalleeDecl());
1170	if (!(MD && isMemcpyEquivalentSpecialMember(D: MD)))
1171	return nullptr;
1172	MemberExpr *IOA = dyn_cast<MemberExpr>(Val: MCE->getImplicitObjectArgument());
1173	if (!IOA)
1174	return nullptr;
1175	FieldDecl *Field = dyn_cast<FieldDecl>(Val: IOA->getMemberDecl());
1176	if (!Field || !isMemcpyableField(F: Field))
1177	return nullptr;
1178	MemberExpr *Arg0 = dyn_cast<MemberExpr>(MCE->getArg(0));
1179	if (!Arg0 || Field != dyn_cast<FieldDecl>(Val: Arg0->getMemberDecl()))
1180	return nullptr;
1181	return Field;
1182	} else if (CallExpr *CE = dyn_cast<CallExpr>(Val: S)) {
1183	FunctionDecl *FD = dyn_cast<FunctionDecl>(Val: CE->getCalleeDecl());
1184	if (!FD || FD->getBuiltinID() != Builtin::BI__builtin_memcpy)
1185	return nullptr;
1186	Expr *DstPtr = CE->getArg(Arg: 0);
1187	if (ImplicitCastExpr *DC = dyn_cast<ImplicitCastExpr>(Val: DstPtr))
1188	DstPtr = DC->getSubExpr();
1189	UnaryOperator *DUO = dyn_cast<UnaryOperator>(Val: DstPtr);
1190	if (!DUO || DUO->getOpcode() != UO_AddrOf)
1191	return nullptr;
1192	MemberExpr *ME = dyn_cast<MemberExpr>(Val: DUO->getSubExpr());
1193	if (!ME)
1194	return nullptr;
1195	FieldDecl *Field = dyn_cast<FieldDecl>(Val: ME->getMemberDecl());
1196	if (!Field || !isMemcpyableField(F: Field))
1197	return nullptr;
1198	Expr *SrcPtr = CE->getArg(Arg: 1);
1199	if (ImplicitCastExpr *SC = dyn_cast<ImplicitCastExpr>(Val: SrcPtr))
1200	SrcPtr = SC->getSubExpr();
1201	UnaryOperator *SUO = dyn_cast<UnaryOperator>(Val: SrcPtr);
1202	if (!SUO || SUO->getOpcode() != UO_AddrOf)
1203	return nullptr;
1204	MemberExpr *ME2 = dyn_cast<MemberExpr>(Val: SUO->getSubExpr());
1205	if (!ME2 || Field != dyn_cast<FieldDecl>(Val: ME2->getMemberDecl()))
1206	return nullptr;
1207	return Field;
1208	}
1209
1210	return nullptr;
1211	}
1212
1213	bool AssignmentsMemcpyable;
1214	SmallVector<Stmt*, 16> AggregatedStmts;
1215
1216	public:
1217	AssignmentMemcpyizer(CodeGenFunction &CGF, const CXXMethodDecl *AD,
1218	FunctionArgList &Args)
1219	: FieldMemcpyizer(CGF, AD->getParent(), Args[Args.size() - 1]),
1220	AssignmentsMemcpyable(CGF.getLangOpts().getGC() == LangOptions::NonGC) {
1221	assert(Args.size() == 2);
1222	}
1223
1224	void emitAssignment(Stmt *S) {
1225	FieldDecl *F = getMemcpyableField(S);
1226	if (F) {
1227	addMemcpyableField(F);
1228	AggregatedStmts.push_back(Elt: S);
1229	} else {
1230	emitAggregatedStmts();
1231	CGF.EmitStmt(S);
1232	}
1233	}
1234
1235	void emitAggregatedStmts() {
1236	if (AggregatedStmts.size() <= 1) {
1237	if (!AggregatedStmts.empty()) {
1238	CopyingValueRepresentation CVR(CGF);
1239	CGF.EmitStmt(S: AggregatedStmts[0]);
1240	}
1241	reset();
1242	}
1243
1244	emitMemcpy();
1245	AggregatedStmts.clear();
1246	}
1247
1248	void finish() {
1249	emitAggregatedStmts();
1250	}
1251	};
1252	} // end anonymous namespace
1253
1254	static bool isInitializerOfDynamicClass(const CXXCtorInitializer *BaseInit) {
1255	const Type *BaseType = BaseInit->getBaseClass();
1256	const auto *BaseClassDecl =
1257	cast<CXXRecordDecl>(Val: BaseType->castAs<RecordType>()->getDecl());
1258	return BaseClassDecl->isDynamicClass();
1259	}
1260
1261	/// EmitCtorPrologue - This routine generates necessary code to initialize
1262	/// base classes and non-static data members belonging to this constructor.
1263	void CodeGenFunction::EmitCtorPrologue(const CXXConstructorDecl *CD,
1264	CXXCtorType CtorType,
1265	FunctionArgList &Args) {
1266	if (CD->isDelegatingConstructor())
1267	return EmitDelegatingCXXConstructorCall(Ctor: CD, Args);
1268
1269	const CXXRecordDecl *ClassDecl = CD->getParent();
1270
1271	CXXConstructorDecl::init_const_iterator B = CD->init_begin(),
1272	E = CD->init_end();
1273
1274	// Virtual base initializers first, if any. They aren't needed if:
1275	// - This is a base ctor variant
1276	// - There are no vbases
1277	// - The class is abstract, so a complete object of it cannot be constructed
1278	//
1279	// The check for an abstract class is necessary because sema may not have
1280	// marked virtual base destructors referenced.
1281	bool ConstructVBases = CtorType != Ctor_Base &&
1282	ClassDecl->getNumVBases() != 0 &&
1283	!ClassDecl->isAbstract();
1284
1285	// In the Microsoft C++ ABI, there are no constructor variants. Instead, the
1286	// constructor of a class with virtual bases takes an additional parameter to
1287	// conditionally construct the virtual bases. Emit that check here.
1288	llvm::BasicBlock *BaseCtorContinueBB = nullptr;
1289	if (ConstructVBases &&
1290	!CGM.getTarget().getCXXABI().hasConstructorVariants()) {
1291	BaseCtorContinueBB =
1292	CGM.getCXXABI().EmitCtorCompleteObjectHandler(CGF&: *this, RD: ClassDecl);
1293	assert(BaseCtorContinueBB);
1294	}
1295
1296	for (; B != E && (*B)->isBaseInitializer() && (*B)->isBaseVirtual(); B++) {
1297	if (!ConstructVBases)
1298	continue;
1299	SaveAndRestore ThisRAII(CXXThisValue);
1300	if (CGM.getCodeGenOpts().StrictVTablePointers &&
1301	CGM.getCodeGenOpts().OptimizationLevel > 0 &&
1302	isInitializerOfDynamicClass(BaseInit: *B))
1303	CXXThisValue = Builder.CreateLaunderInvariantGroup(Ptr: LoadCXXThis());
1304	EmitBaseInitializer(CGF&: *this, ClassDecl, BaseInit: *B);
1305	}
1306
1307	if (BaseCtorContinueBB) {
1308	// Complete object handler should continue to the remaining initializers.
1309	Builder.CreateBr(Dest: BaseCtorContinueBB);
1310	EmitBlock(BB: BaseCtorContinueBB);
1311	}
1312
1313	// Then, non-virtual base initializers.
1314	for (; B != E && (*B)->isBaseInitializer(); B++) {
1315	assert(!(*B)->isBaseVirtual());
1316	SaveAndRestore ThisRAII(CXXThisValue);
1317	if (CGM.getCodeGenOpts().StrictVTablePointers &&
1318	CGM.getCodeGenOpts().OptimizationLevel > 0 &&
1319	isInitializerOfDynamicClass(BaseInit: *B))
1320	CXXThisValue = Builder.CreateLaunderInvariantGroup(Ptr: LoadCXXThis());
1321	EmitBaseInitializer(CGF&: *this, ClassDecl, BaseInit: *B);
1322	}
1323
1324	InitializeVTablePointers(ClassDecl);
1325
1326	// And finally, initialize class members.
1327	FieldConstructionScope FCS(*this, LoadCXXThisAddress());
1328	ConstructorMemcpyizer CM(*this, CD, Args);
1329	for (; B != E; B++) {
1330	CXXCtorInitializer *Member = (*B);
1331	assert(!Member->isBaseInitializer());
1332	assert(Member->isAnyMemberInitializer() &&
1333	"Delegating initializer on non-delegating constructor");
1334	CM.addMemberInitializer(MemberInit: Member);
1335	}
1336	CM.finish();
1337	}
1338
1339	static bool
1340	FieldHasTrivialDestructorBody(ASTContext &Context, const FieldDecl *Field);
1341
1342	static bool
1343	HasTrivialDestructorBody(ASTContext &Context,
1344	const CXXRecordDecl *BaseClassDecl,
1345	const CXXRecordDecl *MostDerivedClassDecl)
1346	{
1347	// If the destructor is trivial we don't have to check anything else.
1348	if (BaseClassDecl->hasTrivialDestructor())
1349	return true;
1350
1351	if (!BaseClassDecl->getDestructor()->hasTrivialBody())
1352	return false;
1353
1354	// Check fields.
1355	for (const auto *Field : BaseClassDecl->fields())
1356	if (!FieldHasTrivialDestructorBody(Context, Field))
1357	return false;
1358
1359	// Check non-virtual bases.
1360	for (const auto &I : BaseClassDecl->bases()) {
1361	if (I.isVirtual())
1362	continue;
1363
1364	const CXXRecordDecl *NonVirtualBase =
1365	cast<CXXRecordDecl>(Val: I.getType()->castAs<RecordType>()->getDecl());
1366	if (!HasTrivialDestructorBody(Context, BaseClassDecl: NonVirtualBase,
1367	MostDerivedClassDecl))
1368	return false;
1369	}
1370
1371	if (BaseClassDecl == MostDerivedClassDecl) {
1372	// Check virtual bases.
1373	for (const auto &I : BaseClassDecl->vbases()) {
1374	const CXXRecordDecl *VirtualBase =
1375	cast<CXXRecordDecl>(Val: I.getType()->castAs<RecordType>()->getDecl());
1376	if (!HasTrivialDestructorBody(Context, BaseClassDecl: VirtualBase,
1377	MostDerivedClassDecl))
1378	return false;
1379	}
1380	}
1381
1382	return true;
1383	}
1384
1385	static bool
1386	FieldHasTrivialDestructorBody(ASTContext &Context,
1387	const FieldDecl *Field)
1388	{
1389	QualType FieldBaseElementType = Context.getBaseElementType(Field->getType());
1390
1391	const RecordType *RT = FieldBaseElementType->getAs<RecordType>();
1392	if (!RT)
1393	return true;
1394
1395	CXXRecordDecl *FieldClassDecl = cast<CXXRecordDecl>(Val: RT->getDecl());
1396
1397	// The destructor for an implicit anonymous union member is never invoked.
1398	if (FieldClassDecl->isUnion() && FieldClassDecl->isAnonymousStructOrUnion())
1399	return false;
1400
1401	return HasTrivialDestructorBody(Context, BaseClassDecl: FieldClassDecl, MostDerivedClassDecl: FieldClassDecl);
1402	}
1403
1404	/// CanSkipVTablePointerInitialization - Check whether we need to initialize
1405	/// any vtable pointers before calling this destructor.
1406	static bool CanSkipVTablePointerInitialization(CodeGenFunction &CGF,
1407	const CXXDestructorDecl *Dtor) {
1408	const CXXRecordDecl *ClassDecl = Dtor->getParent();
1409	if (!ClassDecl->isDynamicClass())
1410	return true;
1411
1412	// For a final class, the vtable pointer is known to already point to the
1413	// class's vtable.
1414	if (ClassDecl->isEffectivelyFinal())
1415	return true;
1416
1417	if (!Dtor->hasTrivialBody())
1418	return false;
1419
1420	// Check the fields.
1421	for (const auto *Field : ClassDecl->fields())
1422	if (!FieldHasTrivialDestructorBody(CGF.getContext(), Field))
1423	return false;
1424
1425	return true;
1426	}
1427
1428	/// EmitDestructorBody - Emits the body of the current destructor.
1429	void CodeGenFunction::EmitDestructorBody(FunctionArgList &Args) {
1430	const CXXDestructorDecl *Dtor = cast<CXXDestructorDecl>(Val: CurGD.getDecl());
1431	CXXDtorType DtorType = CurGD.getDtorType();
1432
1433	// For an abstract class, non-base destructors are never used (and can't
1434	// be emitted in general, because vbase dtors may not have been validated
1435	// by Sema), but the Itanium ABI doesn't make them optional and Clang may
1436	// in fact emit references to them from other compilations, so emit them
1437	// as functions containing a trap instruction.
1438	if (DtorType != Dtor_Base && Dtor->getParent()->isAbstract()) {
1439	llvm::CallInst *TrapCall = EmitTrapCall(llvm::Intrinsic::trap);
1440	TrapCall->setDoesNotReturn();
1441	TrapCall->setDoesNotThrow();
1442	Builder.CreateUnreachable();
1443	Builder.ClearInsertionPoint();
1444	return;
1445	}
1446
1447	Stmt *Body = Dtor->getBody();
1448	if (Body) {
1449	incrementProfileCounter(S: Body);
1450	maybeCreateMCDCCondBitmap();
1451	}
1452
1453	// The call to operator delete in a deleting destructor happens
1454	// outside of the function-try-block, which means it's always
1455	// possible to delegate the destructor body to the complete
1456	// destructor. Do so.
1457	if (DtorType == Dtor_Deleting) {
1458	RunCleanupsScope DtorEpilogue(*this);
1459	EnterDtorCleanups(Dtor, Type: Dtor_Deleting);
1460	if (HaveInsertPoint()) {
1461	QualType ThisTy = Dtor->getFunctionObjectParameterType();
1462	EmitCXXDestructorCall(D: Dtor, Type: Dtor_Complete, /*ForVirtualBase=*/false,
1463	/*Delegating=*/false, This: LoadCXXThisAddress(), ThisTy);
1464	}
1465	return;
1466	}
1467
1468	// If the body is a function-try-block, enter the try before
1469	// anything else.
1470	bool isTryBody = (Body && isa<CXXTryStmt>(Val: Body));
1471	if (isTryBody)
1472	EnterCXXTryStmt(S: *cast<CXXTryStmt>(Val: Body), IsFnTryBlock: true);
1473	EmitAsanPrologueOrEpilogue(Prologue: false);
1474
1475	// Enter the epilogue cleanups.
1476	RunCleanupsScope DtorEpilogue(*this);
1477
1478	// If this is the complete variant, just invoke the base variant;
1479	// the epilogue will destruct the virtual bases. But we can't do
1480	// this optimization if the body is a function-try-block, because
1481	// we'd introduce *two* handler blocks. In the Microsoft ABI, we
1482	// always delegate because we might not have a definition in this TU.
1483	switch (DtorType) {
1484	case Dtor_Comdat: llvm_unreachable("not expecting a COMDAT");
1485	case Dtor_Deleting: llvm_unreachable("already handled deleting case");
1486
1487	case Dtor_Complete:
1488	assert((Body || getTarget().getCXXABI().isMicrosoft()) &&
1489	"can't emit a dtor without a body for non-Microsoft ABIs");
1490
1491	// Enter the cleanup scopes for virtual bases.
1492	EnterDtorCleanups(Dtor, Type: Dtor_Complete);
1493
1494	if (!isTryBody) {
1495	QualType ThisTy = Dtor->getFunctionObjectParameterType();
1496	EmitCXXDestructorCall(D: Dtor, Type: Dtor_Base, /*ForVirtualBase=*/false,
1497	/*Delegating=*/false, This: LoadCXXThisAddress(), ThisTy);
1498	break;
1499	}
1500
1501	// Fallthrough: act like we're in the base variant.
1502	[[fallthrough]];
1503
1504	case Dtor_Base:
1505	assert(Body);
1506
1507	// Enter the cleanup scopes for fields and non-virtual bases.
1508	EnterDtorCleanups(Dtor, Type: Dtor_Base);
1509
1510	// Initialize the vtable pointers before entering the body.
1511	if (!CanSkipVTablePointerInitialization(CGF&: *this, Dtor)) {
1512	// Insert the llvm.launder.invariant.group intrinsic before initializing
1513	// the vptrs to cancel any previous assumptions we might have made.
1514	if (CGM.getCodeGenOpts().StrictVTablePointers &&
1515	CGM.getCodeGenOpts().OptimizationLevel > 0)
1516	CXXThisValue = Builder.CreateLaunderInvariantGroup(Ptr: LoadCXXThis());
1517	InitializeVTablePointers(ClassDecl: Dtor->getParent());
1518	}
1519
1520	if (isTryBody)
1521	EmitStmt(cast<CXXTryStmt>(Val: Body)->getTryBlock());
1522	else if (Body)
1523	EmitStmt(S: Body);
1524	else {
1525	assert(Dtor->isImplicit() && "bodyless dtor not implicit");
1526	// nothing to do besides what's in the epilogue
1527	}
1528	// -fapple-kext must inline any call to this dtor into
1529	// the caller's body.
1530	if (getLangOpts().AppleKext)
1531	CurFn->addFnAttr(llvm::Attribute::AlwaysInline);
1532
1533	break;
1534	}
1535
1536	// Jump out through the epilogue cleanups.
1537	DtorEpilogue.ForceCleanup();
1538
1539	// Exit the try if applicable.
1540	if (isTryBody)
1541	ExitCXXTryStmt(S: *cast<CXXTryStmt>(Val: Body), IsFnTryBlock: true);
1542	}
1543
1544	void CodeGenFunction::emitImplicitAssignmentOperatorBody(FunctionArgList &Args) {
1545	const CXXMethodDecl *AssignOp = cast<CXXMethodDecl>(Val: CurGD.getDecl());
1546	const Stmt *RootS = AssignOp->getBody();
1547	assert(isa<CompoundStmt>(RootS) &&
1548	"Body of an implicit assignment operator should be compound stmt.");
1549	const CompoundStmt *RootCS = cast<CompoundStmt>(Val: RootS);
1550
1551	LexicalScope Scope(*this, RootCS->getSourceRange());
1552
1553	incrementProfileCounter(RootCS);
1554	maybeCreateMCDCCondBitmap();
1555	AssignmentMemcpyizer AM(*this, AssignOp, Args);
1556	for (auto *I : RootCS->body())
1557	AM.emitAssignment(I);
1558	AM.finish();
1559	}
1560
1561	namespace {
1562	llvm::Value *LoadThisForDtorDelete(CodeGenFunction &CGF,
1563	const CXXDestructorDecl *DD) {
1564	if (Expr *ThisArg = DD->getOperatorDeleteThisArg())
1565	return CGF.EmitScalarExpr(E: ThisArg);
1566	return CGF.LoadCXXThis();
1567	}
1568
1569	/// Call the operator delete associated with the current destructor.
1570	struct CallDtorDelete final : EHScopeStack::Cleanup {
1571	CallDtorDelete() {}
1572
1573	void Emit(CodeGenFunction &CGF, Flags flags) override {
1574	const CXXDestructorDecl *Dtor = cast<CXXDestructorDecl>(Val: CGF.CurCodeDecl);
1575	const CXXRecordDecl *ClassDecl = Dtor->getParent();
1576	CGF.EmitDeleteCall(DeleteFD: Dtor->getOperatorDelete(),
1577	Ptr: LoadThisForDtorDelete(CGF, DD: Dtor),
1578	DeleteTy: CGF.getContext().getTagDeclType(ClassDecl));
1579	}
1580	};
1581
1582	void EmitConditionalDtorDeleteCall(CodeGenFunction &CGF,
1583	llvm::Value *ShouldDeleteCondition,
1584	bool ReturnAfterDelete) {
1585	llvm::BasicBlock *callDeleteBB = CGF.createBasicBlock(name: "dtor.call_delete");
1586	llvm::BasicBlock *continueBB = CGF.createBasicBlock(name: "dtor.continue");
1587	llvm::Value *ShouldCallDelete
1588	= CGF.Builder.CreateIsNull(Arg: ShouldDeleteCondition);
1589	CGF.Builder.CreateCondBr(Cond: ShouldCallDelete, True: continueBB, False: callDeleteBB);
1590
1591	CGF.EmitBlock(BB: callDeleteBB);
1592	const CXXDestructorDecl *Dtor = cast<CXXDestructorDecl>(Val: CGF.CurCodeDecl);
1593	const CXXRecordDecl *ClassDecl = Dtor->getParent();
1594	CGF.EmitDeleteCall(DeleteFD: Dtor->getOperatorDelete(),
1595	Ptr: LoadThisForDtorDelete(CGF, DD: Dtor),
1596	DeleteTy: CGF.getContext().getTagDeclType(ClassDecl));
1597	assert(Dtor->getOperatorDelete()->isDestroyingOperatorDelete() ==
1598	ReturnAfterDelete &&
1599	"unexpected value for ReturnAfterDelete");
1600	if (ReturnAfterDelete)
1601	CGF.EmitBranchThroughCleanup(Dest: CGF.ReturnBlock);
1602	else
1603	CGF.Builder.CreateBr(Dest: continueBB);
1604
1605	CGF.EmitBlock(BB: continueBB);
1606	}
1607
1608	struct CallDtorDeleteConditional final : EHScopeStack::Cleanup {
1609	llvm::Value *ShouldDeleteCondition;
1610
1611	public:
1612	CallDtorDeleteConditional(llvm::Value *ShouldDeleteCondition)
1613	: ShouldDeleteCondition(ShouldDeleteCondition) {
1614	assert(ShouldDeleteCondition != nullptr);
1615	}
1616
1617	void Emit(CodeGenFunction &CGF, Flags flags) override {
1618	EmitConditionalDtorDeleteCall(CGF, ShouldDeleteCondition,
1619	/*ReturnAfterDelete*/false);
1620	}
1621	};
1622
1623	class DestroyField final : public EHScopeStack::Cleanup {
1624	const FieldDecl *field;
1625	CodeGenFunction::Destroyer *destroyer;
1626	bool useEHCleanupForArray;
1627
1628	public:
1629	DestroyField(const FieldDecl *field, CodeGenFunction::Destroyer *destroyer,
1630	bool useEHCleanupForArray)
1631	: field(field), destroyer(destroyer),
1632	useEHCleanupForArray(useEHCleanupForArray) {}
1633
1634	void Emit(CodeGenFunction &CGF, Flags flags) override {
1635	// Find the address of the field.
1636	Address thisValue = CGF.LoadCXXThisAddress();
1637	QualType RecordTy = CGF.getContext().getTagDeclType(field->getParent());
1638	LValue ThisLV = CGF.MakeAddrLValue(Addr: thisValue, T: RecordTy);
1639	LValue LV = CGF.EmitLValueForField(Base: ThisLV, Field: field);
1640	assert(LV.isSimple());
1641
1642	CGF.emitDestroy(addr: LV.getAddress(CGF), type: field->getType(), destroyer,
1643	useEHCleanupForArray: flags.isForNormalCleanup() && useEHCleanupForArray);
1644	}
1645	};
1646
1647	class DeclAsInlineDebugLocation {
1648	CGDebugInfo *DI;
1649	llvm::MDNode *InlinedAt;
1650	std::optional<ApplyDebugLocation> Location;
1651
1652	public:
1653	DeclAsInlineDebugLocation(CodeGenFunction &CGF, const NamedDecl &Decl)
1654	: DI(CGF.getDebugInfo()) {
1655	if (!DI)
1656	return;
1657	InlinedAt = DI->getInlinedAt();
1658	DI->setInlinedAt(CGF.Builder.getCurrentDebugLocation());
1659	Location.emplace(CGF, Decl.getLocation());
1660	}
1661
1662	~DeclAsInlineDebugLocation() {
1663	if (!DI)
1664	return;
1665	Location.reset();
1666	DI->setInlinedAt(InlinedAt);
1667	}
1668	};
1669
1670	static void EmitSanitizerDtorCallback(
1671	CodeGenFunction &CGF, StringRef Name, llvm::Value *Ptr,
1672	std::optional<CharUnits::QuantityType> PoisonSize = {}) {
1673	CodeGenFunction::SanitizerScope SanScope(&CGF);
1674	// Pass in void pointer and size of region as arguments to runtime
1675	// function
1676	SmallVector<llvm::Value *, 2> Args = {Ptr};
1677	SmallVector<llvm::Type *, 2> ArgTypes = {CGF.VoidPtrTy};
1678
1679	if (PoisonSize.has_value()) {
1680	Args.emplace_back(Args: llvm::ConstantInt::get(Ty: CGF.SizeTy, V: *PoisonSize));
1681	ArgTypes.emplace_back(Args&: CGF.SizeTy);
1682	}
1683
1684	llvm::FunctionType *FnType =
1685	llvm::FunctionType::get(Result: CGF.VoidTy, Params: ArgTypes, isVarArg: false);
1686	llvm::FunctionCallee Fn = CGF.CGM.CreateRuntimeFunction(Ty: FnType, Name);
1687
1688	CGF.EmitNounwindRuntimeCall(callee: Fn, args: Args);
1689	}
1690
1691	static void
1692	EmitSanitizerDtorFieldsCallback(CodeGenFunction &CGF, llvm::Value *Ptr,
1693	CharUnits::QuantityType PoisonSize) {
1694	EmitSanitizerDtorCallback(CGF, Name: "__sanitizer_dtor_callback_fields", Ptr,
1695	PoisonSize);
1696	}
1697
1698	/// Poison base class with a trivial destructor.
1699	struct SanitizeDtorTrivialBase final : EHScopeStack::Cleanup {
1700	const CXXRecordDecl *BaseClass;
1701	bool BaseIsVirtual;
1702	SanitizeDtorTrivialBase(const CXXRecordDecl *Base, bool BaseIsVirtual)
1703	: BaseClass(Base), BaseIsVirtual(BaseIsVirtual) {}
1704
1705	void Emit(CodeGenFunction &CGF, Flags flags) override {
1706	const CXXRecordDecl *DerivedClass =
1707	cast<CXXMethodDecl>(Val: CGF.CurCodeDecl)->getParent();
1708
1709	Address Addr = CGF.GetAddressOfDirectBaseInCompleteClass(
1710	This: CGF.LoadCXXThisAddress(), Derived: DerivedClass, Base: BaseClass, BaseIsVirtual);
1711
1712	const ASTRecordLayout &BaseLayout =
1713	CGF.getContext().getASTRecordLayout(BaseClass);
1714	CharUnits BaseSize = BaseLayout.getSize();
1715
1716	if (!BaseSize.isPositive())
1717	return;
1718
1719	// Use the base class declaration location as inline DebugLocation. All
1720	// fields of the class are destroyed.
1721	DeclAsInlineDebugLocation InlineHere(CGF, *BaseClass);
1722	EmitSanitizerDtorFieldsCallback(CGF, Ptr: Addr.getPointer(),
1723	PoisonSize: BaseSize.getQuantity());
1724
1725	// Prevent the current stack frame from disappearing from the stack trace.
1726	CGF.CurFn->addFnAttr(Kind: "disable-tail-calls", Val: "true");
1727	}
1728	};
1729
1730	class SanitizeDtorFieldRange final : public EHScopeStack::Cleanup {
1731	const CXXDestructorDecl *Dtor;
1732	unsigned StartIndex;
1733	unsigned EndIndex;
1734
1735	public:
1736	SanitizeDtorFieldRange(const CXXDestructorDecl *Dtor, unsigned StartIndex,
1737	unsigned EndIndex)
1738	: Dtor(Dtor), StartIndex(StartIndex), EndIndex(EndIndex) {}
1739
1740	// Generate function call for handling object poisoning.
1741	// Disables tail call elimination, to prevent the current stack frame
1742	// from disappearing from the stack trace.
1743	void Emit(CodeGenFunction &CGF, Flags flags) override {
1744	const ASTContext &Context = CGF.getContext();
1745	const ASTRecordLayout &Layout =
1746	Context.getASTRecordLayout(D: Dtor->getParent());
1747
1748	// It's a first trivial field so it should be at the begining of a char,
1749	// still round up start offset just in case.
1750	CharUnits PoisonStart = Context.toCharUnitsFromBits(
1751	BitSize: Layout.getFieldOffset(FieldNo: StartIndex) + Context.getCharWidth() - 1);
1752	llvm::ConstantInt *OffsetSizePtr =
1753	llvm::ConstantInt::get(Ty: CGF.SizeTy, V: PoisonStart.getQuantity());
1754
1755	llvm::Value *OffsetPtr =
1756	CGF.Builder.CreateGEP(Ty: CGF.Int8Ty, Ptr: CGF.LoadCXXThis(), IdxList: OffsetSizePtr);
1757
1758	CharUnits PoisonEnd;
1759	if (EndIndex >= Layout.getFieldCount()) {
1760	PoisonEnd = Layout.getNonVirtualSize();
1761	} else {
1762	PoisonEnd =
1763	Context.toCharUnitsFromBits(BitSize: Layout.getFieldOffset(FieldNo: EndIndex));
1764	}
1765	CharUnits PoisonSize = PoisonEnd - PoisonStart;
1766	if (!PoisonSize.isPositive())
1767	return;
1768
1769	// Use the top field declaration location as inline DebugLocation.
1770	DeclAsInlineDebugLocation InlineHere(
1771	CGF, **std::next(Dtor->getParent()->field_begin(), StartIndex));
1772	EmitSanitizerDtorFieldsCallback(CGF, Ptr: OffsetPtr, PoisonSize: PoisonSize.getQuantity());
1773
1774	// Prevent the current stack frame from disappearing from the stack trace.
1775	CGF.CurFn->addFnAttr(Kind: "disable-tail-calls", Val: "true");
1776	}
1777	};
1778
1779	class SanitizeDtorVTable final : public EHScopeStack::Cleanup {
1780	const CXXDestructorDecl *Dtor;
1781
1782	public:
1783	SanitizeDtorVTable(const CXXDestructorDecl *Dtor) : Dtor(Dtor) {}
1784
1785	// Generate function call for handling vtable pointer poisoning.
1786	void Emit(CodeGenFunction &CGF, Flags flags) override {
1787	assert(Dtor->getParent()->isDynamicClass());
1788	(void)Dtor;
1789	// Poison vtable and vtable ptr if they exist for this class.
1790	llvm::Value *VTablePtr = CGF.LoadCXXThis();
1791
1792	// Pass in void pointer and size of region as arguments to runtime
1793	// function
1794	EmitSanitizerDtorCallback(CGF, Name: "__sanitizer_dtor_callback_vptr",
1795	Ptr: VTablePtr);
1796	}
1797	};
1798
1799	class SanitizeDtorCleanupBuilder {
1800	ASTContext &Context;
1801	EHScopeStack &EHStack;
1802	const CXXDestructorDecl *DD;
1803	std::optional<unsigned> StartIndex;
1804
1805	public:
1806	SanitizeDtorCleanupBuilder(ASTContext &Context, EHScopeStack &EHStack,
1807	const CXXDestructorDecl *DD)
1808	: Context(Context), EHStack(EHStack), DD(DD), StartIndex(std::nullopt) {}
1809	void PushCleanupForField(const FieldDecl *Field) {
1810	if (Field->isZeroSize(Ctx: Context))
1811	return;
1812	unsigned FieldIndex = Field->getFieldIndex();
1813	if (FieldHasTrivialDestructorBody(Context, Field)) {
1814	if (!StartIndex)
1815	StartIndex = FieldIndex;
1816	} else if (StartIndex) {
1817	EHStack.pushCleanup<SanitizeDtorFieldRange>(Kind: NormalAndEHCleanup, A: DD,
1818	A: *StartIndex, A: FieldIndex);
1819	StartIndex = std::nullopt;
1820	}
1821	}
1822	void End() {
1823	if (StartIndex)
1824	EHStack.pushCleanup<SanitizeDtorFieldRange>(Kind: NormalAndEHCleanup, A: DD,
1825	A: *StartIndex, A: -1);
1826	}
1827	};
1828	} // end anonymous namespace
1829
1830	/// Emit all code that comes at the end of class's
1831	/// destructor. This is to call destructors on members and base classes
1832	/// in reverse order of their construction.
1833	///
1834	/// For a deleting destructor, this also handles the case where a destroying
1835	/// operator delete completely overrides the definition.
1836	void CodeGenFunction::EnterDtorCleanups(const CXXDestructorDecl *DD,
1837	CXXDtorType DtorType) {
1838	assert((!DD->isTrivial() || DD->hasAttr<DLLExportAttr>()) &&
1839	"Should not emit dtor epilogue for non-exported trivial dtor!");
1840
1841	// The deleting-destructor phase just needs to call the appropriate
1842	// operator delete that Sema picked up.
1843	if (DtorType == Dtor_Deleting) {
1844	assert(DD->getOperatorDelete() &&
1845	"operator delete missing - EnterDtorCleanups");
1846	if (CXXStructorImplicitParamValue) {
1847	// If there is an implicit param to the deleting dtor, it's a boolean
1848	// telling whether this is a deleting destructor.
1849	if (DD->getOperatorDelete()->isDestroyingOperatorDelete())
1850	EmitConditionalDtorDeleteCall(CGF&: *this, ShouldDeleteCondition: CXXStructorImplicitParamValue,
1851	/*ReturnAfterDelete*/true);
1852	else
1853	EHStack.pushCleanup<CallDtorDeleteConditional>(
1854	Kind: NormalAndEHCleanup, A: CXXStructorImplicitParamValue);
1855	} else {
1856	if (DD->getOperatorDelete()->isDestroyingOperatorDelete()) {
1857	const CXXRecordDecl *ClassDecl = DD->getParent();
1858	EmitDeleteCall(DeleteFD: DD->getOperatorDelete(),
1859	Ptr: LoadThisForDtorDelete(CGF&: *this, DD),
1860	DeleteTy: getContext().getTagDeclType(ClassDecl));
1861	EmitBranchThroughCleanup(Dest: ReturnBlock);
1862	} else {
1863	EHStack.pushCleanup<CallDtorDelete>(Kind: NormalAndEHCleanup);
1864	}
1865	}
1866	return;
1867	}
1868
1869	const CXXRecordDecl *ClassDecl = DD->getParent();
1870
1871	// Unions have no bases and do not call field destructors.
1872	if (ClassDecl->isUnion())
1873	return;
1874
1875	// The complete-destructor phase just destructs all the virtual bases.
1876	if (DtorType == Dtor_Complete) {
1877	// Poison the vtable pointer such that access after the base
1878	// and member destructors are invoked is invalid.
1879	if (CGM.getCodeGenOpts().SanitizeMemoryUseAfterDtor &&
1880	SanOpts.has(K: SanitizerKind::Memory) && ClassDecl->getNumVBases() &&
1881	ClassDecl->isPolymorphic())
1882	EHStack.pushCleanup<SanitizeDtorVTable>(Kind: NormalAndEHCleanup, A: DD);
1883
1884	// We push them in the forward order so that they'll be popped in
1885	// the reverse order.
1886	for (const auto &Base : ClassDecl->vbases()) {
1887	auto *BaseClassDecl =
1888	cast<CXXRecordDecl>(Base.getType()->castAs<RecordType>()->getDecl());
1889
1890	if (BaseClassDecl->hasTrivialDestructor()) {
1891	// Under SanitizeMemoryUseAfterDtor, poison the trivial base class
1892	// memory. For non-trival base classes the same is done in the class
1893	// destructor.
1894	if (CGM.getCodeGenOpts().SanitizeMemoryUseAfterDtor &&
1895	SanOpts.has(SanitizerKind::Memory) && !BaseClassDecl->isEmpty())
1896	EHStack.pushCleanup<SanitizeDtorTrivialBase>(NormalAndEHCleanup,
1897	BaseClassDecl,
1898	/*BaseIsVirtual*/ true);
1899	} else {
1900	EHStack.pushCleanup<CallBaseDtor>(NormalAndEHCleanup, BaseClassDecl,
1901	/*BaseIsVirtual*/ true);
1902	}
1903	}
1904
1905	return;
1906	}
1907
1908	assert(DtorType == Dtor_Base);
1909	// Poison the vtable pointer if it has no virtual bases, but inherits
1910	// virtual functions.
1911	if (CGM.getCodeGenOpts().SanitizeMemoryUseAfterDtor &&
1912	SanOpts.has(K: SanitizerKind::Memory) && !ClassDecl->getNumVBases() &&
1913	ClassDecl->isPolymorphic())
1914	EHStack.pushCleanup<SanitizeDtorVTable>(Kind: NormalAndEHCleanup, A: DD);
1915
1916	// Destroy non-virtual bases.
1917	for (const auto &Base : ClassDecl->bases()) {
1918	// Ignore virtual bases.
1919	if (Base.isVirtual())
1920	continue;
1921
1922	CXXRecordDecl *BaseClassDecl = Base.getType()->getAsCXXRecordDecl();
1923
1924	if (BaseClassDecl->hasTrivialDestructor()) {
1925	if (CGM.getCodeGenOpts().SanitizeMemoryUseAfterDtor &&
1926	SanOpts.has(SanitizerKind::Memory) && !BaseClassDecl->isEmpty())
1927	EHStack.pushCleanup<SanitizeDtorTrivialBase>(NormalAndEHCleanup,
1928	BaseClassDecl,
1929	/*BaseIsVirtual*/ false);
1930	} else {
1931	EHStack.pushCleanup<CallBaseDtor>(NormalAndEHCleanup, BaseClassDecl,
1932	/*BaseIsVirtual*/ false);
1933	}
1934	}
1935
1936	// Poison fields such that access after their destructors are
1937	// invoked, and before the base class destructor runs, is invalid.
1938	bool SanitizeFields = CGM.getCodeGenOpts().SanitizeMemoryUseAfterDtor &&
1939	SanOpts.has(K: SanitizerKind::Memory);
1940	SanitizeDtorCleanupBuilder SanitizeBuilder(getContext(), EHStack, DD);
1941
1942	// Destroy direct fields.
1943	for (const auto *Field : ClassDecl->fields()) {
1944	if (SanitizeFields)
1945	SanitizeBuilder.PushCleanupForField(Field);
1946
1947	QualType type = Field->getType();
1948	QualType::DestructionKind dtorKind = type.isDestructedType();
1949	if (!dtorKind)
1950	continue;
1951
1952	// Anonymous union members do not have their destructors called.
1953	const RecordType *RT = type->getAsUnionType();
1954	if (RT && RT->getDecl()->isAnonymousStructOrUnion())
1955	continue;
1956
1957	CleanupKind cleanupKind = getCleanupKind(dtorKind);
1958	EHStack.pushCleanup<DestroyField>(
1959	cleanupKind, Field, getDestroyer(dtorKind), cleanupKind & EHCleanup);
1960	}
1961
1962	if (SanitizeFields)
1963	SanitizeBuilder.End();
1964	}
1965
1966	/// EmitCXXAggrConstructorCall - Emit a loop to call a particular
1967	/// constructor for each of several members of an array.
1968	///
1969	/// \param ctor the constructor to call for each element
1970	/// \param arrayType the type of the array to initialize
1971	/// \param arrayBegin an arrayType*
1972	/// \param zeroInitialize true if each element should be
1973	/// zero-initialized before it is constructed
1974	void CodeGenFunction::EmitCXXAggrConstructorCall(
1975	const CXXConstructorDecl *ctor, const ArrayType *arrayType,
1976	Address arrayBegin, const CXXConstructExpr *E, bool NewPointerIsChecked,
1977	bool zeroInitialize) {
1978	QualType elementType;
1979	llvm::Value *numElements =
1980	emitArrayLength(arrayType, baseType&: elementType, addr&: arrayBegin);
1981
1982	EmitCXXAggrConstructorCall(D: ctor, NumElements: numElements, ArrayPtr: arrayBegin, E,
1983	NewPointerIsChecked, ZeroInitialization: zeroInitialize);
1984	}
1985
1986	/// EmitCXXAggrConstructorCall - Emit a loop to call a particular
1987	/// constructor for each of several members of an array.
1988	///
1989	/// \param ctor the constructor to call for each element
1990	/// \param numElements the number of elements in the array;
1991	/// may be zero
1992	/// \param arrayBase a T*, where T is the type constructed by ctor
1993	/// \param zeroInitialize true if each element should be
1994	/// zero-initialized before it is constructed
1995	void CodeGenFunction::EmitCXXAggrConstructorCall(const CXXConstructorDecl *ctor,
1996	llvm::Value *numElements,
1997	Address arrayBase,
1998	const CXXConstructExpr *E,
1999	bool NewPointerIsChecked,
2000	bool zeroInitialize) {
2001	// It's legal for numElements to be zero. This can happen both
2002	// dynamically, because x can be zero in 'new A[x]', and statically,
2003	// because of GCC extensions that permit zero-length arrays. There
2004	// are probably legitimate places where we could assume that this
2005	// doesn't happen, but it's not clear that it's worth it.
2006	llvm::BranchInst *zeroCheckBranch = nullptr;
2007
2008	// Optimize for a constant count.
2009	llvm::ConstantInt *constantCount
2010	= dyn_cast<llvm::ConstantInt>(Val: numElements);
2011	if (constantCount) {
2012	// Just skip out if the constant count is zero.
2013	if (constantCount->isZero()) return;
2014
2015	// Otherwise, emit the check.
2016	} else {
2017	llvm::BasicBlock *loopBB = createBasicBlock(name: "new.ctorloop");
2018	llvm::Value *iszero = Builder.CreateIsNull(Arg: numElements, Name: "isempty");
2019	zeroCheckBranch = Builder.CreateCondBr(Cond: iszero, True: loopBB, False: loopBB);
2020	EmitBlock(BB: loopBB);
2021	}
2022
2023	// Find the end of the array.
2024	llvm::Type *elementType = arrayBase.getElementType();
2025	llvm::Value *arrayBegin = arrayBase.getPointer();
2026	llvm::Value *arrayEnd = Builder.CreateInBoundsGEP(
2027	Ty: elementType, Ptr: arrayBegin, IdxList: numElements, Name: "arrayctor.end");
2028
2029	// Enter the loop, setting up a phi for the current location to initialize.
2030	llvm::BasicBlock *entryBB = Builder.GetInsertBlock();
2031	llvm::BasicBlock *loopBB = createBasicBlock(name: "arrayctor.loop");
2032	EmitBlock(BB: loopBB);
2033	llvm::PHINode *cur = Builder.CreatePHI(Ty: arrayBegin->getType(), NumReservedValues: 2,
2034	Name: "arrayctor.cur");
2035	cur->addIncoming(V: arrayBegin, BB: entryBB);
2036
2037	// Inside the loop body, emit the constructor call on the array element.
2038
2039	// The alignment of the base, adjusted by the size of a single element,
2040	// provides a conservative estimate of the alignment of every element.
2041	// (This assumes we never start tracking offsetted alignments.)
2042	//
2043	// Note that these are complete objects and so we don't need to
2044	// use the non-virtual size or alignment.
2045	QualType type = getContext().getTypeDeclType(Decl: ctor->getParent());
2046	CharUnits eltAlignment =
2047	arrayBase.getAlignment()
2048	.alignmentOfArrayElement(elementSize: getContext().getTypeSizeInChars(T: type));
2049	Address curAddr = Address(cur, elementType, eltAlignment);
2050
2051	// Zero initialize the storage, if requested.
2052	if (zeroInitialize)
2053	EmitNullInitialization(DestPtr: curAddr, Ty: type);
2054
2055	// C++ [class.temporary]p4:
2056	// There are two contexts in which temporaries are destroyed at a different
2057	// point than the end of the full-expression. The first context is when a
2058	// default constructor is called to initialize an element of an array.
2059	// If the constructor has one or more default arguments, the destruction of
2060	// every temporary created in a default argument expression is sequenced
2061	// before the construction of the next array element, if any.
2062
2063	{
2064	RunCleanupsScope Scope(*this);
2065
2066	// Evaluate the constructor and its arguments in a regular
2067	// partial-destroy cleanup.
2068	if (getLangOpts().Exceptions &&
2069	!ctor->getParent()->hasTrivialDestructor()) {
2070	Destroyer *destroyer = destroyCXXObject;
2071	pushRegularPartialArrayCleanup(arrayBegin, arrayEnd: cur, elementType: type, elementAlignment: eltAlignment,
2072	destroyer: *destroyer);
2073	}
2074	auto currAVS = AggValueSlot::forAddr(
2075	addr: curAddr, quals: type.getQualifiers(), isDestructed: AggValueSlot::IsDestructed,
2076	needsGC: AggValueSlot::DoesNotNeedGCBarriers, isAliased: AggValueSlot::IsNotAliased,
2077	mayOverlap: AggValueSlot::DoesNotOverlap, isZeroed: AggValueSlot::IsNotZeroed,
2078	isChecked: NewPointerIsChecked ? AggValueSlot::IsSanitizerChecked
2079	: AggValueSlot::IsNotSanitizerChecked);
2080	EmitCXXConstructorCall(ctor, Ctor_Complete, /*ForVirtualBase=*/false,
2081	/*Delegating=*/false, currAVS, E);
2082	}
2083
2084	// Go to the next element.
2085	llvm::Value *next = Builder.CreateInBoundsGEP(
2086	Ty: elementType, Ptr: cur, IdxList: llvm::ConstantInt::get(Ty: SizeTy, V: 1), Name: "arrayctor.next");
2087	cur->addIncoming(V: next, BB: Builder.GetInsertBlock());
2088
2089	// Check whether that's the end of the loop.
2090	llvm::Value *done = Builder.CreateICmpEQ(LHS: next, RHS: arrayEnd, Name: "arrayctor.done");
2091	llvm::BasicBlock *contBB = createBasicBlock(name: "arrayctor.cont");
2092	Builder.CreateCondBr(Cond: done, True: contBB, False: loopBB);
2093
2094	// Patch the earlier check to skip over the loop.
2095	if (zeroCheckBranch) zeroCheckBranch->setSuccessor(idx: 0, NewSucc: contBB);
2096
2097	EmitBlock(BB: contBB);
2098	}
2099
2100	void CodeGenFunction::destroyCXXObject(CodeGenFunction &CGF,
2101	Address addr,
2102	QualType type) {
2103	const RecordType *rtype = type->castAs<RecordType>();
2104	const CXXRecordDecl *record = cast<CXXRecordDecl>(Val: rtype->getDecl());
2105	const CXXDestructorDecl *dtor = record->getDestructor();
2106	assert(!dtor->isTrivial());
2107	CGF.EmitCXXDestructorCall(D: dtor, Type: Dtor_Complete, /*for vbase*/ ForVirtualBase: false,
2108	/*Delegating=*/false, This: addr, ThisTy: type);
2109	}
2110
2111	void CodeGenFunction::EmitCXXConstructorCall(const CXXConstructorDecl *D,
2112	CXXCtorType Type,
2113	bool ForVirtualBase,
2114	bool Delegating,
2115	AggValueSlot ThisAVS,
2116	const CXXConstructExpr *E) {
2117	CallArgList Args;
2118	Address This = ThisAVS.getAddress();
2119	LangAS SlotAS = ThisAVS.getQualifiers().getAddressSpace();
2120	LangAS ThisAS = D->getFunctionObjectParameterType().getAddressSpace();
2121	llvm::Value *ThisPtr = This.getPointer();
2122
2123	if (SlotAS != ThisAS) {
2124	unsigned TargetThisAS = getContext().getTargetAddressSpace(AS: ThisAS);
2125	llvm::Type *NewType =
2126	llvm::PointerType::get(C&: getLLVMContext(), AddressSpace: TargetThisAS);
2127	ThisPtr = getTargetHooks().performAddrSpaceCast(CGF&: *this, V: This.getPointer(),
2128	SrcAddr: ThisAS, DestAddr: SlotAS, DestTy: NewType);
2129	}
2130
2131	// Push the this ptr.
2132	Args.add(rvalue: RValue::get(V: ThisPtr), type: D->getThisType());
2133
2134	// If this is a trivial constructor, emit a memcpy now before we lose
2135	// the alignment information on the argument.
2136	// FIXME: It would be better to preserve alignment information into CallArg.
2137	if (isMemcpyEquivalentSpecialMember(D)) {
2138	assert(E->getNumArgs() == 1 && "unexpected argcount for trivial ctor");
2139
2140	const Expr *Arg = E->getArg(Arg: 0);
2141	LValue Src = EmitLValue(E: Arg);
2142	QualType DestTy = getContext().getTypeDeclType(Decl: D->getParent());
2143	LValue Dest = MakeAddrLValue(Addr: This, T: DestTy);
2144	EmitAggregateCopyCtor(Dest, Src, MayOverlap: ThisAVS.mayOverlap());
2145	return;
2146	}
2147
2148	// Add the rest of the user-supplied arguments.
2149	const FunctionProtoType *FPT = D->getType()->castAs<FunctionProtoType>();
2150	EvaluationOrder Order = E->isListInitialization()
2151	? EvaluationOrder::ForceLeftToRight
2152	: EvaluationOrder::Default;
2153	EmitCallArgs(Args, Prototype: FPT, ArgRange: E->arguments(), AC: E->getConstructor(),
2154	/*ParamsToSkip*/ 0, Order);
2155
2156	EmitCXXConstructorCall(D, Type, ForVirtualBase, Delegating, This, Args,
2157	ThisAVS.mayOverlap(), E->getExprLoc(),
2158	ThisAVS.isSanitizerChecked());
2159	}
2160
2161	static bool canEmitDelegateCallArgs(CodeGenFunction &CGF,
2162	const CXXConstructorDecl *Ctor,
2163	CXXCtorType Type, CallArgList &Args) {
2164	// We can't forward a variadic call.
2165	if (Ctor->isVariadic())
2166	return false;
2167
2168	if (CGF.getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee()) {
2169	// If the parameters are callee-cleanup, it's not safe to forward.
2170	for (auto *P : Ctor->parameters())
2171	if (P->needsDestruction(CGF.getContext()))
2172	return false;
2173
2174	// Likewise if they're inalloca.
2175	const CGFunctionInfo &Info =
2176	CGF.CGM.getTypes().arrangeCXXConstructorCall(Args, D: Ctor, CtorKind: Type, ExtraPrefixArgs: 0, ExtraSuffixArgs: 0);
2177	if (Info.usesInAlloca())
2178	return false;
2179	}
2180
2181	// Anything else should be OK.
2182	return true;
2183	}
2184
2185	void CodeGenFunction::EmitCXXConstructorCall(const CXXConstructorDecl *D,
2186	CXXCtorType Type,
2187	bool ForVirtualBase,
2188	bool Delegating,
2189	Address This,
2190	CallArgList &Args,
2191	AggValueSlot::Overlap_t Overlap,
2192	SourceLocation Loc,
2193	bool NewPointerIsChecked) {
2194	const CXXRecordDecl *ClassDecl = D->getParent();
2195
2196	if (!NewPointerIsChecked)
2197	EmitTypeCheck(TCK: CodeGenFunction::TCK_ConstructorCall, Loc, V: This.getPointer(),
2198	Type: getContext().getRecordType(ClassDecl), Alignment: CharUnits::Zero());
2199
2200	if (D->isTrivial() && D->isDefaultConstructor()) {
2201	assert(Args.size() == 1 && "trivial default ctor with args");
2202	return;
2203	}
2204
2205	// If this is a trivial constructor, just emit what's needed. If this is a
2206	// union copy constructor, we must emit a memcpy, because the AST does not
2207	// model that copy.
2208	if (isMemcpyEquivalentSpecialMember(D)) {
2209	assert(Args.size() == 2 && "unexpected argcount for trivial ctor");
2210
2211	QualType SrcTy = D->getParamDecl(0)->getType().getNonReferenceType();
2212	Address Src = Address(Args[1].getRValue(CGF&: *this).getScalarVal(), ConvertTypeForMem(T: SrcTy),
2213	CGM.getNaturalTypeAlignment(T: SrcTy));
2214	LValue SrcLVal = MakeAddrLValue(Addr: Src, T: SrcTy);
2215	QualType DestTy = getContext().getTypeDeclType(ClassDecl);
2216	LValue DestLVal = MakeAddrLValue(Addr: This, T: DestTy);
2217	EmitAggregateCopyCtor(Dest: DestLVal, Src: SrcLVal, MayOverlap: Overlap);
2218	return;
2219	}
2220
2221	bool PassPrototypeArgs = true;
2222	// Check whether we can actually emit the constructor before trying to do so.
2223	if (auto Inherited = D->getInheritedConstructor()) {
2224	PassPrototypeArgs = getTypes().inheritingCtorHasParams(Inherited, Type);
2225	if (PassPrototypeArgs && !canEmitDelegateCallArgs(CGF&: *this, Ctor: D, Type, Args)) {
2226	EmitInlinedInheritingCXXConstructorCall(Ctor: D, CtorType: Type, ForVirtualBase,
2227	Delegating, Args);
2228	return;
2229	}
2230	}
2231
2232	// Insert any ABI-specific implicit constructor arguments.
2233	CGCXXABI::AddedStructorArgCounts ExtraArgs =
2234	CGM.getCXXABI().addImplicitConstructorArgs(CGF&: *this, D, Type, ForVirtualBase,
2235	Delegating, Args);
2236
2237	// Emit the call.
2238	llvm::Constant *CalleePtr = CGM.getAddrOfCXXStructor(GD: GlobalDecl(D, Type));
2239	const CGFunctionInfo &Info = CGM.getTypes().arrangeCXXConstructorCall(
2240	Args, D, CtorKind: Type, ExtraPrefixArgs: ExtraArgs.Prefix, ExtraSuffixArgs: ExtraArgs.Suffix, PassProtoArgs: PassPrototypeArgs);
2241	CGCallee Callee = CGCallee::forDirect(functionPtr: CalleePtr, abstractInfo: GlobalDecl(D, Type));
2242	EmitCall(CallInfo: Info, Callee, ReturnValue: ReturnValueSlot(), Args, callOrInvoke: nullptr, IsMustTail: false, Loc);
2243
2244	// Generate vtable assumptions if we're constructing a complete object
2245	// with a vtable. We don't do this for base subobjects for two reasons:
2246	// first, it's incorrect for classes with virtual bases, and second, we're
2247	// about to overwrite the vptrs anyway.
2248	// We also have to make sure if we can refer to vtable:
2249	// - Otherwise we can refer to vtable if it's safe to speculatively emit.
2250	// FIXME: If vtable is used by ctor/dtor, or if vtable is external and we are
2251	// sure that definition of vtable is not hidden,
2252	// then we are always safe to refer to it.
2253	// FIXME: It looks like InstCombine is very inefficient on dealing with
2254	// assumes. Make assumption loads require -fstrict-vtable-pointers temporarily.
2255	if (CGM.getCodeGenOpts().OptimizationLevel > 0 &&
2256	ClassDecl->isDynamicClass() && Type != Ctor_Base &&
2257	CGM.getCXXABI().canSpeculativelyEmitVTable(RD: ClassDecl) &&
2258	CGM.getCodeGenOpts().StrictVTablePointers)
2259	EmitVTableAssumptionLoads(ClassDecl, This);
2260	}
2261
2262	void CodeGenFunction::EmitInheritedCXXConstructorCall(
2263	const CXXConstructorDecl *D, bool ForVirtualBase, Address This,
2264	bool InheritedFromVBase, const CXXInheritedCtorInitExpr *E) {
2265	CallArgList Args;
2266	CallArg ThisArg(RValue::get(V: This.getPointer()), D->getThisType());
2267
2268	// Forward the parameters.
2269	if (InheritedFromVBase &&
2270	CGM.getTarget().getCXXABI().hasConstructorVariants()) {
2271	// Nothing to do; this construction is not responsible for constructing
2272	// the base class containing the inherited constructor.
2273	// FIXME: Can we just pass undef's for the remaining arguments if we don't
2274	// have constructor variants?
2275	Args.push_back(Elt: ThisArg);
2276	} else if (!CXXInheritedCtorInitExprArgs.empty()) {
2277	// The inheriting constructor was inlined; just inject its arguments.
2278	assert(CXXInheritedCtorInitExprArgs.size() >= D->getNumParams() &&
2279	"wrong number of parameters for inherited constructor call");
2280	Args = CXXInheritedCtorInitExprArgs;
2281	Args[0] = ThisArg;
2282	} else {
2283	// The inheriting constructor was not inlined. Emit delegating arguments.
2284	Args.push_back(Elt: ThisArg);
2285	const auto *OuterCtor = cast<CXXConstructorDecl>(Val: CurCodeDecl);
2286	assert(OuterCtor->getNumParams() == D->getNumParams());
2287	assert(!OuterCtor->isVariadic() && "should have been inlined");
2288
2289	for (const auto *Param : OuterCtor->parameters()) {
2290	assert(getContext().hasSameUnqualifiedType(
2291	OuterCtor->getParamDecl(Param->getFunctionScopeIndex())->getType(),
2292	Param->getType()));
2293	EmitDelegateCallArg(Args, Param, E->getLocation());
2294
2295	// Forward __attribute__(pass_object_size).
2296	if (Param->hasAttr<PassObjectSizeAttr>()) {
2297	auto *POSParam = SizeArguments[Param];
2298	assert(POSParam && "missing pass_object_size value for forwarding");
2299	EmitDelegateCallArg(Args, POSParam, E->getLocation());
2300	}
2301	}
2302	}
2303
2304	EmitCXXConstructorCall(D, Type: Ctor_Base, ForVirtualBase, /*Delegating*/false,
2305	This, Args, Overlap: AggValueSlot::MayOverlap,
2306	Loc: E->getLocation(), /*NewPointerIsChecked*/true);
2307	}
2308
2309	void CodeGenFunction::EmitInlinedInheritingCXXConstructorCall(
2310	const CXXConstructorDecl *Ctor, CXXCtorType CtorType, bool ForVirtualBase,
2311	bool Delegating, CallArgList &Args) {
2312	GlobalDecl GD(Ctor, CtorType);
2313	InlinedInheritingConstructorScope Scope(*this, GD);
2314	ApplyInlineDebugLocation DebugScope(*this, GD);
2315	RunCleanupsScope RunCleanups(*this);
2316
2317	// Save the arguments to be passed to the inherited constructor.
2318	CXXInheritedCtorInitExprArgs = Args;
2319
2320	FunctionArgList Params;
2321	QualType RetType = BuildFunctionArgList(GD: CurGD, Args&: Params);
2322	FnRetTy = RetType;
2323
2324	// Insert any ABI-specific implicit constructor arguments.
2325	CGM.getCXXABI().addImplicitConstructorArgs(CGF&: *this, D: Ctor, Type: CtorType,
2326	ForVirtualBase, Delegating, Args);
2327
2328	// Emit a simplified prolog. We only need to emit the implicit params.
2329	assert(Args.size() >= Params.size() && "too few arguments for call");
2330	for (unsigned I = 0, N = Args.size(); I != N; ++I) {
2331	if (I < Params.size() && isa<ImplicitParamDecl>(Val: Params[I])) {
2332	const RValue &RV = Args[I].getRValue(CGF&: *this);
2333	assert(!RV.isComplex() && "complex indirect params not supported");
2334	ParamValue Val = RV.isScalar()
2335	? ParamValue::forDirect(value: RV.getScalarVal())
2336	: ParamValue::forIndirect(addr: RV.getAggregateAddress());
2337	EmitParmDecl(D: *Params[I], Arg: Val, ArgNo: I + 1);
2338	}
2339	}
2340
2341	// Create a return value slot if the ABI implementation wants one.
2342	// FIXME: This is dumb, we should ask the ABI not to try to set the return
2343	// value instead.
2344	if (!RetType->isVoidType())
2345	ReturnValue = CreateIRTemp(T: RetType, Name: "retval.inhctor");
2346
2347	CGM.getCXXABI().EmitInstanceFunctionProlog(CGF&: *this);
2348	CXXThisValue = CXXABIThisValue;
2349
2350	// Directly emit the constructor initializers.
2351	EmitCtorPrologue(CD: Ctor, CtorType, Args&: Params);
2352	}
2353
2354	void CodeGenFunction::EmitVTableAssumptionLoad(const VPtr &Vptr, Address This) {
2355	llvm::Value *VTableGlobal =
2356	CGM.getCXXABI().getVTableAddressPoint(Base: Vptr.Base, VTableClass: Vptr.VTableClass);
2357	if (!VTableGlobal)
2358	return;
2359
2360	// We can just use the base offset in the complete class.
2361	CharUnits NonVirtualOffset = Vptr.Base.getBaseOffset();
2362
2363	if (!NonVirtualOffset.isZero())
2364	This =
2365	ApplyNonVirtualAndVirtualOffset(CGF&: *this, addr: This, nonVirtualOffset: NonVirtualOffset, virtualOffset: nullptr,
2366	derivedClass: Vptr.VTableClass, nearestVBase: Vptr.NearestVBase);
2367
2368	llvm::Value *VPtrValue =
2369	GetVTablePtr(This, VTableTy: VTableGlobal->getType(), VTableClass: Vptr.VTableClass);
2370	llvm::Value *Cmp =
2371	Builder.CreateICmpEQ(LHS: VPtrValue, RHS: VTableGlobal, Name: "cmp.vtables");
2372	Builder.CreateAssumption(Cond: Cmp);
2373	}
2374
2375	void CodeGenFunction::EmitVTableAssumptionLoads(const CXXRecordDecl *ClassDecl,
2376	Address This) {
2377	if (CGM.getCXXABI().doStructorsInitializeVPtrs(VTableClass: ClassDecl))
2378	for (const VPtr &Vptr : getVTablePointers(VTableClass: ClassDecl))
2379	EmitVTableAssumptionLoad(Vptr, This);
2380	}
2381
2382	void
2383	CodeGenFunction::EmitSynthesizedCXXCopyCtorCall(const CXXConstructorDecl *D,
2384	Address This, Address Src,
2385	const CXXConstructExpr *E) {
2386	const FunctionProtoType *FPT = D->getType()->castAs<FunctionProtoType>();
2387
2388	CallArgList Args;
2389
2390	// Push the this ptr.
2391	Args.add(rvalue: RValue::get(V: This.getPointer()), type: D->getThisType());
2392
2393	// Push the src ptr.
2394	QualType QT = *(FPT->param_type_begin());
2395	llvm::Type *t = CGM.getTypes().ConvertType(T: QT);
2396	llvm::Value *SrcVal = Builder.CreateBitCast(V: Src.getPointer(), DestTy: t);
2397	Args.add(rvalue: RValue::get(V: SrcVal), type: QT);
2398
2399	// Skip over first argument (Src).
2400	EmitCallArgs(Args, Prototype: FPT, ArgRange: drop_begin(E->arguments(), 1), AC: E->getConstructor(),
2401	/*ParamsToSkip*/ 1);
2402
2403	EmitCXXConstructorCall(D, Ctor_Complete, /*ForVirtualBase*/false,
2404	/*Delegating*/false, This, Args,
2405	AggValueSlot::MayOverlap, E->getExprLoc(),
2406	/*NewPointerIsChecked*/false);
2407	}
2408
2409	void
2410	CodeGenFunction::EmitDelegateCXXConstructorCall(const CXXConstructorDecl *Ctor,
2411	CXXCtorType CtorType,
2412	const FunctionArgList &Args,
2413	SourceLocation Loc) {
2414	CallArgList DelegateArgs;
2415
2416	FunctionArgList::const_iterator I = Args.begin(), E = Args.end();
2417	assert(I != E && "no parameters to constructor");
2418
2419	// this
2420	Address This = LoadCXXThisAddress();
2421	DelegateArgs.add(rvalue: RValue::get(V: This.getPointer()), type: (*I)->getType());
2422	++I;
2423
2424	// FIXME: The location of the VTT parameter in the parameter list is
2425	// specific to the Itanium ABI and shouldn't be hardcoded here.
2426	if (CGM.getCXXABI().NeedsVTTParameter(GD: CurGD)) {
2427	assert(I != E && "cannot skip vtt parameter, already done with args");
2428	assert((*I)->getType()->isPointerType() &&
2429	"skipping parameter not of vtt type");
2430	++I;
2431	}
2432
2433	// Explicit arguments.
2434	for (; I != E; ++I) {
2435	const VarDecl *param = *I;
2436	// FIXME: per-argument source location
2437	EmitDelegateCallArg(args&: DelegateArgs, param, loc: Loc);
2438	}
2439
2440	EmitCXXConstructorCall(D: Ctor, Type: CtorType, /*ForVirtualBase=*/false,
2441	/*Delegating=*/true, This, Args&: DelegateArgs,
2442	Overlap: AggValueSlot::MayOverlap, Loc,
2443	/*NewPointerIsChecked=*/true);
2444	}
2445
2446	namespace {
2447	struct CallDelegatingCtorDtor final : EHScopeStack::Cleanup {
2448	const CXXDestructorDecl *Dtor;
2449	Address Addr;
2450	CXXDtorType Type;
2451
2452	CallDelegatingCtorDtor(const CXXDestructorDecl *D, Address Addr,
2453	CXXDtorType Type)
2454	: Dtor(D), Addr(Addr), Type(Type) {}
2455
2456	void Emit(CodeGenFunction &CGF, Flags flags) override {
2457	// We are calling the destructor from within the constructor.
2458	// Therefore, "this" should have the expected type.
2459	QualType ThisTy = Dtor->getFunctionObjectParameterType();
2460	CGF.EmitCXXDestructorCall(D: Dtor, Type, /*ForVirtualBase=*/false,
2461	/*Delegating=*/true, This: Addr, ThisTy);
2462	}
2463	};
2464	} // end anonymous namespace
2465
2466	void
2467	CodeGenFunction::EmitDelegatingCXXConstructorCall(const CXXConstructorDecl *Ctor,
2468	const FunctionArgList &Args) {
2469	assert(Ctor->isDelegatingConstructor());
2470
2471	Address ThisPtr = LoadCXXThisAddress();
2472
2473	AggValueSlot AggSlot =
2474	AggValueSlot::forAddr(addr: ThisPtr, quals: Qualifiers(),
2475	isDestructed: AggValueSlot::IsDestructed,
2476	needsGC: AggValueSlot::DoesNotNeedGCBarriers,
2477	isAliased: AggValueSlot::IsNotAliased,
2478	mayOverlap: AggValueSlot::MayOverlap,
2479	isZeroed: AggValueSlot::IsNotZeroed,
2480	// Checks are made by the code that calls constructor.
2481	isChecked: AggValueSlot::IsSanitizerChecked);
2482
2483	EmitAggExpr(E: Ctor->init_begin()[0]->getInit(), AS: AggSlot);
2484
2485	const CXXRecordDecl *ClassDecl = Ctor->getParent();
2486	if (CGM.getLangOpts().Exceptions && !ClassDecl->hasTrivialDestructor()) {
2487	CXXDtorType Type =
2488	CurGD.getCtorType() == Ctor_Complete ? Dtor_Complete : Dtor_Base;
2489
2490	EHStack.pushCleanup<CallDelegatingCtorDtor>(Kind: EHCleanup,
2491	A: ClassDecl->getDestructor(),
2492	A: ThisPtr, A: Type);
2493	}
2494	}
2495
2496	void CodeGenFunction::EmitCXXDestructorCall(const CXXDestructorDecl *DD,
2497	CXXDtorType Type,
2498	bool ForVirtualBase,
2499	bool Delegating, Address This,
2500	QualType ThisTy) {
2501	CGM.getCXXABI().EmitDestructorCall(CGF&: *this, DD, Type, ForVirtualBase,
2502	Delegating, This, ThisTy);
2503	}
2504
2505	namespace {
2506	struct CallLocalDtor final : EHScopeStack::Cleanup {
2507	const CXXDestructorDecl *Dtor;
2508	Address Addr;
2509	QualType Ty;
2510
2511	CallLocalDtor(const CXXDestructorDecl *D, Address Addr, QualType Ty)
2512	: Dtor(D), Addr(Addr), Ty(Ty) {}
2513
2514	void Emit(CodeGenFunction &CGF, Flags flags) override {
2515	CGF.EmitCXXDestructorCall(Dtor, Dtor_Complete,
2516	/*ForVirtualBase=*/false,
2517	/*Delegating=*/false, Addr, Ty);
2518	}
2519	};
2520	} // end anonymous namespace
2521
2522	void CodeGenFunction::PushDestructorCleanup(const CXXDestructorDecl *D,
2523	QualType T, Address Addr) {
2524	EHStack.pushCleanup<CallLocalDtor>(Kind: NormalAndEHCleanup, A: D, A: Addr, A: T);
2525	}
2526
2527	void CodeGenFunction::PushDestructorCleanup(QualType T, Address Addr) {
2528	CXXRecordDecl *ClassDecl = T->getAsCXXRecordDecl();
2529	if (!ClassDecl) return;
2530	if (ClassDecl->hasTrivialDestructor()) return;
2531
2532	const CXXDestructorDecl *D = ClassDecl->getDestructor();
2533	assert(D && D->isUsed() && "destructor not marked as used!");
2534	PushDestructorCleanup(D, T, Addr);
2535	}
2536
2537	void CodeGenFunction::InitializeVTablePointer(const VPtr &Vptr) {
2538	// Compute the address point.
2539	llvm::Value *VTableAddressPoint =
2540	CGM.getCXXABI().getVTableAddressPointInStructor(
2541	CGF&: *this, RD: Vptr.VTableClass, Base: Vptr.Base, NearestVBase: Vptr.NearestVBase);
2542
2543	if (!VTableAddressPoint)
2544	return;
2545
2546	// Compute where to store the address point.
2547	llvm::Value *VirtualOffset = nullptr;
2548	CharUnits NonVirtualOffset = CharUnits::Zero();
2549
2550	if (CGM.getCXXABI().isVirtualOffsetNeededForVTableField(CGF&: *this, Vptr)) {
2551	// We need to use the virtual base offset offset because the virtual base
2552	// might have a different offset in the most derived class.
2553
2554	VirtualOffset = CGM.getCXXABI().GetVirtualBaseClassOffset(
2555	CGF&: *this, This: LoadCXXThisAddress(), ClassDecl: Vptr.VTableClass, BaseClassDecl: Vptr.NearestVBase);
2556	NonVirtualOffset = Vptr.OffsetFromNearestVBase;
2557	} else {
2558	// We can just use the base offset in the complete class.
2559	NonVirtualOffset = Vptr.Base.getBaseOffset();
2560	}
2561
2562	// Apply the offsets.
2563	Address VTableField = LoadCXXThisAddress();
2564	if (!NonVirtualOffset.isZero() || VirtualOffset)
2565	VTableField = ApplyNonVirtualAndVirtualOffset(
2566	CGF&: *this, addr: VTableField, nonVirtualOffset: NonVirtualOffset, virtualOffset: VirtualOffset, derivedClass: Vptr.VTableClass,
2567	nearestVBase: Vptr.NearestVBase);
2568
2569	// Finally, store the address point. Use the same LLVM types as the field to
2570	// support optimization.
2571	unsigned GlobalsAS = CGM.getDataLayout().getDefaultGlobalsAddressSpace();
2572	llvm::Type *PtrTy = llvm::PointerType::get(C&: CGM.getLLVMContext(), AddressSpace: GlobalsAS);
2573	// vtable field is derived from `this` pointer, therefore they should be in
2574	// the same addr space. Note that this might not be LLVM address space 0.
2575	VTableField = VTableField.withElementType(ElemTy: PtrTy);
2576
2577	llvm::StoreInst *Store = Builder.CreateStore(Val: VTableAddressPoint, Addr: VTableField);
2578	TBAAAccessInfo TBAAInfo = CGM.getTBAAVTablePtrAccessInfo(VTablePtrType: PtrTy);
2579	CGM.DecorateInstructionWithTBAA(Inst: Store, TBAAInfo);
2580	if (CGM.getCodeGenOpts().OptimizationLevel > 0 &&
2581	CGM.getCodeGenOpts().StrictVTablePointers)
2582	CGM.DecorateInstructionWithInvariantGroup(I: Store, RD: Vptr.VTableClass);
2583	}
2584
2585	CodeGenFunction::VPtrsVector
2586	CodeGenFunction::getVTablePointers(const CXXRecordDecl *VTableClass) {
2587	CodeGenFunction::VPtrsVector VPtrsResult;
2588	VisitedVirtualBasesSetTy VBases;
2589	getVTablePointers(Base: BaseSubobject(VTableClass, CharUnits::Zero()),
2590	/*NearestVBase=*/nullptr,
2591	/*OffsetFromNearestVBase=*/CharUnits::Zero(),
2592	/*BaseIsNonVirtualPrimaryBase=*/false, VTableClass, VBases,
2593	vptrs&: VPtrsResult);
2594	return VPtrsResult;
2595	}
2596
2597	void CodeGenFunction::getVTablePointers(BaseSubobject Base,
2598	const CXXRecordDecl *NearestVBase,
2599	CharUnits OffsetFromNearestVBase,
2600	bool BaseIsNonVirtualPrimaryBase,
2601	const CXXRecordDecl *VTableClass,
2602	VisitedVirtualBasesSetTy &VBases,
2603	VPtrsVector &Vptrs) {
2604	// If this base is a non-virtual primary base the address point has already
2605	// been set.
2606	if (!BaseIsNonVirtualPrimaryBase) {
2607	// Initialize the vtable pointer for this base.
2608	VPtr Vptr = {.Base: Base, .NearestVBase: NearestVBase, .OffsetFromNearestVBase: OffsetFromNearestVBase, .VTableClass: VTableClass};
2609	Vptrs.push_back(Elt: Vptr);
2610	}
2611
2612	const CXXRecordDecl *RD = Base.getBase();
2613
2614	// Traverse bases.
2615	for (const auto &I : RD->bases()) {
2616	auto *BaseDecl =
2617	cast<CXXRecordDecl>(Val: I.getType()->castAs<RecordType>()->getDecl());
2618
2619	// Ignore classes without a vtable.
2620	if (!BaseDecl->isDynamicClass())
2621	continue;
2622
2623	CharUnits BaseOffset;
2624	CharUnits BaseOffsetFromNearestVBase;
2625	bool BaseDeclIsNonVirtualPrimaryBase;
2626
2627	if (I.isVirtual()) {
2628	// Check if we've visited this virtual base before.
2629	if (!VBases.insert(Ptr: BaseDecl).second)
2630	continue;
2631
2632	const ASTRecordLayout &Layout =
2633	getContext().getASTRecordLayout(VTableClass);
2634
2635	BaseOffset = Layout.getVBaseClassOffset(VBase: BaseDecl);
2636	BaseOffsetFromNearestVBase = CharUnits::Zero();
2637	BaseDeclIsNonVirtualPrimaryBase = false;
2638	} else {
2639	const ASTRecordLayout &Layout = getContext().getASTRecordLayout(RD);
2640
2641	BaseOffset = Base.getBaseOffset() + Layout.getBaseClassOffset(Base: BaseDecl);
2642	BaseOffsetFromNearestVBase =
2643	OffsetFromNearestVBase + Layout.getBaseClassOffset(Base: BaseDecl);
2644	BaseDeclIsNonVirtualPrimaryBase = Layout.getPrimaryBase() == BaseDecl;
2645	}
2646
2647	getVTablePointers(
2648	Base: BaseSubobject(BaseDecl, BaseOffset),
2649	NearestVBase: I.isVirtual() ? BaseDecl : NearestVBase, OffsetFromNearestVBase: BaseOffsetFromNearestVBase,
2650	BaseIsNonVirtualPrimaryBase: BaseDeclIsNonVirtualPrimaryBase, VTableClass, VBases, Vptrs);
2651	}
2652	}
2653
2654	void CodeGenFunction::InitializeVTablePointers(const CXXRecordDecl *RD) {
2655	// Ignore classes without a vtable.
2656	if (!RD->isDynamicClass())
2657	return;
2658
2659	// Initialize the vtable pointers for this class and all of its bases.
2660	if (CGM.getCXXABI().doStructorsInitializeVPtrs(VTableClass: RD))
2661	for (const VPtr &Vptr : getVTablePointers(VTableClass: RD))
2662	InitializeVTablePointer(Vptr);
2663
2664	if (RD->getNumVBases())
2665	CGM.getCXXABI().initializeHiddenVirtualInheritanceMembers(CGF&: *this, RD);
2666	}
2667
2668	llvm::Value *CodeGenFunction::GetVTablePtr(Address This,
2669	llvm::Type *VTableTy,
2670	const CXXRecordDecl *RD) {
2671	Address VTablePtrSrc = This.withElementType(ElemTy: VTableTy);
2672	llvm::Instruction *VTable = Builder.CreateLoad(Addr: VTablePtrSrc, Name: "vtable");
2673	TBAAAccessInfo TBAAInfo = CGM.getTBAAVTablePtrAccessInfo(VTablePtrType: VTableTy);
2674	CGM.DecorateInstructionWithTBAA(Inst: VTable, TBAAInfo);
2675
2676	if (CGM.getCodeGenOpts().OptimizationLevel > 0 &&
2677	CGM.getCodeGenOpts().StrictVTablePointers)
2678	CGM.DecorateInstructionWithInvariantGroup(I: VTable, RD);
2679
2680	return VTable;
2681	}
2682
2683	// If a class has a single non-virtual base and does not introduce or override
2684	// virtual member functions or fields, it will have the same layout as its base.
2685	// This function returns the least derived such class.
2686	//
2687	// Casting an instance of a base class to such a derived class is technically
2688	// undefined behavior, but it is a relatively common hack for introducing member
2689	// functions on class instances with specific properties (e.g. llvm::Operator)
2690	// that works under most compilers and should not have security implications, so
2691	// we allow it by default. It can be disabled with -fsanitize=cfi-cast-strict.
2692	static const CXXRecordDecl *
2693	LeastDerivedClassWithSameLayout(const CXXRecordDecl *RD) {
2694	if (!RD->field_empty())
2695	return RD;
2696
2697	if (RD->getNumVBases() != 0)
2698	return RD;
2699
2700	if (RD->getNumBases() != 1)
2701	return RD;
2702
2703	for (const CXXMethodDecl *MD : RD->methods()) {
2704	if (MD->isVirtual()) {
2705	// Virtual member functions are only ok if they are implicit destructors
2706	// because the implicit destructor will have the same semantics as the
2707	// base class's destructor if no fields are added.
2708	if (isa<CXXDestructorDecl>(Val: MD) && MD->isImplicit())
2709	continue;
2710	return RD;
2711	}
2712	}
2713
2714	return LeastDerivedClassWithSameLayout(
2715	RD: RD->bases_begin()->getType()->getAsCXXRecordDecl());
2716	}
2717
2718	void CodeGenFunction::EmitTypeMetadataCodeForVCall(const CXXRecordDecl *RD,
2719	llvm::Value *VTable,
2720	SourceLocation Loc) {
2721	if (SanOpts.has(K: SanitizerKind::CFIVCall))
2722	EmitVTablePtrCheckForCall(RD, VTable, TCK: CodeGenFunction::CFITCK_VCall, Loc);
2723	else if (CGM.getCodeGenOpts().WholeProgramVTables &&
2724	// Don't insert type test assumes if we are forcing public
2725	// visibility.
2726	!CGM.AlwaysHasLTOVisibilityPublic(RD)) {
2727	QualType Ty = QualType(RD->getTypeForDecl(), 0);
2728	llvm::Metadata *MD = CGM.CreateMetadataIdentifierForType(T: Ty);
2729	llvm::Value *TypeId =
2730	llvm::MetadataAsValue::get(Context&: CGM.getLLVMContext(), MD);
2731
2732	// If we already know that the call has hidden LTO visibility, emit
2733	// @llvm.type.test(). Otherwise emit @llvm.public.type.test(), which WPD
2734	// will convert to @llvm.type.test() if we assert at link time that we have
2735	// whole program visibility.
2736	llvm::Intrinsic::ID IID = CGM.HasHiddenLTOVisibility(RD)
2737	? llvm::Intrinsic::type_test
2738	: llvm::Intrinsic::public_type_test;
2739	llvm::Value *TypeTest =
2740	Builder.CreateCall(Callee: CGM.getIntrinsic(IID), Args: {VTable, TypeId});
2741	Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::assume), TypeTest);
2742	}
2743	}
2744
2745	void CodeGenFunction::EmitVTablePtrCheckForCall(const CXXRecordDecl *RD,
2746	llvm::Value *VTable,
2747	CFITypeCheckKind TCK,
2748	SourceLocation Loc) {
2749	if (!SanOpts.has(K: SanitizerKind::CFICastStrict))
2750	RD = LeastDerivedClassWithSameLayout(RD);
2751
2752	EmitVTablePtrCheck(RD, VTable, TCK, Loc);
2753	}
2754
2755	void CodeGenFunction::EmitVTablePtrCheckForCast(QualType T, Address Derived,
2756	bool MayBeNull,
2757	CFITypeCheckKind TCK,
2758	SourceLocation Loc) {
2759	if (!getLangOpts().CPlusPlus)
2760	return;
2761
2762	auto *ClassTy = T->getAs<RecordType>();
2763	if (!ClassTy)
2764	return;
2765
2766	const CXXRecordDecl *ClassDecl = cast<CXXRecordDecl>(Val: ClassTy->getDecl());
2767
2768	if (!ClassDecl->isCompleteDefinition() || !ClassDecl->isDynamicClass())
2769	return;
2770
2771	if (!SanOpts.has(K: SanitizerKind::CFICastStrict))
2772	ClassDecl = LeastDerivedClassWithSameLayout(RD: ClassDecl);
2773
2774	llvm::BasicBlock *ContBlock = nullptr;
2775
2776	if (MayBeNull) {
2777	llvm::Value *DerivedNotNull =
2778	Builder.CreateIsNotNull(Arg: Derived.getPointer(), Name: "cast.nonnull");
2779
2780	llvm::BasicBlock *CheckBlock = createBasicBlock(name: "cast.check");
2781	ContBlock = createBasicBlock(name: "cast.cont");
2782
2783	Builder.CreateCondBr(Cond: DerivedNotNull, True: CheckBlock, False: ContBlock);
2784
2785	EmitBlock(BB: CheckBlock);
2786	}
2787
2788	llvm::Value *VTable;
2789	std::tie(args&: VTable, args&: ClassDecl) =
2790	CGM.getCXXABI().LoadVTablePtr(CGF&: *this, This: Derived, RD: ClassDecl);
2791
2792	EmitVTablePtrCheck(RD: ClassDecl, VTable, TCK, Loc);
2793
2794	if (MayBeNull) {
2795	Builder.CreateBr(Dest: ContBlock);
2796	EmitBlock(BB: ContBlock);
2797	}
2798	}
2799
2800	void CodeGenFunction::EmitVTablePtrCheck(const CXXRecordDecl *RD,
2801	llvm::Value *VTable,
2802	CFITypeCheckKind TCK,
2803	SourceLocation Loc) {
2804	if (!CGM.getCodeGenOpts().SanitizeCfiCrossDso &&
2805	!CGM.HasHiddenLTOVisibility(RD))
2806	return;
2807
2808	SanitizerMask M;
2809	llvm::SanitizerStatKind SSK;
2810	switch (TCK) {
2811	case CFITCK_VCall:
2812	M = SanitizerKind::CFIVCall;
2813	SSK = llvm::SanStat_CFI_VCall;
2814	break;
2815	case CFITCK_NVCall:
2816	M = SanitizerKind::CFINVCall;
2817	SSK = llvm::SanStat_CFI_NVCall;
2818	break;
2819	case CFITCK_DerivedCast:
2820	M = SanitizerKind::CFIDerivedCast;
2821	SSK = llvm::SanStat_CFI_DerivedCast;
2822	break;
2823	case CFITCK_UnrelatedCast:
2824	M = SanitizerKind::CFIUnrelatedCast;
2825	SSK = llvm::SanStat_CFI_UnrelatedCast;
2826	break;
2827	case CFITCK_ICall:
2828	case CFITCK_NVMFCall:
2829	case CFITCK_VMFCall:
2830	llvm_unreachable("unexpected sanitizer kind");
2831	}
2832
2833	std::string TypeName = RD->getQualifiedNameAsString();
2834	if (getContext().getNoSanitizeList().containsType(Mask: M, MangledTypeName: TypeName))
2835	return;
2836
2837	SanitizerScope SanScope(this);
2838	EmitSanitizerStatReport(SSK);
2839
2840	llvm::Metadata *MD =
2841	CGM.CreateMetadataIdentifierForType(T: QualType(RD->getTypeForDecl(), 0));
2842	llvm::Value *TypeId = llvm::MetadataAsValue::get(Context&: getLLVMContext(), MD);
2843
2844	llvm::Value *TypeTest = Builder.CreateCall(
2845	CGM.getIntrinsic(llvm::Intrinsic::type_test), {VTable, TypeId});
2846
2847	llvm::Constant *StaticData[] = {
2848	llvm::ConstantInt::get(Ty: Int8Ty, V: TCK),
2849	EmitCheckSourceLocation(Loc),
2850	EmitCheckTypeDescriptor(T: QualType(RD->getTypeForDecl(), 0)),
2851	};
2852
2853	auto CrossDsoTypeId = CGM.CreateCrossDsoCfiTypeId(MD);
2854	if (CGM.getCodeGenOpts().SanitizeCfiCrossDso && CrossDsoTypeId) {
2855	EmitCfiSlowPathCheck(Kind: M, Cond: TypeTest, TypeId: CrossDsoTypeId, Ptr: VTable, StaticArgs: StaticData);
2856	return;
2857	}
2858
2859	if (CGM.getCodeGenOpts().SanitizeTrap.has(K: M)) {
2860	EmitTrapCheck(Checked: TypeTest, CheckHandlerID: SanitizerHandler::CFICheckFail);
2861	return;
2862	}
2863
2864	llvm::Value *AllVtables = llvm::MetadataAsValue::get(
2865	Context&: CGM.getLLVMContext(),
2866	MD: llvm::MDString::get(Context&: CGM.getLLVMContext(), Str: "all-vtables"));
2867	llvm::Value *ValidVtable = Builder.CreateCall(
2868	CGM.getIntrinsic(llvm::Intrinsic::type_test), {VTable, AllVtables});
2869	EmitCheck(Checked: std::make_pair(x&: TypeTest, y&: M), Check: SanitizerHandler::CFICheckFail,
2870	StaticArgs: StaticData, DynamicArgs: {VTable, ValidVtable});
2871	}
2872
2873	bool CodeGenFunction::ShouldEmitVTableTypeCheckedLoad(const CXXRecordDecl *RD) {
2874	if (!CGM.getCodeGenOpts().WholeProgramVTables ||
2875	!CGM.HasHiddenLTOVisibility(RD))
2876	return false;
2877
2878	if (CGM.getCodeGenOpts().VirtualFunctionElimination)
2879	return true;
2880
2881	if (!SanOpts.has(K: SanitizerKind::CFIVCall) ||
2882	!CGM.getCodeGenOpts().SanitizeTrap.has(K: SanitizerKind::CFIVCall))
2883	return false;
2884
2885	std::string TypeName = RD->getQualifiedNameAsString();
2886	return !getContext().getNoSanitizeList().containsType(Mask: SanitizerKind::CFIVCall,
2887	MangledTypeName: TypeName);
2888	}
2889
2890	llvm::Value *CodeGenFunction::EmitVTableTypeCheckedLoad(
2891	const CXXRecordDecl *RD, llvm::Value *VTable, llvm::Type *VTableTy,
2892	uint64_t VTableByteOffset) {
2893	SanitizerScope SanScope(this);
2894
2895	EmitSanitizerStatReport(SSK: llvm::SanStat_CFI_VCall);
2896
2897	llvm::Metadata *MD =
2898	CGM.CreateMetadataIdentifierForType(T: QualType(RD->getTypeForDecl(), 0));
2899	llvm::Value *TypeId = llvm::MetadataAsValue::get(Context&: CGM.getLLVMContext(), MD);
2900
2901	llvm::Value *CheckedLoad = Builder.CreateCall(
2902	CGM.getIntrinsic(llvm::Intrinsic::type_checked_load),
2903	{VTable, llvm::ConstantInt::get(Int32Ty, VTableByteOffset), TypeId});
2904	llvm::Value *CheckResult = Builder.CreateExtractValue(Agg: CheckedLoad, Idxs: 1);
2905
2906	std::string TypeName = RD->getQualifiedNameAsString();
2907	if (SanOpts.has(K: SanitizerKind::CFIVCall) &&
2908	!getContext().getNoSanitizeList().containsType(Mask: SanitizerKind::CFIVCall,
2909	MangledTypeName: TypeName)) {
2910	EmitCheck(Checked: std::make_pair(x&: CheckResult, y: SanitizerKind::CFIVCall),
2911	Check: SanitizerHandler::CFICheckFail, StaticArgs: {}, DynamicArgs: {});
2912	}
2913
2914	return Builder.CreateBitCast(V: Builder.CreateExtractValue(Agg: CheckedLoad, Idxs: 0),
2915	DestTy: VTableTy);
2916	}
2917
2918	void CodeGenFunction::EmitForwardingCallToLambda(
2919	const CXXMethodDecl *callOperator, CallArgList &callArgs,
2920	const CGFunctionInfo *calleeFnInfo, llvm::Constant *calleePtr) {
2921	// Get the address of the call operator.
2922	if (!calleeFnInfo)
2923	calleeFnInfo = &CGM.getTypes().arrangeCXXMethodDeclaration(MD: callOperator);
2924
2925	if (!calleePtr)
2926	calleePtr =
2927	CGM.GetAddrOfFunction(GD: GlobalDecl(callOperator),
2928	Ty: CGM.getTypes().GetFunctionType(Info: *calleeFnInfo));
2929
2930	// Prepare the return slot.
2931	const FunctionProtoType *FPT =
2932	callOperator->getType()->castAs<FunctionProtoType>();
2933	QualType resultType = FPT->getReturnType();
2934	ReturnValueSlot returnSlot;
2935	if (!resultType->isVoidType() &&
2936	calleeFnInfo->getReturnInfo().getKind() == ABIArgInfo::Indirect &&
2937	!hasScalarEvaluationKind(T: calleeFnInfo->getReturnType()))
2938	returnSlot =
2939	ReturnValueSlot(ReturnValue, resultType.isVolatileQualified(),
2940	/*IsUnused=*/false, /*IsExternallyDestructed=*/true);
2941
2942	// We don't need to separately arrange the call arguments because
2943	// the call can't be variadic anyway --- it's impossible to forward
2944	// variadic arguments.
2945
2946	// Now emit our call.
2947	auto callee = CGCallee::forDirect(functionPtr: calleePtr, abstractInfo: GlobalDecl(callOperator));
2948	RValue RV = EmitCall(*calleeFnInfo, callee, returnSlot, callArgs);
2949
2950	// If necessary, copy the returned value into the slot.
2951	if (!resultType->isVoidType() && returnSlot.isNull()) {
2952	if (getLangOpts().ObjCAutoRefCount && resultType->isObjCRetainableType()) {
2953	RV = RValue::get(V: EmitARCRetainAutoreleasedReturnValue(value: RV.getScalarVal()));
2954	}
2955	EmitReturnOfRValue(RV, Ty: resultType);
2956	} else
2957	EmitBranchThroughCleanup(Dest: ReturnBlock);
2958	}
2959
2960	void CodeGenFunction::EmitLambdaBlockInvokeBody() {
2961	const BlockDecl *BD = BlockInfo->getBlockDecl();
2962	const VarDecl *variable = BD->capture_begin()->getVariable();
2963	const CXXRecordDecl *Lambda = variable->getType()->getAsCXXRecordDecl();
2964	const CXXMethodDecl *CallOp = Lambda->getLambdaCallOperator();
2965
2966	if (CallOp->isVariadic()) {
2967	// FIXME: Making this work correctly is nasty because it requires either
2968	// cloning the body of the call operator or making the call operator
2969	// forward.
2970	CGM.ErrorUnsupported(D: CurCodeDecl, Type: "lambda conversion to variadic function");
2971	return;
2972	}
2973
2974	// Start building arguments for forwarding call
2975	CallArgList CallArgs;
2976
2977	QualType ThisType = getContext().getPointerType(T: getContext().getRecordType(Lambda));
2978	Address ThisPtr = GetAddrOfBlockDecl(var: variable);
2979	CallArgs.add(rvalue: RValue::get(V: ThisPtr.getPointer()), type: ThisType);
2980
2981	// Add the rest of the parameters.
2982	for (auto *param : BD->parameters())
2983	EmitDelegateCallArg(args&: CallArgs, param, loc: param->getBeginLoc());
2984
2985	assert(!Lambda->isGenericLambda() &&
2986	"generic lambda interconversion to block not implemented");
2987	EmitForwardingCallToLambda(callOperator: CallOp, callArgs&: CallArgs);
2988	}
2989
2990	void CodeGenFunction::EmitLambdaStaticInvokeBody(const CXXMethodDecl *MD) {
2991	if (MD->isVariadic()) {
2992	// FIXME: Making this work correctly is nasty because it requires either
2993	// cloning the body of the call operator or making the call operator
2994	// forward.
2995	CGM.ErrorUnsupported(MD, "lambda conversion to variadic function");
2996	return;
2997	}
2998
2999	const CXXRecordDecl *Lambda = MD->getParent();
3000
3001	// Start building arguments for forwarding call
3002	CallArgList CallArgs;
3003
3004	QualType LambdaType = getContext().getRecordType(Lambda);
3005	QualType ThisType = getContext().getPointerType(T: LambdaType);
3006	Address ThisPtr = CreateMemTemp(T: LambdaType, Name: "unused.capture");
3007	CallArgs.add(rvalue: RValue::get(V: ThisPtr.getPointer()), type: ThisType);
3008
3009	EmitLambdaDelegatingInvokeBody(MD, CallArgs);
3010	}
3011
3012	void CodeGenFunction::EmitLambdaDelegatingInvokeBody(const CXXMethodDecl *MD,
3013	CallArgList &CallArgs) {
3014	// Add the rest of the forwarded parameters.
3015	for (auto *Param : MD->parameters())
3016	EmitDelegateCallArg(CallArgs, Param, Param->getBeginLoc());
3017
3018	const CXXRecordDecl *Lambda = MD->getParent();
3019	const CXXMethodDecl *CallOp = Lambda->getLambdaCallOperator();
3020	// For a generic lambda, find the corresponding call operator specialization
3021	// to which the call to the static-invoker shall be forwarded.
3022	if (Lambda->isGenericLambda()) {
3023	assert(MD->isFunctionTemplateSpecialization());
3024	const TemplateArgumentList *TAL = MD->getTemplateSpecializationArgs();
3025	FunctionTemplateDecl *CallOpTemplate = CallOp->getDescribedFunctionTemplate();
3026	void *InsertPos = nullptr;
3027	FunctionDecl *CorrespondingCallOpSpecialization =
3028	CallOpTemplate->findSpecialization(Args: TAL->asArray(), InsertPos);
3029	assert(CorrespondingCallOpSpecialization);
3030	CallOp = cast<CXXMethodDecl>(Val: CorrespondingCallOpSpecialization);
3031	}
3032
3033	// Special lambda forwarding when there are inalloca parameters.
3034	if (hasInAllocaArg(MD)) {
3035	const CGFunctionInfo *ImplFnInfo = nullptr;
3036	llvm::Function *ImplFn = nullptr;
3037	EmitLambdaInAllocaImplFn(CallOp, ImplFnInfo: &ImplFnInfo, ImplFn: &ImplFn);
3038
3039	EmitForwardingCallToLambda(callOperator: CallOp, callArgs&: CallArgs, calleeFnInfo: ImplFnInfo, calleePtr: ImplFn);
3040	return;
3041	}
3042
3043	EmitForwardingCallToLambda(callOperator: CallOp, callArgs&: CallArgs);
3044	}
3045
3046	void CodeGenFunction::EmitLambdaInAllocaCallOpBody(const CXXMethodDecl *MD) {
3047	if (MD->isVariadic()) {
3048	// FIXME: Making this work correctly is nasty because it requires either
3049	// cloning the body of the call operator or making the call operator forward.
3050	CGM.ErrorUnsupported(MD, "lambda conversion to variadic function");
3051	return;
3052	}
3053
3054	// Forward %this argument.
3055	CallArgList CallArgs;
3056	QualType LambdaType = getContext().getRecordType(MD->getParent());
3057	QualType ThisType = getContext().getPointerType(T: LambdaType);
3058	llvm::Value *ThisArg = CurFn->getArg(i: 0);
3059	CallArgs.add(rvalue: RValue::get(V: ThisArg), type: ThisType);
3060
3061	EmitLambdaDelegatingInvokeBody(MD, CallArgs);
3062	}
3063
3064	void CodeGenFunction::EmitLambdaInAllocaImplFn(
3065	const CXXMethodDecl *CallOp, const CGFunctionInfo **ImplFnInfo,
3066	llvm::Function **ImplFn) {
3067	const CGFunctionInfo &FnInfo =
3068	CGM.getTypes().arrangeCXXMethodDeclaration(MD: CallOp);
3069	llvm::Function *CallOpFn =
3070	cast<llvm::Function>(Val: CGM.GetAddrOfFunction(GD: GlobalDecl(CallOp)));
3071
3072	// Emit function containing the original call op body. __invoke will delegate
3073	// to this function.
3074	SmallVector<CanQualType, 4> ArgTypes;
3075	for (auto I = FnInfo.arg_begin(); I != FnInfo.arg_end(); ++I)
3076	ArgTypes.push_back(Elt: I->type);
3077	*ImplFnInfo = &CGM.getTypes().arrangeLLVMFunctionInfo(
3078	returnType: FnInfo.getReturnType(), opts: FnInfoOpts::IsDelegateCall, argTypes: ArgTypes,
3079	info: FnInfo.getExtInfo(), paramInfos: {}, args: FnInfo.getRequiredArgs());
3080
3081	// Create mangled name as if this was a method named __impl. If for some
3082	// reason the name doesn't look as expected then just tack __impl to the
3083	// front.
3084	// TODO: Use the name mangler to produce the right name instead of using
3085	// string replacement.
3086	StringRef CallOpName = CallOpFn->getName();
3087	std::string ImplName;
3088	if (size_t Pos = CallOpName.find_first_of(Chars: "<lambda"))
3089	ImplName = ("?__impl@" + CallOpName.drop_front(N: Pos)).str();
3090	else
3091	ImplName = ("__impl" + CallOpName).str();
3092
3093	llvm::Function *Fn = CallOpFn->getParent()->getFunction(Name: ImplName);
3094	if (!Fn) {
3095	Fn = llvm::Function::Create(Ty: CGM.getTypes().GetFunctionType(Info: **ImplFnInfo),
3096	Linkage: llvm::GlobalValue::InternalLinkage, N: ImplName,
3097	M&: CGM.getModule());
3098	CGM.SetInternalFunctionAttributes(CallOp, Fn, **ImplFnInfo);
3099
3100	const GlobalDecl &GD = GlobalDecl(CallOp);
3101	const auto *D = cast<FunctionDecl>(Val: GD.getDecl());
3102	CodeGenFunction(CGM).GenerateCode(GD, Fn, FnInfo: **ImplFnInfo);
3103	CGM.SetLLVMFunctionAttributesForDefinition(D: D, F: Fn);
3104	}
3105	*ImplFn = Fn;
3106	}
3107