1	//===--- MicrosoftCXXABI.cpp - Emit LLVM Code from ASTs for a Module ------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This provides C++ code generation targeting the Microsoft Visual C++ ABI.
10	// The class in this file generates structures that follow the Microsoft
11	// Visual C++ ABI, which is actually not very well documented at all outside
12	// of Microsoft.
13	//
14	//===----------------------------------------------------------------------===//
15
16	#include "ABIInfo.h"
17	#include "CGCXXABI.h"
18	#include "CGCleanup.h"
19	#include "CGVTables.h"
20	#include "CodeGenModule.h"
21	#include "CodeGenTypes.h"
22	#include "TargetInfo.h"
23	#include "clang/AST/Attr.h"
24	#include "clang/AST/CXXInheritance.h"
25	#include "clang/AST/Decl.h"
26	#include "clang/AST/DeclCXX.h"
27	#include "clang/AST/StmtCXX.h"
28	#include "clang/AST/VTableBuilder.h"
29	#include "clang/CodeGen/ConstantInitBuilder.h"
30	#include "llvm/ADT/StringExtras.h"
31	#include "llvm/ADT/StringSet.h"
32	#include "llvm/IR/Intrinsics.h"
33
34	using namespace clang;
35	using namespace CodeGen;
36
37	namespace {
38
39	/// Holds all the vbtable globals for a given class.
40	struct VBTableGlobals {
41	const VPtrInfoVector *VBTables;
42	SmallVector<llvm::GlobalVariable *, 2> Globals;
43	};
44
45	class MicrosoftCXXABI : public CGCXXABI {
46	public:
47	MicrosoftCXXABI(CodeGenModule &CGM)
48	: CGCXXABI(CGM), BaseClassDescriptorType(nullptr),
49	ClassHierarchyDescriptorType(nullptr),
50	CompleteObjectLocatorType(nullptr), CatchableTypeType(nullptr),
51	ThrowInfoType(nullptr) {
52	assert(!(CGM.getLangOpts().isExplicitDefaultVisibilityExportMapping() ||
53	CGM.getLangOpts().isAllDefaultVisibilityExportMapping()) &&
54	"visibility export mapping option unimplemented in this ABI");
55	}
56
57	bool HasThisReturn(GlobalDecl GD) const override;
58	bool hasMostDerivedReturn(GlobalDecl GD) const override;
59
60	bool classifyReturnType(CGFunctionInfo &FI) const override;
61
62	RecordArgABI getRecordArgABI(const CXXRecordDecl *RD) const override;
63
64	bool isSRetParameterAfterThis() const override { return true; }
65
66	bool isThisCompleteObject(GlobalDecl GD) const override {
67	// The Microsoft ABI doesn't use separate complete-object vs.
68	// base-object variants of constructors, but it does of destructors.
69	if (isa<CXXDestructorDecl>(Val: GD.getDecl())) {
70	switch (GD.getDtorType()) {
71	case Dtor_Complete:
72	case Dtor_Deleting:
73	return true;
74
75	case Dtor_Base:
76	return false;
77
78	case Dtor_Comdat: llvm_unreachable("emitting dtor comdat as function?");
79	}
80	llvm_unreachable("bad dtor kind");
81	}
82
83	// No other kinds.
84	return false;
85	}
86
87	size_t getSrcArgforCopyCtor(const CXXConstructorDecl *CD,
88	FunctionArgList &Args) const override {
89	assert(Args.size() >= 2 &&
90	"expected the arglist to have at least two args!");
91	// The 'most_derived' parameter goes second if the ctor is variadic and
92	// has v-bases.
93	if (CD->getParent()->getNumVBases() > 0 &&
94	CD->getType()->castAs<FunctionProtoType>()->isVariadic())
95	return 2;
96	return 1;
97	}
98
99	std::vector<CharUnits> getVBPtrOffsets(const CXXRecordDecl *RD) override {
100	std::vector<CharUnits> VBPtrOffsets;
101	const ASTContext &Context = getContext();
102	const ASTRecordLayout &Layout = Context.getASTRecordLayout(RD);
103
104	const VBTableGlobals &VBGlobals = enumerateVBTables(RD);
105	for (const std::unique_ptr<VPtrInfo> &VBT : *VBGlobals.VBTables) {
106	const ASTRecordLayout &SubobjectLayout =
107	Context.getASTRecordLayout(VBT->IntroducingObject);
108	CharUnits Offs = VBT->NonVirtualOffset;
109	Offs += SubobjectLayout.getVBPtrOffset();
110	if (VBT->getVBaseWithVPtr())
111	Offs += Layout.getVBaseClassOffset(VBase: VBT->getVBaseWithVPtr());
112	VBPtrOffsets.push_back(x: Offs);
113	}
114	llvm::array_pod_sort(Start: VBPtrOffsets.begin(), End: VBPtrOffsets.end());
115	return VBPtrOffsets;
116	}
117
118	StringRef GetPureVirtualCallName() override { return "_purecall"; }
119	StringRef GetDeletedVirtualCallName() override { return "_purecall"; }
120
121	void emitVirtualObjectDelete(CodeGenFunction &CGF, const CXXDeleteExpr *DE,
122	Address Ptr, QualType ElementType,
123	const CXXDestructorDecl *Dtor) override;
124
125	void emitRethrow(CodeGenFunction &CGF, bool isNoReturn) override;
126	void emitThrow(CodeGenFunction &CGF, const CXXThrowExpr *E) override;
127
128	void emitBeginCatch(CodeGenFunction &CGF, const CXXCatchStmt *C) override;
129
130	llvm::GlobalVariable *getMSCompleteObjectLocator(const CXXRecordDecl *RD,
131	const VPtrInfo &Info);
132
133	llvm::Constant *getAddrOfRTTIDescriptor(QualType Ty) override;
134	CatchTypeInfo
135	getAddrOfCXXCatchHandlerType(QualType Ty, QualType CatchHandlerType) override;
136
137	/// MSVC needs an extra flag to indicate a catchall.
138	CatchTypeInfo getCatchAllTypeInfo() override {
139	// For -EHa catch(...) must handle HW exception
140	// Adjective = HT_IsStdDotDot (0x40), only catch C++ exceptions
141	if (getContext().getLangOpts().EHAsynch)
142	return CatchTypeInfo{.RTTI: nullptr, .Flags: 0};
143	else
144	return CatchTypeInfo{.RTTI: nullptr, .Flags: 0x40};
145	}
146
147	bool shouldTypeidBeNullChecked(bool IsDeref, QualType SrcRecordTy) override;
148	void EmitBadTypeidCall(CodeGenFunction &CGF) override;
149	llvm::Value *EmitTypeid(CodeGenFunction &CGF, QualType SrcRecordTy,
150	Address ThisPtr,
151	llvm::Type *StdTypeInfoPtrTy) override;
152
153	bool shouldDynamicCastCallBeNullChecked(bool SrcIsPtr,
154	QualType SrcRecordTy) override;
155
156	bool shouldEmitExactDynamicCast(QualType DestRecordTy) override {
157	// TODO: Add support for exact dynamic_casts.
158	return false;
159	}
160	llvm::Value *emitExactDynamicCast(CodeGenFunction &CGF, Address Value,
161	QualType SrcRecordTy, QualType DestTy,
162	QualType DestRecordTy,
163	llvm::BasicBlock *CastSuccess,
164	llvm::BasicBlock *CastFail) override {
165	llvm_unreachable("unsupported");
166	}
167
168	llvm::Value *emitDynamicCastCall(CodeGenFunction &CGF, Address Value,
169	QualType SrcRecordTy, QualType DestTy,
170	QualType DestRecordTy,
171	llvm::BasicBlock *CastEnd) override;
172
173	llvm::Value *emitDynamicCastToVoid(CodeGenFunction &CGF, Address Value,
174	QualType SrcRecordTy) override;
175
176	bool EmitBadCastCall(CodeGenFunction &CGF) override;
177	bool canSpeculativelyEmitVTable(const CXXRecordDecl *RD) const override {
178	return false;
179	}
180
181	llvm::Value *
182	GetVirtualBaseClassOffset(CodeGenFunction &CGF, Address This,
183	const CXXRecordDecl *ClassDecl,
184	const CXXRecordDecl *BaseClassDecl) override;
185
186	llvm::BasicBlock *
187	EmitCtorCompleteObjectHandler(CodeGenFunction &CGF,
188	const CXXRecordDecl *RD) override;
189
190	llvm::BasicBlock *
191	EmitDtorCompleteObjectHandler(CodeGenFunction &CGF);
192
193	void initializeHiddenVirtualInheritanceMembers(CodeGenFunction &CGF,
194	const CXXRecordDecl *RD) override;
195
196	void EmitCXXConstructors(const CXXConstructorDecl *D) override;
197
198	// Background on MSVC destructors
199	// ==============================
200	//
201	// Both Itanium and MSVC ABIs have destructor variants. The variant names
202	// roughly correspond in the following way:
203	// Itanium Microsoft
204	// Base -> no name, just ~Class
205	// Complete -> vbase destructor
206	// Deleting -> scalar deleting destructor
207	// vector deleting destructor
208	//
209	// The base and complete destructors are the same as in Itanium, although the
210	// complete destructor does not accept a VTT parameter when there are virtual
211	// bases. A separate mechanism involving vtordisps is used to ensure that
212	// virtual methods of destroyed subobjects are not called.
213	//
214	// The deleting destructors accept an i32 bitfield as a second parameter. Bit
215	// 1 indicates if the memory should be deleted. Bit 2 indicates if the this
216	// pointer points to an array. The scalar deleting destructor assumes that
217	// bit 2 is zero, and therefore does not contain a loop.
218	//
219	// For virtual destructors, only one entry is reserved in the vftable, and it
220	// always points to the vector deleting destructor. The vector deleting
221	// destructor is the most general, so it can be used to destroy objects in
222	// place, delete single heap objects, or delete arrays.
223	//
224	// A TU defining a non-inline destructor is only guaranteed to emit a base
225	// destructor, and all of the other variants are emitted on an as-needed basis
226	// in COMDATs. Because a non-base destructor can be emitted in a TU that
227	// lacks a definition for the destructor, non-base destructors must always
228	// delegate to or alias the base destructor.
229
230	AddedStructorArgCounts
231	buildStructorSignature(GlobalDecl GD,
232	SmallVectorImpl<CanQualType> &ArgTys) override;
233
234	/// Non-base dtors should be emitted as delegating thunks in this ABI.
235	bool useThunkForDtorVariant(const CXXDestructorDecl *Dtor,
236	CXXDtorType DT) const override {
237	return DT != Dtor_Base;
238	}
239
240	void setCXXDestructorDLLStorage(llvm::GlobalValue *GV,
241	const CXXDestructorDecl *Dtor,
242	CXXDtorType DT) const override;
243
244	llvm::GlobalValue::LinkageTypes
245	getCXXDestructorLinkage(GVALinkage Linkage, const CXXDestructorDecl *Dtor,
246	CXXDtorType DT) const override;
247
248	void EmitCXXDestructors(const CXXDestructorDecl *D) override;
249
250	const CXXRecordDecl *getThisArgumentTypeForMethod(GlobalDecl GD) override {
251	auto *MD = cast<CXXMethodDecl>(Val: GD.getDecl());
252
253	if (MD->isVirtual()) {
254	GlobalDecl LookupGD = GD;
255	if (const auto *DD = dyn_cast<CXXDestructorDecl>(Val: MD)) {
256	// Complete dtors take a pointer to the complete object,
257	// thus don't need adjustment.
258	if (GD.getDtorType() == Dtor_Complete)
259	return MD->getParent();
260
261	// There's only Dtor_Deleting in vftable but it shares the this
262	// adjustment with the base one, so look up the deleting one instead.
263	LookupGD = GlobalDecl(DD, Dtor_Deleting);
264	}
265	MethodVFTableLocation ML =
266	CGM.getMicrosoftVTableContext().getMethodVFTableLocation(GD: LookupGD);
267
268	// The vbases might be ordered differently in the final overrider object
269	// and the complete object, so the "this" argument may sometimes point to
270	// memory that has no particular type (e.g. past the complete object).
271	// In this case, we just use a generic pointer type.
272	// FIXME: might want to have a more precise type in the non-virtual
273	// multiple inheritance case.
274	if (ML.VBase || !ML.VFPtrOffset.isZero())
275	return nullptr;
276	}
277	return MD->getParent();
278	}
279
280	Address
281	adjustThisArgumentForVirtualFunctionCall(CodeGenFunction &CGF, GlobalDecl GD,
282	Address This,
283	bool VirtualCall) override;
284
285	void addImplicitStructorParams(CodeGenFunction &CGF, QualType &ResTy,
286	FunctionArgList &Params) override;
287
288	void EmitInstanceFunctionProlog(CodeGenFunction &CGF) override;
289
290	AddedStructorArgs getImplicitConstructorArgs(CodeGenFunction &CGF,
291	const CXXConstructorDecl *D,
292	CXXCtorType Type,
293	bool ForVirtualBase,
294	bool Delegating) override;
295
296	llvm::Value *getCXXDestructorImplicitParam(CodeGenFunction &CGF,
297	const CXXDestructorDecl *DD,
298	CXXDtorType Type,
299	bool ForVirtualBase,
300	bool Delegating) override;
301
302	void EmitDestructorCall(CodeGenFunction &CGF, const CXXDestructorDecl *DD,
303	CXXDtorType Type, bool ForVirtualBase,
304	bool Delegating, Address This,
305	QualType ThisTy) override;
306
307	void emitVTableTypeMetadata(const VPtrInfo &Info, const CXXRecordDecl *RD,
308	llvm::GlobalVariable *VTable);
309
310	void emitVTableDefinitions(CodeGenVTables &CGVT,
311	const CXXRecordDecl *RD) override;
312
313	bool isVirtualOffsetNeededForVTableField(CodeGenFunction &CGF,
314	CodeGenFunction::VPtr Vptr) override;
315
316	/// Don't initialize vptrs if dynamic class
317	/// is marked with the 'novtable' attribute.
318	bool doStructorsInitializeVPtrs(const CXXRecordDecl *VTableClass) override {
319	return !VTableClass->hasAttr<MSNoVTableAttr>();
320	}
321
322	llvm::Constant *
323	getVTableAddressPoint(BaseSubobject Base,
324	const CXXRecordDecl *VTableClass) override;
325
326	llvm::Value *getVTableAddressPointInStructor(
327	CodeGenFunction &CGF, const CXXRecordDecl *VTableClass,
328	BaseSubobject Base, const CXXRecordDecl *NearestVBase) override;
329
330	llvm::GlobalVariable *getAddrOfVTable(const CXXRecordDecl *RD,
331	CharUnits VPtrOffset) override;
332
333	CGCallee getVirtualFunctionPointer(CodeGenFunction &CGF, GlobalDecl GD,
334	Address This, llvm::Type *Ty,
335	SourceLocation Loc) override;
336
337	llvm::Value *EmitVirtualDestructorCall(CodeGenFunction &CGF,
338	const CXXDestructorDecl *Dtor,
339	CXXDtorType DtorType, Address This,
340	DeleteOrMemberCallExpr E) override;
341
342	void adjustCallArgsForDestructorThunk(CodeGenFunction &CGF, GlobalDecl GD,
343	CallArgList &CallArgs) override {
344	assert(GD.getDtorType() == Dtor_Deleting &&
345	"Only deleting destructor thunks are available in this ABI");
346	CallArgs.add(rvalue: RValue::get(V: getStructorImplicitParamValue(CGF)),
347	type: getContext().IntTy);
348	}
349
350	void emitVirtualInheritanceTables(const CXXRecordDecl *RD) override;
351
352	llvm::GlobalVariable *
353	getAddrOfVBTable(const VPtrInfo &VBT, const CXXRecordDecl *RD,
354	llvm::GlobalVariable::LinkageTypes Linkage);
355
356	llvm::GlobalVariable *
357	getAddrOfVirtualDisplacementMap(const CXXRecordDecl *SrcRD,
358	const CXXRecordDecl *DstRD) {
359	SmallString<256> OutName;
360	llvm::raw_svector_ostream Out(OutName);
361	getMangleContext().mangleCXXVirtualDisplacementMap(SrcRD, DstRD, Out);
362	StringRef MangledName = OutName.str();
363
364	if (auto *VDispMap = CGM.getModule().getNamedGlobal(Name: MangledName))
365	return VDispMap;
366
367	MicrosoftVTableContext &VTContext = CGM.getMicrosoftVTableContext();
368	unsigned NumEntries = 1 + SrcRD->getNumVBases();
369	SmallVector<llvm::Constant *, 4> Map(NumEntries,
370	llvm::UndefValue::get(T: CGM.IntTy));
371	Map[0] = llvm::ConstantInt::get(Ty: CGM.IntTy, V: 0);
372	bool AnyDifferent = false;
373	for (const auto &I : SrcRD->vbases()) {
374	const CXXRecordDecl *VBase = I.getType()->getAsCXXRecordDecl();
375	if (!DstRD->isVirtuallyDerivedFrom(Base: VBase))
376	continue;
377
378	unsigned SrcVBIndex = VTContext.getVBTableIndex(Derived: SrcRD, VBase);
379	unsigned DstVBIndex = VTContext.getVBTableIndex(Derived: DstRD, VBase);
380	Map[SrcVBIndex] = llvm::ConstantInt::get(Ty: CGM.IntTy, V: DstVBIndex * 4);
381	AnyDifferent |= SrcVBIndex != DstVBIndex;
382	}
383	// This map would be useless, don't use it.
384	if (!AnyDifferent)
385	return nullptr;
386
387	llvm::ArrayType *VDispMapTy = llvm::ArrayType::get(ElementType: CGM.IntTy, NumElements: Map.size());
388	llvm::Constant *Init = llvm::ConstantArray::get(T: VDispMapTy, V: Map);
389	llvm::GlobalValue::LinkageTypes Linkage =
390	SrcRD->isExternallyVisible() && DstRD->isExternallyVisible()
391	? llvm::GlobalValue::LinkOnceODRLinkage
392	: llvm::GlobalValue::InternalLinkage;
393	auto *VDispMap = new llvm::GlobalVariable(
394	CGM.getModule(), VDispMapTy, /*isConstant=*/true, Linkage,
395	/*Initializer=*/Init, MangledName);
396	return VDispMap;
397	}
398
399	void emitVBTableDefinition(const VPtrInfo &VBT, const CXXRecordDecl *RD,
400	llvm::GlobalVariable *GV) const;
401
402	void setThunkLinkage(llvm::Function *Thunk, bool ForVTable,
403	GlobalDecl GD, bool ReturnAdjustment) override {
404	GVALinkage Linkage =
405	getContext().GetGVALinkageForFunction(FD: cast<FunctionDecl>(Val: GD.getDecl()));
406
407	if (Linkage == GVA_Internal)
408	Thunk->setLinkage(llvm::GlobalValue::InternalLinkage);
409	else if (ReturnAdjustment)
410	Thunk->setLinkage(llvm::GlobalValue::WeakODRLinkage);
411	else
412	Thunk->setLinkage(llvm::GlobalValue::LinkOnceODRLinkage);
413	}
414
415	bool exportThunk() override { return false; }
416
417	llvm::Value *performThisAdjustment(CodeGenFunction &CGF, Address This,
418	const ThisAdjustment &TA) override;
419
420	llvm::Value *performReturnAdjustment(CodeGenFunction &CGF, Address Ret,
421	const ReturnAdjustment &RA) override;
422
423	void EmitThreadLocalInitFuncs(
424	CodeGenModule &CGM, ArrayRef<const VarDecl *> CXXThreadLocals,
425	ArrayRef<llvm::Function *> CXXThreadLocalInits,
426	ArrayRef<const VarDecl *> CXXThreadLocalInitVars) override;
427
428	bool usesThreadWrapperFunction(const VarDecl *VD) const override {
429	return getContext().getLangOpts().isCompatibleWithMSVC(
430	MajorVersion: LangOptions::MSVC2019_5) &&
431	(!isEmittedWithConstantInitializer(VD) || mayNeedDestruction(VD));
432	}
433	LValue EmitThreadLocalVarDeclLValue(CodeGenFunction &CGF, const VarDecl *VD,
434	QualType LValType) override;
435
436	void EmitGuardedInit(CodeGenFunction &CGF, const VarDecl &D,
437	llvm::GlobalVariable *DeclPtr,
438	bool PerformInit) override;
439	void registerGlobalDtor(CodeGenFunction &CGF, const VarDecl &D,
440	llvm::FunctionCallee Dtor,
441	llvm::Constant *Addr) override;
442
443	// ==== Notes on array cookies =========
444	//
445	// MSVC seems to only use cookies when the class has a destructor; a
446	// two-argument usual array deallocation function isn't sufficient.
447	//
448	// For example, this code prints "100" and "1":
449	// struct A {
450	// char x;
451	// void *operator new[](size_t sz) {
452	// printf("%u\n", sz);
453	// return malloc(sz);
454	// }
455	// void operator delete[](void *p, size_t sz) {
456	// printf("%u\n", sz);
457	// free(p);
458	// }
459	// };
460	// int main() {
461	// A *p = new A[100];
462	// delete[] p;
463	// }
464	// Whereas it prints "104" and "104" if you give A a destructor.
465
466	bool requiresArrayCookie(const CXXDeleteExpr *expr,
467	QualType elementType) override;
468	bool requiresArrayCookie(const CXXNewExpr *expr) override;
469	CharUnits getArrayCookieSizeImpl(QualType type) override;
470	Address InitializeArrayCookie(CodeGenFunction &CGF,
471	Address NewPtr,
472	llvm::Value *NumElements,
473	const CXXNewExpr *expr,
474	QualType ElementType) override;
475	llvm::Value *readArrayCookieImpl(CodeGenFunction &CGF,
476	Address allocPtr,
477	CharUnits cookieSize) override;
478
479	friend struct MSRTTIBuilder;
480
481	bool isImageRelative() const {
482	return CGM.getTarget().getPointerWidth(AddrSpace: LangAS::Default) == 64;
483	}
484
485	// 5 routines for constructing the llvm types for MS RTTI structs.
486	llvm::StructType *getTypeDescriptorType(StringRef TypeInfoString) {
487	llvm::SmallString<32> TDTypeName("rtti.TypeDescriptor");
488	TDTypeName += llvm::utostr(X: TypeInfoString.size());
489	llvm::StructType *&TypeDescriptorType =
490	TypeDescriptorTypeMap[TypeInfoString.size()];
491	if (TypeDescriptorType)
492	return TypeDescriptorType;
493	llvm::Type *FieldTypes[] = {
494	CGM.Int8PtrPtrTy,
495	CGM.Int8PtrTy,
496	llvm::ArrayType::get(ElementType: CGM.Int8Ty, NumElements: TypeInfoString.size() + 1)};
497	TypeDescriptorType =
498	llvm::StructType::create(Context&: CGM.getLLVMContext(), Elements: FieldTypes, Name: TDTypeName);
499	return TypeDescriptorType;
500	}
501
502	llvm::Type *getImageRelativeType(llvm::Type *PtrType) {
503	if (!isImageRelative())
504	return PtrType;
505	return CGM.IntTy;
506	}
507
508	llvm::StructType *getBaseClassDescriptorType() {
509	if (BaseClassDescriptorType)
510	return BaseClassDescriptorType;
511	llvm::Type *FieldTypes[] = {
512	getImageRelativeType(PtrType: CGM.Int8PtrTy),
513	CGM.IntTy,
514	CGM.IntTy,
515	CGM.IntTy,
516	CGM.IntTy,
517	CGM.IntTy,
518	getImageRelativeType(PtrType: getClassHierarchyDescriptorType()->getPointerTo()),
519	};
520	BaseClassDescriptorType = llvm::StructType::create(
521	Context&: CGM.getLLVMContext(), Elements: FieldTypes, Name: "rtti.BaseClassDescriptor");
522	return BaseClassDescriptorType;
523	}
524
525	llvm::StructType *getClassHierarchyDescriptorType() {
526	if (ClassHierarchyDescriptorType)
527	return ClassHierarchyDescriptorType;
528	// Forward-declare RTTIClassHierarchyDescriptor to break a cycle.
529	ClassHierarchyDescriptorType = llvm::StructType::create(
530	Context&: CGM.getLLVMContext(), Name: "rtti.ClassHierarchyDescriptor");
531	llvm::Type *FieldTypes[] = {
532	CGM.IntTy,
533	CGM.IntTy,
534	CGM.IntTy,
535	getImageRelativeType(
536	PtrType: getBaseClassDescriptorType()->getPointerTo()->getPointerTo()),
537	};
538	ClassHierarchyDescriptorType->setBody(Elements: FieldTypes);
539	return ClassHierarchyDescriptorType;
540	}
541
542	llvm::StructType *getCompleteObjectLocatorType() {
543	if (CompleteObjectLocatorType)
544	return CompleteObjectLocatorType;
545	CompleteObjectLocatorType = llvm::StructType::create(
546	Context&: CGM.getLLVMContext(), Name: "rtti.CompleteObjectLocator");
547	llvm::Type *FieldTypes[] = {
548	CGM.IntTy,
549	CGM.IntTy,
550	CGM.IntTy,
551	getImageRelativeType(PtrType: CGM.Int8PtrTy),
552	getImageRelativeType(PtrType: getClassHierarchyDescriptorType()->getPointerTo()),
553	getImageRelativeType(PtrType: CompleteObjectLocatorType),
554	};
555	llvm::ArrayRef<llvm::Type *> FieldTypesRef(FieldTypes);
556	if (!isImageRelative())
557	FieldTypesRef = FieldTypesRef.drop_back();
558	CompleteObjectLocatorType->setBody(Elements: FieldTypesRef);
559	return CompleteObjectLocatorType;
560	}
561
562	llvm::GlobalVariable *getImageBase() {
563	StringRef Name = "__ImageBase";
564	if (llvm::GlobalVariable *GV = CGM.getModule().getNamedGlobal(Name))
565	return GV;
566
567	auto *GV = new llvm::GlobalVariable(CGM.getModule(), CGM.Int8Ty,
568	/*isConstant=*/true,
569	llvm::GlobalValue::ExternalLinkage,
570	/*Initializer=*/nullptr, Name);
571	CGM.setDSOLocal(GV);
572	return GV;
573	}
574
575	llvm::Constant *getImageRelativeConstant(llvm::Constant *PtrVal) {
576	if (!isImageRelative())
577	return PtrVal;
578
579	if (PtrVal->isNullValue())
580	return llvm::Constant::getNullValue(Ty: CGM.IntTy);
581
582	llvm::Constant *ImageBaseAsInt =
583	llvm::ConstantExpr::getPtrToInt(C: getImageBase(), Ty: CGM.IntPtrTy);
584	llvm::Constant *PtrValAsInt =
585	llvm::ConstantExpr::getPtrToInt(C: PtrVal, Ty: CGM.IntPtrTy);
586	llvm::Constant *Diff =
587	llvm::ConstantExpr::getSub(C1: PtrValAsInt, C2: ImageBaseAsInt,
588	/*HasNUW=*/true, /*HasNSW=*/true);
589	return llvm::ConstantExpr::getTrunc(C: Diff, Ty: CGM.IntTy);
590	}
591
592	private:
593	MicrosoftMangleContext &getMangleContext() {
594	return cast<MicrosoftMangleContext>(Val&: CodeGen::CGCXXABI::getMangleContext());
595	}
596
597	llvm::Constant *getZeroInt() {
598	return llvm::ConstantInt::get(Ty: CGM.IntTy, V: 0);
599	}
600
601	llvm::Constant *getAllOnesInt() {
602	return llvm::Constant::getAllOnesValue(Ty: CGM.IntTy);
603	}
604
605	CharUnits getVirtualFunctionPrologueThisAdjustment(GlobalDecl GD) override;
606
607	void
608	GetNullMemberPointerFields(const MemberPointerType *MPT,
609	llvm::SmallVectorImpl<llvm::Constant *> &fields);
610
611	/// Shared code for virtual base adjustment. Returns the offset from
612	/// the vbptr to the virtual base. Optionally returns the address of the
613	/// vbptr itself.
614	llvm::Value *GetVBaseOffsetFromVBPtr(CodeGenFunction &CGF,
615	Address Base,
616	llvm::Value *VBPtrOffset,
617	llvm::Value *VBTableOffset,
618	llvm::Value **VBPtr = nullptr);
619
620	llvm::Value *GetVBaseOffsetFromVBPtr(CodeGenFunction &CGF,
621	Address Base,
622	int32_t VBPtrOffset,
623	int32_t VBTableOffset,
624	llvm::Value **VBPtr = nullptr) {
625	assert(VBTableOffset % 4 == 0 && "should be byte offset into table of i32s");
626	llvm::Value *VBPOffset = llvm::ConstantInt::get(Ty: CGM.IntTy, V: VBPtrOffset),
627	*VBTOffset = llvm::ConstantInt::get(Ty: CGM.IntTy, V: VBTableOffset);
628	return GetVBaseOffsetFromVBPtr(CGF, Base, VBPtrOffset: VBPOffset, VBTableOffset: VBTOffset, VBPtr);
629	}
630
631	std::tuple<Address, llvm::Value *, const CXXRecordDecl *>
632	performBaseAdjustment(CodeGenFunction &CGF, Address Value,
633	QualType SrcRecordTy);
634
635	/// Performs a full virtual base adjustment. Used to dereference
636	/// pointers to members of virtual bases.
637	llvm::Value *AdjustVirtualBase(CodeGenFunction &CGF, const Expr *E,
638	const CXXRecordDecl *RD, Address Base,
639	llvm::Value *VirtualBaseAdjustmentOffset,
640	llvm::Value *VBPtrOffset /* optional */);
641
642	/// Emits a full member pointer with the fields common to data and
643	/// function member pointers.
644	llvm::Constant *EmitFullMemberPointer(llvm::Constant *FirstField,
645	bool IsMemberFunction,
646	const CXXRecordDecl *RD,
647	CharUnits NonVirtualBaseAdjustment,
648	unsigned VBTableIndex);
649
650	bool MemberPointerConstantIsNull(const MemberPointerType *MPT,
651	llvm::Constant *MP);
652
653	/// - Initialize all vbptrs of 'this' with RD as the complete type.
654	void EmitVBPtrStores(CodeGenFunction &CGF, const CXXRecordDecl *RD);
655
656	/// Caching wrapper around VBTableBuilder::enumerateVBTables().
657	const VBTableGlobals &enumerateVBTables(const CXXRecordDecl *RD);
658
659	/// Generate a thunk for calling a virtual member function MD.
660	llvm::Function *EmitVirtualMemPtrThunk(const CXXMethodDecl *MD,
661	const MethodVFTableLocation &ML);
662
663	llvm::Constant *EmitMemberDataPointer(const CXXRecordDecl *RD,
664	CharUnits offset);
665
666	public:
667	llvm::Type *ConvertMemberPointerType(const MemberPointerType *MPT) override;
668
669	bool isZeroInitializable(const MemberPointerType *MPT) override;
670
671	bool isMemberPointerConvertible(const MemberPointerType *MPT) const override {
672	const CXXRecordDecl *RD = MPT->getMostRecentCXXRecordDecl();
673	return RD->hasAttr<MSInheritanceAttr>();
674	}
675
676	llvm::Constant *EmitNullMemberPointer(const MemberPointerType *MPT) override;
677
678	llvm::Constant *EmitMemberDataPointer(const MemberPointerType *MPT,
679	CharUnits offset) override;
680	llvm::Constant *EmitMemberFunctionPointer(const CXXMethodDecl *MD) override;
681	llvm::Constant *EmitMemberPointer(const APValue &MP, QualType MPT) override;
682
683	llvm::Value *EmitMemberPointerComparison(CodeGenFunction &CGF,
684	llvm::Value *L,
685	llvm::Value *R,
686	const MemberPointerType *MPT,
687	bool Inequality) override;
688
689	llvm::Value *EmitMemberPointerIsNotNull(CodeGenFunction &CGF,
690	llvm::Value *MemPtr,
691	const MemberPointerType *MPT) override;
692
693	llvm::Value *
694	EmitMemberDataPointerAddress(CodeGenFunction &CGF, const Expr *E,
695	Address Base, llvm::Value *MemPtr,
696	const MemberPointerType *MPT) override;
697
698	llvm::Value *EmitNonNullMemberPointerConversion(
699	const MemberPointerType *SrcTy, const MemberPointerType *DstTy,
700	CastKind CK, CastExpr::path_const_iterator PathBegin,
701	CastExpr::path_const_iterator PathEnd, llvm::Value *Src,
702	CGBuilderTy &Builder);
703
704	llvm::Value *EmitMemberPointerConversion(CodeGenFunction &CGF,
705	const CastExpr *E,
706	llvm::Value *Src) override;
707
708	llvm::Constant *EmitMemberPointerConversion(const CastExpr *E,
709	llvm::Constant *Src) override;
710
711	llvm::Constant *EmitMemberPointerConversion(
712	const MemberPointerType *SrcTy, const MemberPointerType *DstTy,
713	CastKind CK, CastExpr::path_const_iterator PathBegin,
714	CastExpr::path_const_iterator PathEnd, llvm::Constant *Src);
715
716	CGCallee
717	EmitLoadOfMemberFunctionPointer(CodeGenFunction &CGF, const Expr *E,
718	Address This, llvm::Value *&ThisPtrForCall,
719	llvm::Value *MemPtr,
720	const MemberPointerType *MPT) override;
721
722	void emitCXXStructor(GlobalDecl GD) override;
723
724	llvm::StructType *getCatchableTypeType() {
725	if (CatchableTypeType)
726	return CatchableTypeType;
727	llvm::Type *FieldTypes[] = {
728	CGM.IntTy, // Flags
729	getImageRelativeType(PtrType: CGM.Int8PtrTy), // TypeDescriptor
730	CGM.IntTy, // NonVirtualAdjustment
731	CGM.IntTy, // OffsetToVBPtr
732	CGM.IntTy, // VBTableIndex
733	CGM.IntTy, // Size
734	getImageRelativeType(PtrType: CGM.Int8PtrTy) // CopyCtor
735	};
736	CatchableTypeType = llvm::StructType::create(
737	Context&: CGM.getLLVMContext(), Elements: FieldTypes, Name: "eh.CatchableType");
738	return CatchableTypeType;
739	}
740
741	llvm::StructType *getCatchableTypeArrayType(uint32_t NumEntries) {
742	llvm::StructType *&CatchableTypeArrayType =
743	CatchableTypeArrayTypeMap[NumEntries];
744	if (CatchableTypeArrayType)
745	return CatchableTypeArrayType;
746
747	llvm::SmallString<23> CTATypeName("eh.CatchableTypeArray.");
748	CTATypeName += llvm::utostr(X: NumEntries);
749	llvm::Type *CTType =
750	getImageRelativeType(PtrType: getCatchableTypeType()->getPointerTo());
751	llvm::Type *FieldTypes[] = {
752	CGM.IntTy, // NumEntries
753	llvm::ArrayType::get(ElementType: CTType, NumElements: NumEntries) // CatchableTypes
754	};
755	CatchableTypeArrayType =
756	llvm::StructType::create(Context&: CGM.getLLVMContext(), Elements: FieldTypes, Name: CTATypeName);
757	return CatchableTypeArrayType;
758	}
759
760	llvm::StructType *getThrowInfoType() {
761	if (ThrowInfoType)
762	return ThrowInfoType;
763	llvm::Type *FieldTypes[] = {
764	CGM.IntTy, // Flags
765	getImageRelativeType(PtrType: CGM.Int8PtrTy), // CleanupFn
766	getImageRelativeType(PtrType: CGM.Int8PtrTy), // ForwardCompat
767	getImageRelativeType(PtrType: CGM.Int8PtrTy) // CatchableTypeArray
768	};
769	ThrowInfoType = llvm::StructType::create(Context&: CGM.getLLVMContext(), Elements: FieldTypes,
770	Name: "eh.ThrowInfo");
771	return ThrowInfoType;
772	}
773
774	llvm::FunctionCallee getThrowFn() {
775	// _CxxThrowException is passed an exception object and a ThrowInfo object
776	// which describes the exception.
777	llvm::Type *Args[] = {CGM.Int8PtrTy, getThrowInfoType()->getPointerTo()};
778	llvm::FunctionType *FTy =
779	llvm::FunctionType::get(Result: CGM.VoidTy, Params: Args, /*isVarArg=*/false);
780	llvm::FunctionCallee Throw =
781	CGM.CreateRuntimeFunction(Ty: FTy, Name: "_CxxThrowException");
782	// _CxxThrowException is stdcall on 32-bit x86 platforms.
783	if (CGM.getTarget().getTriple().getArch() == llvm::Triple::x86) {
784	if (auto *Fn = dyn_cast<llvm::Function>(Val: Throw.getCallee()))
785	Fn->setCallingConv(llvm::CallingConv::X86_StdCall);
786	}
787	return Throw;
788	}
789
790	llvm::Function *getAddrOfCXXCtorClosure(const CXXConstructorDecl *CD,
791	CXXCtorType CT);
792
793	llvm::Constant *getCatchableType(QualType T,
794	uint32_t NVOffset = 0,
795	int32_t VBPtrOffset = -1,
796	uint32_t VBIndex = 0);
797
798	llvm::GlobalVariable *getCatchableTypeArray(QualType T);
799
800	llvm::GlobalVariable *getThrowInfo(QualType T) override;
801
802	std::pair<llvm::Value *, const CXXRecordDecl *>
803	LoadVTablePtr(CodeGenFunction &CGF, Address This,
804	const CXXRecordDecl *RD) override;
805
806	bool
807	isPermittedToBeHomogeneousAggregate(const CXXRecordDecl *RD) const override;
808
809	private:
810	typedef std::pair<const CXXRecordDecl *, CharUnits> VFTableIdTy;
811	typedef llvm::DenseMap<VFTableIdTy, llvm::GlobalVariable *> VTablesMapTy;
812	typedef llvm::DenseMap<VFTableIdTy, llvm::GlobalValue *> VFTablesMapTy;
813	/// All the vftables that have been referenced.
814	VFTablesMapTy VFTablesMap;
815	VTablesMapTy VTablesMap;
816
817	/// This set holds the record decls we've deferred vtable emission for.
818	llvm::SmallPtrSet<const CXXRecordDecl *, 4> DeferredVFTables;
819
820
821	/// All the vbtables which have been referenced.
822	llvm::DenseMap<const CXXRecordDecl *, VBTableGlobals> VBTablesMap;
823
824	/// Info on the global variable used to guard initialization of static locals.
825	/// The BitIndex field is only used for externally invisible declarations.
826	struct GuardInfo {
827	GuardInfo() = default;
828	llvm::GlobalVariable *Guard = nullptr;
829	unsigned BitIndex = 0;
830	};
831
832	/// Map from DeclContext to the current guard variable. We assume that the
833	/// AST is visited in source code order.
834	llvm::DenseMap<const DeclContext *, GuardInfo> GuardVariableMap;
835	llvm::DenseMap<const DeclContext *, GuardInfo> ThreadLocalGuardVariableMap;
836	llvm::DenseMap<const DeclContext *, unsigned> ThreadSafeGuardNumMap;
837
838	llvm::DenseMap<size_t, llvm::StructType *> TypeDescriptorTypeMap;
839	llvm::StructType *BaseClassDescriptorType;
840	llvm::StructType *ClassHierarchyDescriptorType;
841	llvm::StructType *CompleteObjectLocatorType;
842
843	llvm::DenseMap<QualType, llvm::GlobalVariable *> CatchableTypeArrays;
844
845	llvm::StructType *CatchableTypeType;
846	llvm::DenseMap<uint32_t, llvm::StructType *> CatchableTypeArrayTypeMap;
847	llvm::StructType *ThrowInfoType;
848	};
849
850	}
851
852	CGCXXABI::RecordArgABI
853	MicrosoftCXXABI::getRecordArgABI(const CXXRecordDecl *RD) const {
854	// Use the default C calling convention rules for things that can be passed in
855	// registers, i.e. non-trivially copyable records or records marked with
856	// [[trivial_abi]].
857	if (RD->canPassInRegisters())
858	return RAA_Default;
859
860	switch (CGM.getTarget().getTriple().getArch()) {
861	default:
862	// FIXME: Implement for other architectures.
863	return RAA_Indirect;
864
865	case llvm::Triple::thumb:
866	// Pass things indirectly for now because it is simple.
867	// FIXME: This is incompatible with MSVC for arguments with a dtor and no
868	// copy ctor.
869	return RAA_Indirect;
870
871	case llvm::Triple::x86: {
872	// If the argument has *required* alignment greater than four bytes, pass
873	// it indirectly. Prior to MSVC version 19.14, passing overaligned
874	// arguments was not supported and resulted in a compiler error. In 19.14
875	// and later versions, such arguments are now passed indirectly.
876	TypeInfo Info = getContext().getTypeInfo(RD->getTypeForDecl());
877	if (Info.isAlignRequired() && Info.Align > 4)
878	return RAA_Indirect;
879
880	// If C++ prohibits us from making a copy, construct the arguments directly
881	// into argument memory.
882	return RAA_DirectInMemory;
883	}
884
885	case llvm::Triple::x86_64:
886	case llvm::Triple::aarch64:
887	return RAA_Indirect;
888	}
889
890	llvm_unreachable("invalid enum");
891	}
892
893	void MicrosoftCXXABI::emitVirtualObjectDelete(CodeGenFunction &CGF,
894	const CXXDeleteExpr *DE,
895	Address Ptr,
896	QualType ElementType,
897	const CXXDestructorDecl *Dtor) {
898	// FIXME: Provide a source location here even though there's no
899	// CXXMemberCallExpr for dtor call.
900	bool UseGlobalDelete = DE->isGlobalDelete();
901	CXXDtorType DtorType = UseGlobalDelete ? Dtor_Complete : Dtor_Deleting;
902	llvm::Value *MDThis = EmitVirtualDestructorCall(CGF, Dtor, DtorType, This: Ptr, E: DE);
903	if (UseGlobalDelete)
904	CGF.EmitDeleteCall(DeleteFD: DE->getOperatorDelete(), Ptr: MDThis, DeleteTy: ElementType);
905	}
906
907	void MicrosoftCXXABI::emitRethrow(CodeGenFunction &CGF, bool isNoReturn) {
908	llvm::Value *Args[] = {
909	llvm::ConstantPointerNull::get(T: CGM.Int8PtrTy),
910	llvm::ConstantPointerNull::get(T: getThrowInfoType()->getPointerTo())};
911	llvm::FunctionCallee Fn = getThrowFn();
912	if (isNoReturn)
913	CGF.EmitNoreturnRuntimeCallOrInvoke(callee: Fn, args: Args);
914	else
915	CGF.EmitRuntimeCallOrInvoke(callee: Fn, args: Args);
916	}
917
918	void MicrosoftCXXABI::emitBeginCatch(CodeGenFunction &CGF,
919	const CXXCatchStmt *S) {
920	// In the MS ABI, the runtime handles the copy, and the catch handler is
921	// responsible for destruction.
922	VarDecl *CatchParam = S->getExceptionDecl();
923	llvm::BasicBlock *CatchPadBB = CGF.Builder.GetInsertBlock();
924	llvm::CatchPadInst *CPI =
925	cast<llvm::CatchPadInst>(Val: CatchPadBB->getFirstNonPHI());
926	CGF.CurrentFuncletPad = CPI;
927
928	// If this is a catch-all or the catch parameter is unnamed, we don't need to
929	// emit an alloca to the object.
930	if (!CatchParam || !CatchParam->getDeclName()) {
931	CGF.EHStack.pushCleanup<CatchRetScope>(Kind: NormalCleanup, A: CPI);
932	return;
933	}
934
935	CodeGenFunction::AutoVarEmission var = CGF.EmitAutoVarAlloca(var: *CatchParam);
936	CPI->setArgOperand(i: 2, v: var.getObjectAddress(CGF).emitRawPointer(CGF));
937	CGF.EHStack.pushCleanup<CatchRetScope>(Kind: NormalCleanup, A: CPI);
938	CGF.EmitAutoVarCleanups(emission: var);
939	}
940
941	/// We need to perform a generic polymorphic operation (like a typeid
942	/// or a cast), which requires an object with a vfptr. Adjust the
943	/// address to point to an object with a vfptr.
944	std::tuple<Address, llvm::Value *, const CXXRecordDecl *>
945	MicrosoftCXXABI::performBaseAdjustment(CodeGenFunction &CGF, Address Value,
946	QualType SrcRecordTy) {
947	Value = Value.withElementType(ElemTy: CGF.Int8Ty);
948	const CXXRecordDecl *SrcDecl = SrcRecordTy->getAsCXXRecordDecl();
949	const ASTContext &Context = getContext();
950
951	// If the class itself has a vfptr, great. This check implicitly
952	// covers non-virtual base subobjects: a class with its own virtual
953	// functions would be a candidate to be a primary base.
954	if (Context.getASTRecordLayout(SrcDecl).hasExtendableVFPtr())
955	return std::make_tuple(args&: Value, args: llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: 0),
956	args&: SrcDecl);
957
958	// Okay, one of the vbases must have a vfptr, or else this isn't
959	// actually a polymorphic class.
960	const CXXRecordDecl *PolymorphicBase = nullptr;
961	for (auto &Base : SrcDecl->vbases()) {
962	const CXXRecordDecl *BaseDecl = Base.getType()->getAsCXXRecordDecl();
963	if (Context.getASTRecordLayout(BaseDecl).hasExtendableVFPtr()) {
964	PolymorphicBase = BaseDecl;
965	break;
966	}
967	}
968	assert(PolymorphicBase && "polymorphic class has no apparent vfptr?");
969
970	llvm::Value *Offset =
971	GetVirtualBaseClassOffset(CGF, This: Value, ClassDecl: SrcDecl, BaseClassDecl: PolymorphicBase);
972	llvm::Value *Ptr = CGF.Builder.CreateInBoundsGEP(
973	Ty: Value.getElementType(), Ptr: Value.emitRawPointer(CGF), IdxList: Offset);
974	CharUnits VBaseAlign =
975	CGF.CGM.getVBaseAlignment(DerivedAlign: Value.getAlignment(), Derived: SrcDecl, VBase: PolymorphicBase);
976	return std::make_tuple(args: Address(Ptr, CGF.Int8Ty, VBaseAlign), args&: Offset,
977	args&: PolymorphicBase);
978	}
979
980	bool MicrosoftCXXABI::shouldTypeidBeNullChecked(bool IsDeref,
981	QualType SrcRecordTy) {
982	const CXXRecordDecl *SrcDecl = SrcRecordTy->getAsCXXRecordDecl();
983	return IsDeref &&
984	!getContext().getASTRecordLayout(SrcDecl).hasExtendableVFPtr();
985	}
986
987	static llvm::CallBase *emitRTtypeidCall(CodeGenFunction &CGF,
988	llvm::Value *Argument) {
989	llvm::Type *ArgTypes[] = {CGF.Int8PtrTy};
990	llvm::FunctionType *FTy =
991	llvm::FunctionType::get(Result: CGF.Int8PtrTy, Params: ArgTypes, isVarArg: false);
992	llvm::Value *Args[] = {Argument};
993	llvm::FunctionCallee Fn = CGF.CGM.CreateRuntimeFunction(Ty: FTy, Name: "__RTtypeid");
994	return CGF.EmitRuntimeCallOrInvoke(callee: Fn, args: Args);
995	}
996
997	void MicrosoftCXXABI::EmitBadTypeidCall(CodeGenFunction &CGF) {
998	llvm::CallBase *Call =
999	emitRTtypeidCall(CGF, Argument: llvm::Constant::getNullValue(Ty: CGM.VoidPtrTy));
1000	Call->setDoesNotReturn();
1001	CGF.Builder.CreateUnreachable();
1002	}
1003
1004	llvm::Value *MicrosoftCXXABI::EmitTypeid(CodeGenFunction &CGF,
1005	QualType SrcRecordTy,
1006	Address ThisPtr,
1007	llvm::Type *StdTypeInfoPtrTy) {
1008	std::tie(args&: ThisPtr, args: std::ignore, args: std::ignore) =
1009	performBaseAdjustment(CGF, Value: ThisPtr, SrcRecordTy);
1010	llvm::CallBase *Typeid = emitRTtypeidCall(CGF, Argument: ThisPtr.emitRawPointer(CGF));
1011	return CGF.Builder.CreateBitCast(V: Typeid, DestTy: StdTypeInfoPtrTy);
1012	}
1013
1014	bool MicrosoftCXXABI::shouldDynamicCastCallBeNullChecked(bool SrcIsPtr,
1015	QualType SrcRecordTy) {
1016	const CXXRecordDecl *SrcDecl = SrcRecordTy->getAsCXXRecordDecl();
1017	return SrcIsPtr &&
1018	!getContext().getASTRecordLayout(SrcDecl).hasExtendableVFPtr();
1019	}
1020
1021	llvm::Value *MicrosoftCXXABI::emitDynamicCastCall(
1022	CodeGenFunction &CGF, Address This, QualType SrcRecordTy, QualType DestTy,
1023	QualType DestRecordTy, llvm::BasicBlock *CastEnd) {
1024	llvm::Value *SrcRTTI =
1025	CGF.CGM.GetAddrOfRTTIDescriptor(Ty: SrcRecordTy.getUnqualifiedType());
1026	llvm::Value *DestRTTI =
1027	CGF.CGM.GetAddrOfRTTIDescriptor(Ty: DestRecordTy.getUnqualifiedType());
1028
1029	llvm::Value *Offset;
1030	std::tie(args&: This, args&: Offset, args: std::ignore) =
1031	performBaseAdjustment(CGF, Value: This, SrcRecordTy);
1032	llvm::Value *ThisPtr = This.emitRawPointer(CGF);
1033	Offset = CGF.Builder.CreateTrunc(V: Offset, DestTy: CGF.Int32Ty);
1034
1035	// PVOID __RTDynamicCast(
1036	// PVOID inptr,
1037	// LONG VfDelta,
1038	// PVOID SrcType,
1039	// PVOID TargetType,
1040	// BOOL isReference)
1041	llvm::Type *ArgTypes[] = {CGF.Int8PtrTy, CGF.Int32Ty, CGF.Int8PtrTy,
1042	CGF.Int8PtrTy, CGF.Int32Ty};
1043	llvm::FunctionCallee Function = CGF.CGM.CreateRuntimeFunction(
1044	Ty: llvm::FunctionType::get(Result: CGF.Int8PtrTy, Params: ArgTypes, isVarArg: false),
1045	Name: "__RTDynamicCast");
1046	llvm::Value *Args[] = {
1047	ThisPtr, Offset, SrcRTTI, DestRTTI,
1048	llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: DestTy->isReferenceType())};
1049	return CGF.EmitRuntimeCallOrInvoke(callee: Function, args: Args);
1050	}
1051
1052	llvm::Value *MicrosoftCXXABI::emitDynamicCastToVoid(CodeGenFunction &CGF,
1053	Address Value,
1054	QualType SrcRecordTy) {
1055	std::tie(args&: Value, args: std::ignore, args: std::ignore) =
1056	performBaseAdjustment(CGF, Value, SrcRecordTy);
1057
1058	// PVOID __RTCastToVoid(
1059	// PVOID inptr)
1060	llvm::Type *ArgTypes[] = {CGF.Int8PtrTy};
1061	llvm::FunctionCallee Function = CGF.CGM.CreateRuntimeFunction(
1062	Ty: llvm::FunctionType::get(Result: CGF.Int8PtrTy, Params: ArgTypes, isVarArg: false),
1063	Name: "__RTCastToVoid");
1064	llvm::Value *Args[] = {Value.emitRawPointer(CGF)};
1065	return CGF.EmitRuntimeCall(callee: Function, args: Args);
1066	}
1067
1068	bool MicrosoftCXXABI::EmitBadCastCall(CodeGenFunction &CGF) {
1069	return false;
1070	}
1071
1072	llvm::Value *MicrosoftCXXABI::GetVirtualBaseClassOffset(
1073	CodeGenFunction &CGF, Address This, const CXXRecordDecl *ClassDecl,
1074	const CXXRecordDecl *BaseClassDecl) {
1075	const ASTContext &Context = getContext();
1076	int64_t VBPtrChars =
1077	Context.getASTRecordLayout(ClassDecl).getVBPtrOffset().getQuantity();
1078	llvm::Value *VBPtrOffset = llvm::ConstantInt::get(Ty: CGM.PtrDiffTy, V: VBPtrChars);
1079	CharUnits IntSize = Context.getTypeSizeInChars(Context.IntTy);
1080	CharUnits VBTableChars =
1081	IntSize *
1082	CGM.getMicrosoftVTableContext().getVBTableIndex(Derived: ClassDecl, VBase: BaseClassDecl);
1083	llvm::Value *VBTableOffset =
1084	llvm::ConstantInt::get(Ty: CGM.IntTy, V: VBTableChars.getQuantity());
1085
1086	llvm::Value *VBPtrToNewBase =
1087	GetVBaseOffsetFromVBPtr(CGF, Base: This, VBPtrOffset, VBTableOffset);
1088	VBPtrToNewBase =
1089	CGF.Builder.CreateSExtOrBitCast(V: VBPtrToNewBase, DestTy: CGM.PtrDiffTy);
1090	return CGF.Builder.CreateNSWAdd(LHS: VBPtrOffset, RHS: VBPtrToNewBase);
1091	}
1092
1093	bool MicrosoftCXXABI::HasThisReturn(GlobalDecl GD) const {
1094	return isa<CXXConstructorDecl>(Val: GD.getDecl());
1095	}
1096
1097	static bool isDeletingDtor(GlobalDecl GD) {
1098	return isa<CXXDestructorDecl>(Val: GD.getDecl()) &&
1099	GD.getDtorType() == Dtor_Deleting;
1100	}
1101
1102	bool MicrosoftCXXABI::hasMostDerivedReturn(GlobalDecl GD) const {
1103	return isDeletingDtor(GD);
1104	}
1105
1106	static bool isTrivialForMSVC(const CXXRecordDecl *RD, QualType Ty,
1107	CodeGenModule &CGM) {
1108	// On AArch64, HVAs that can be passed in registers can also be returned
1109	// in registers. (Note this is using the MSVC definition of an HVA; see
1110	// isPermittedToBeHomogeneousAggregate().)
1111	const Type *Base = nullptr;
1112	uint64_t NumElts = 0;
1113	if (CGM.getTarget().getTriple().isAArch64() &&
1114	CGM.getTypes().getABIInfo().isHomogeneousAggregate(Ty, Base, Members&: NumElts) &&
1115	isa<VectorType>(Val: Base)) {
1116	return true;
1117	}
1118
1119	// We use the C++14 definition of an aggregate, so we also
1120	// check for:
1121	// No private or protected non static data members.
1122	// No base classes
1123	// No virtual functions
1124	// Additionally, we need to ensure that there is a trivial copy assignment
1125	// operator, a trivial destructor and no user-provided constructors.
1126	if (RD->hasProtectedFields() || RD->hasPrivateFields())
1127	return false;
1128	if (RD->getNumBases() > 0)
1129	return false;
1130	if (RD->isPolymorphic())
1131	return false;
1132	if (RD->hasNonTrivialCopyAssignment())
1133	return false;
1134	for (const CXXConstructorDecl *Ctor : RD->ctors())
1135	if (Ctor->isUserProvided())
1136	return false;
1137	if (RD->hasNonTrivialDestructor())
1138	return false;
1139	return true;
1140	}
1141
1142	bool MicrosoftCXXABI::classifyReturnType(CGFunctionInfo &FI) const {
1143	const CXXRecordDecl *RD = FI.getReturnType()->getAsCXXRecordDecl();
1144	if (!RD)
1145	return false;
1146
1147	bool isTrivialForABI = RD->canPassInRegisters() &&
1148	isTrivialForMSVC(RD, Ty: FI.getReturnType(), CGM);
1149
1150	// MSVC always returns structs indirectly from C++ instance methods.
1151	bool isIndirectReturn = !isTrivialForABI || FI.isInstanceMethod();
1152
1153	if (isIndirectReturn) {
1154	CharUnits Align = CGM.getContext().getTypeAlignInChars(T: FI.getReturnType());
1155	FI.getReturnInfo() = ABIArgInfo::getIndirect(Alignment: Align, /*ByVal=*/false);
1156
1157	// MSVC always passes `this` before the `sret` parameter.
1158	FI.getReturnInfo().setSRetAfterThis(FI.isInstanceMethod());
1159
1160	// On AArch64, use the `inreg` attribute if the object is considered to not
1161	// be trivially copyable, or if this is an instance method struct return.
1162	FI.getReturnInfo().setInReg(CGM.getTarget().getTriple().isAArch64());
1163
1164	return true;
1165	}
1166
1167	// Otherwise, use the C ABI rules.
1168	return false;
1169	}
1170
1171	llvm::BasicBlock *
1172	MicrosoftCXXABI::EmitCtorCompleteObjectHandler(CodeGenFunction &CGF,
1173	const CXXRecordDecl *RD) {
1174	llvm::Value *IsMostDerivedClass = getStructorImplicitParamValue(CGF);
1175	assert(IsMostDerivedClass &&
1176	"ctor for a class with virtual bases must have an implicit parameter");
1177	llvm::Value *IsCompleteObject =
1178	CGF.Builder.CreateIsNotNull(Arg: IsMostDerivedClass, Name: "is_complete_object");
1179
1180	llvm::BasicBlock *CallVbaseCtorsBB = CGF.createBasicBlock(name: "ctor.init_vbases");
1181	llvm::BasicBlock *SkipVbaseCtorsBB = CGF.createBasicBlock(name: "ctor.skip_vbases");
1182	CGF.Builder.CreateCondBr(Cond: IsCompleteObject,
1183	True: CallVbaseCtorsBB, False: SkipVbaseCtorsBB);
1184
1185	CGF.EmitBlock(BB: CallVbaseCtorsBB);
1186
1187	// Fill in the vbtable pointers here.
1188	EmitVBPtrStores(CGF, RD);
1189
1190	// CGF will put the base ctor calls in this basic block for us later.
1191
1192	return SkipVbaseCtorsBB;
1193	}
1194
1195	llvm::BasicBlock *
1196	MicrosoftCXXABI::EmitDtorCompleteObjectHandler(CodeGenFunction &CGF) {
1197	llvm::Value *IsMostDerivedClass = getStructorImplicitParamValue(CGF);
1198	assert(IsMostDerivedClass &&
1199	"ctor for a class with virtual bases must have an implicit parameter");
1200	llvm::Value *IsCompleteObject =
1201	CGF.Builder.CreateIsNotNull(Arg: IsMostDerivedClass, Name: "is_complete_object");
1202
1203	llvm::BasicBlock *CallVbaseDtorsBB = CGF.createBasicBlock(name: "Dtor.dtor_vbases");
1204	llvm::BasicBlock *SkipVbaseDtorsBB = CGF.createBasicBlock(name: "Dtor.skip_vbases");
1205	CGF.Builder.CreateCondBr(Cond: IsCompleteObject,
1206	True: CallVbaseDtorsBB, False: SkipVbaseDtorsBB);
1207
1208	CGF.EmitBlock(BB: CallVbaseDtorsBB);
1209	// CGF will put the base dtor calls in this basic block for us later.
1210
1211	return SkipVbaseDtorsBB;
1212	}
1213
1214	void MicrosoftCXXABI::initializeHiddenVirtualInheritanceMembers(
1215	CodeGenFunction &CGF, const CXXRecordDecl *RD) {
1216	// In most cases, an override for a vbase virtual method can adjust
1217	// the "this" parameter by applying a constant offset.
1218	// However, this is not enough while a constructor or a destructor of some
1219	// class X is being executed if all the following conditions are met:
1220	// - X has virtual bases, (1)
1221	// - X overrides a virtual method M of a vbase Y, (2)
1222	// - X itself is a vbase of the most derived class.
1223	//
1224	// If (1) and (2) are true, the vtorDisp for vbase Y is a hidden member of X
1225	// which holds the extra amount of "this" adjustment we must do when we use
1226	// the X vftables (i.e. during X ctor or dtor).
1227	// Outside the ctors and dtors, the values of vtorDisps are zero.
1228
1229	const ASTRecordLayout &Layout = getContext().getASTRecordLayout(RD);
1230	typedef ASTRecordLayout::VBaseOffsetsMapTy VBOffsets;
1231	const VBOffsets &VBaseMap = Layout.getVBaseOffsetsMap();
1232	CGBuilderTy &Builder = CGF.Builder;
1233
1234	llvm::Value *Int8This = nullptr; // Initialize lazily.
1235
1236	for (const CXXBaseSpecifier &S : RD->vbases()) {
1237	const CXXRecordDecl *VBase = S.getType()->getAsCXXRecordDecl();
1238	auto I = VBaseMap.find(Val: VBase);
1239	assert(I != VBaseMap.end());
1240	if (!I->second.hasVtorDisp())
1241	continue;
1242
1243	llvm::Value *VBaseOffset =
1244	GetVirtualBaseClassOffset(CGF, This: getThisAddress(CGF), ClassDecl: RD, BaseClassDecl: VBase);
1245	uint64_t ConstantVBaseOffset = I->second.VBaseOffset.getQuantity();
1246
1247	// vtorDisp_for_vbase = vbptr[vbase_idx] - offsetof(RD, vbase).
1248	llvm::Value *VtorDispValue = Builder.CreateSub(
1249	LHS: VBaseOffset, RHS: llvm::ConstantInt::get(Ty: CGM.PtrDiffTy, V: ConstantVBaseOffset),
1250	Name: "vtordisp.value");
1251	VtorDispValue = Builder.CreateTruncOrBitCast(V: VtorDispValue, DestTy: CGF.Int32Ty);
1252
1253	if (!Int8This)
1254	Int8This = getThisValue(CGF);
1255
1256	llvm::Value *VtorDispPtr =
1257	Builder.CreateInBoundsGEP(Ty: CGF.Int8Ty, Ptr: Int8This, IdxList: VBaseOffset);
1258	// vtorDisp is always the 32-bits before the vbase in the class layout.
1259	VtorDispPtr = Builder.CreateConstGEP1_32(Ty: CGF.Int8Ty, Ptr: VtorDispPtr, Idx0: -4);
1260
1261	Builder.CreateAlignedStore(Val: VtorDispValue, Addr: VtorDispPtr,
1262	Align: CharUnits::fromQuantity(Quantity: 4));
1263	}
1264	}
1265
1266	static bool hasDefaultCXXMethodCC(ASTContext &Context,
1267	const CXXMethodDecl *MD) {
1268	CallingConv ExpectedCallingConv = Context.getDefaultCallingConvention(
1269	/*IsVariadic=*/false, /*IsCXXMethod=*/true);
1270	CallingConv ActualCallingConv =
1271	MD->getType()->castAs<FunctionProtoType>()->getCallConv();
1272	return ExpectedCallingConv == ActualCallingConv;
1273	}
1274
1275	void MicrosoftCXXABI::EmitCXXConstructors(const CXXConstructorDecl *D) {
1276	// There's only one constructor type in this ABI.
1277	CGM.EmitGlobal(D: GlobalDecl(D, Ctor_Complete));
1278
1279	// Exported default constructors either have a simple call-site where they use
1280	// the typical calling convention and have a single 'this' pointer for an
1281	// argument -or- they get a wrapper function which appropriately thunks to the
1282	// real default constructor. This thunk is the default constructor closure.
1283	if (D->hasAttr<DLLExportAttr>() && D->isDefaultConstructor() &&
1284	D->isDefined()) {
1285	if (!hasDefaultCXXMethodCC(getContext(), D) || D->getNumParams() != 0) {
1286	llvm::Function *Fn = getAddrOfCXXCtorClosure(CD: D, CT: Ctor_DefaultClosure);
1287	Fn->setLinkage(llvm::GlobalValue::WeakODRLinkage);
1288	CGM.setGVProperties(Fn, D);
1289	}
1290	}
1291	}
1292
1293	void MicrosoftCXXABI::EmitVBPtrStores(CodeGenFunction &CGF,
1294	const CXXRecordDecl *RD) {
1295	Address This = getThisAddress(CGF);
1296	This = This.withElementType(ElemTy: CGM.Int8Ty);
1297	const ASTContext &Context = getContext();
1298	const ASTRecordLayout &Layout = Context.getASTRecordLayout(RD);
1299
1300	const VBTableGlobals &VBGlobals = enumerateVBTables(RD);
1301	for (unsigned I = 0, E = VBGlobals.VBTables->size(); I != E; ++I) {
1302	const std::unique_ptr<VPtrInfo> &VBT = (*VBGlobals.VBTables)[I];
1303	llvm::GlobalVariable *GV = VBGlobals.Globals[I];
1304	const ASTRecordLayout &SubobjectLayout =
1305	Context.getASTRecordLayout(VBT->IntroducingObject);
1306	CharUnits Offs = VBT->NonVirtualOffset;
1307	Offs += SubobjectLayout.getVBPtrOffset();
1308	if (VBT->getVBaseWithVPtr())
1309	Offs += Layout.getVBaseClassOffset(VBase: VBT->getVBaseWithVPtr());
1310	Address VBPtr = CGF.Builder.CreateConstInBoundsByteGEP(Addr: This, Offset: Offs);
1311	llvm::Value *GVPtr =
1312	CGF.Builder.CreateConstInBoundsGEP2_32(Ty: GV->getValueType(), Ptr: GV, Idx0: 0, Idx1: 0);
1313	VBPtr = VBPtr.withElementType(ElemTy: GVPtr->getType());
1314	CGF.Builder.CreateStore(Val: GVPtr, Addr: VBPtr);
1315	}
1316	}
1317
1318	CGCXXABI::AddedStructorArgCounts
1319	MicrosoftCXXABI::buildStructorSignature(GlobalDecl GD,
1320	SmallVectorImpl<CanQualType> &ArgTys) {
1321	AddedStructorArgCounts Added;
1322	// TODO: 'for base' flag
1323	if (isa<CXXDestructorDecl>(Val: GD.getDecl()) &&
1324	GD.getDtorType() == Dtor_Deleting) {
1325	// The scalar deleting destructor takes an implicit int parameter.
1326	ArgTys.push_back(Elt: getContext().IntTy);
1327	++Added.Suffix;
1328	}
1329	auto *CD = dyn_cast<CXXConstructorDecl>(Val: GD.getDecl());
1330	if (!CD)
1331	return Added;
1332
1333	// All parameters are already in place except is_most_derived, which goes
1334	// after 'this' if it's variadic and last if it's not.
1335
1336	const CXXRecordDecl *Class = CD->getParent();
1337	const FunctionProtoType *FPT = CD->getType()->castAs<FunctionProtoType>();
1338	if (Class->getNumVBases()) {
1339	if (FPT->isVariadic()) {
1340	ArgTys.insert(ArgTys.begin() + 1, getContext().IntTy);
1341	++Added.Prefix;
1342	} else {
1343	ArgTys.push_back(Elt: getContext().IntTy);
1344	++Added.Suffix;
1345	}
1346	}
1347
1348	return Added;
1349	}
1350
1351	void MicrosoftCXXABI::setCXXDestructorDLLStorage(llvm::GlobalValue *GV,
1352	const CXXDestructorDecl *Dtor,
1353	CXXDtorType DT) const {
1354	// Deleting destructor variants are never imported or exported. Give them the
1355	// default storage class.
1356	if (DT == Dtor_Deleting) {
1357	GV->setDLLStorageClass(llvm::GlobalValue::DefaultStorageClass);
1358	} else {
1359	const NamedDecl *ND = Dtor;
1360	CGM.setDLLImportDLLExport(GV, D: ND);
1361	}
1362	}
1363
1364	llvm::GlobalValue::LinkageTypes MicrosoftCXXABI::getCXXDestructorLinkage(
1365	GVALinkage Linkage, const CXXDestructorDecl *Dtor, CXXDtorType DT) const {
1366	// Internal things are always internal, regardless of attributes. After this,
1367	// we know the thunk is externally visible.
1368	if (Linkage == GVA_Internal)
1369	return llvm::GlobalValue::InternalLinkage;
1370
1371	switch (DT) {
1372	case Dtor_Base:
1373	// The base destructor most closely tracks the user-declared constructor, so
1374	// we delegate back to the normal declarator case.
1375	return CGM.getLLVMLinkageForDeclarator(Dtor, Linkage);
1376	case Dtor_Complete:
1377	// The complete destructor is like an inline function, but it may be
1378	// imported and therefore must be exported as well. This requires changing
1379	// the linkage if a DLL attribute is present.
1380	if (Dtor->hasAttr<DLLExportAttr>())
1381	return llvm::GlobalValue::WeakODRLinkage;
1382	if (Dtor->hasAttr<DLLImportAttr>())
1383	return llvm::GlobalValue::AvailableExternallyLinkage;
1384	return llvm::GlobalValue::LinkOnceODRLinkage;
1385	case Dtor_Deleting:
1386	// Deleting destructors are like inline functions. They have vague linkage
1387	// and are emitted everywhere they are used. They are internal if the class
1388	// is internal.
1389	return llvm::GlobalValue::LinkOnceODRLinkage;
1390	case Dtor_Comdat:
1391	llvm_unreachable("MS C++ ABI does not support comdat dtors");
1392	}
1393	llvm_unreachable("invalid dtor type");
1394	}
1395
1396	void MicrosoftCXXABI::EmitCXXDestructors(const CXXDestructorDecl *D) {
1397	// The TU defining a dtor is only guaranteed to emit a base destructor. All
1398	// other destructor variants are delegating thunks.
1399	CGM.EmitGlobal(D: GlobalDecl(D, Dtor_Base));
1400
1401	// If the class is dllexported, emit the complete (vbase) destructor wherever
1402	// the base dtor is emitted.
1403	// FIXME: To match MSVC, this should only be done when the class is exported
1404	// with -fdllexport-inlines enabled.
1405	if (D->getParent()->getNumVBases() > 0 && D->hasAttr<DLLExportAttr>())
1406	CGM.EmitGlobal(D: GlobalDecl(D, Dtor_Complete));
1407	}
1408
1409	CharUnits
1410	MicrosoftCXXABI::getVirtualFunctionPrologueThisAdjustment(GlobalDecl GD) {
1411	const CXXMethodDecl *MD = cast<CXXMethodDecl>(Val: GD.getDecl());
1412
1413	if (const CXXDestructorDecl *DD = dyn_cast<CXXDestructorDecl>(Val: MD)) {
1414	// Complete destructors take a pointer to the complete object as a
1415	// parameter, thus don't need this adjustment.
1416	if (GD.getDtorType() == Dtor_Complete)
1417	return CharUnits();
1418
1419	// There's no Dtor_Base in vftable but it shares the this adjustment with
1420	// the deleting one, so look it up instead.
1421	GD = GlobalDecl(DD, Dtor_Deleting);
1422	}
1423
1424	MethodVFTableLocation ML =
1425	CGM.getMicrosoftVTableContext().getMethodVFTableLocation(GD);
1426	CharUnits Adjustment = ML.VFPtrOffset;
1427
1428	// Normal virtual instance methods need to adjust from the vfptr that first
1429	// defined the virtual method to the virtual base subobject, but destructors
1430	// do not. The vector deleting destructor thunk applies this adjustment for
1431	// us if necessary.
1432	if (isa<CXXDestructorDecl>(Val: MD))
1433	Adjustment = CharUnits::Zero();
1434
1435	if (ML.VBase) {
1436	const ASTRecordLayout &DerivedLayout =
1437	getContext().getASTRecordLayout(MD->getParent());
1438	Adjustment += DerivedLayout.getVBaseClassOffset(VBase: ML.VBase);
1439	}
1440
1441	return Adjustment;
1442	}
1443
1444	Address MicrosoftCXXABI::adjustThisArgumentForVirtualFunctionCall(
1445	CodeGenFunction &CGF, GlobalDecl GD, Address This,
1446	bool VirtualCall) {
1447	if (!VirtualCall) {
1448	// If the call of a virtual function is not virtual, we just have to
1449	// compensate for the adjustment the virtual function does in its prologue.
1450	CharUnits Adjustment = getVirtualFunctionPrologueThisAdjustment(GD);
1451	if (Adjustment.isZero())
1452	return This;
1453
1454	This = This.withElementType(ElemTy: CGF.Int8Ty);
1455	assert(Adjustment.isPositive());
1456	return CGF.Builder.CreateConstByteGEP(Addr: This, Offset: Adjustment);
1457	}
1458
1459	const CXXMethodDecl *MD = cast<CXXMethodDecl>(Val: GD.getDecl());
1460
1461	GlobalDecl LookupGD = GD;
1462	if (const CXXDestructorDecl *DD = dyn_cast<CXXDestructorDecl>(Val: MD)) {
1463	// Complete dtors take a pointer to the complete object,
1464	// thus don't need adjustment.
1465	if (GD.getDtorType() == Dtor_Complete)
1466	return This;
1467
1468	// There's only Dtor_Deleting in vftable but it shares the this adjustment
1469	// with the base one, so look up the deleting one instead.
1470	LookupGD = GlobalDecl(DD, Dtor_Deleting);
1471	}
1472	MethodVFTableLocation ML =
1473	CGM.getMicrosoftVTableContext().getMethodVFTableLocation(GD: LookupGD);
1474
1475	CharUnits StaticOffset = ML.VFPtrOffset;
1476
1477	// Base destructors expect 'this' to point to the beginning of the base
1478	// subobject, not the first vfptr that happens to contain the virtual dtor.
1479	// However, we still need to apply the virtual base adjustment.
1480	if (isa<CXXDestructorDecl>(Val: MD) && GD.getDtorType() == Dtor_Base)
1481	StaticOffset = CharUnits::Zero();
1482
1483	Address Result = This;
1484	if (ML.VBase) {
1485	Result = Result.withElementType(ElemTy: CGF.Int8Ty);
1486
1487	const CXXRecordDecl *Derived = MD->getParent();
1488	const CXXRecordDecl *VBase = ML.VBase;
1489	llvm::Value *VBaseOffset =
1490	GetVirtualBaseClassOffset(CGF, This: Result, ClassDecl: Derived, BaseClassDecl: VBase);
1491	llvm::Value *VBasePtr = CGF.Builder.CreateInBoundsGEP(
1492	Ty: Result.getElementType(), Ptr: Result.emitRawPointer(CGF), IdxList: VBaseOffset);
1493	CharUnits VBaseAlign =
1494	CGF.CGM.getVBaseAlignment(DerivedAlign: Result.getAlignment(), Derived, VBase);
1495	Result = Address(VBasePtr, CGF.Int8Ty, VBaseAlign);
1496	}
1497	if (!StaticOffset.isZero()) {
1498	assert(StaticOffset.isPositive());
1499	Result = Result.withElementType(ElemTy: CGF.Int8Ty);
1500	if (ML.VBase) {
1501	// Non-virtual adjustment might result in a pointer outside the allocated
1502	// object, e.g. if the final overrider class is laid out after the virtual
1503	// base that declares a method in the most derived class.
1504	// FIXME: Update the code that emits this adjustment in thunks prologues.
1505	Result = CGF.Builder.CreateConstByteGEP(Addr: Result, Offset: StaticOffset);
1506	} else {
1507	Result = CGF.Builder.CreateConstInBoundsByteGEP(Addr: Result, Offset: StaticOffset);
1508	}
1509	}
1510	return Result;
1511	}
1512
1513	void MicrosoftCXXABI::addImplicitStructorParams(CodeGenFunction &CGF,
1514	QualType &ResTy,
1515	FunctionArgList &Params) {
1516	ASTContext &Context = getContext();
1517	const CXXMethodDecl *MD = cast<CXXMethodDecl>(Val: CGF.CurGD.getDecl());
1518	assert(isa<CXXConstructorDecl>(MD) || isa<CXXDestructorDecl>(MD));
1519	if (isa<CXXConstructorDecl>(Val: MD) && MD->getParent()->getNumVBases()) {
1520	auto *IsMostDerived = ImplicitParamDecl::Create(
1521	Context, /*DC=*/nullptr, CGF.CurGD.getDecl()->getLocation(),
1522	&Context.Idents.get(Name: "is_most_derived"), Context.IntTy,
1523	ImplicitParamKind::Other);
1524	// The 'most_derived' parameter goes second if the ctor is variadic and last
1525	// if it's not. Dtors can't be variadic.
1526	const FunctionProtoType *FPT = MD->getType()->castAs<FunctionProtoType>();
1527	if (FPT->isVariadic())
1528	Params.insert(Params.begin() + 1, IsMostDerived);
1529	else
1530	Params.push_back(Elt: IsMostDerived);
1531	getStructorImplicitParamDecl(CGF) = IsMostDerived;
1532	} else if (isDeletingDtor(GD: CGF.CurGD)) {
1533	auto *ShouldDelete = ImplicitParamDecl::Create(
1534	Context, /*DC=*/nullptr, CGF.CurGD.getDecl()->getLocation(),
1535	&Context.Idents.get(Name: "should_call_delete"), Context.IntTy,
1536	ImplicitParamKind::Other);
1537	Params.push_back(Elt: ShouldDelete);
1538	getStructorImplicitParamDecl(CGF) = ShouldDelete;
1539	}
1540	}
1541
1542	void MicrosoftCXXABI::EmitInstanceFunctionProlog(CodeGenFunction &CGF) {
1543	// Naked functions have no prolog.
1544	if (CGF.CurFuncDecl && CGF.CurFuncDecl->hasAttr<NakedAttr>())
1545	return;
1546
1547	// Overridden virtual methods of non-primary bases need to adjust the incoming
1548	// 'this' pointer in the prologue. In this hierarchy, C::b will subtract
1549	// sizeof(void*) to adjust from B* to C*:
1550	// struct A { virtual void a(); };
1551	// struct B { virtual void b(); };
1552	// struct C : A, B { virtual void b(); };
1553	//
1554	// Leave the value stored in the 'this' alloca unadjusted, so that the
1555	// debugger sees the unadjusted value. Microsoft debuggers require this, and
1556	// will apply the ThisAdjustment in the method type information.
1557	// FIXME: Do something better for DWARF debuggers, which won't expect this,
1558	// without making our codegen depend on debug info settings.
1559	llvm::Value *This = loadIncomingCXXThis(CGF);
1560	const CXXMethodDecl *MD = cast<CXXMethodDecl>(Val: CGF.CurGD.getDecl());
1561	if (!CGF.CurFuncIsThunk && MD->isVirtual()) {
1562	CharUnits Adjustment = getVirtualFunctionPrologueThisAdjustment(GD: CGF.CurGD);
1563	if (!Adjustment.isZero()) {
1564	assert(Adjustment.isPositive());
1565	This = CGF.Builder.CreateConstInBoundsGEP1_32(Ty: CGF.Int8Ty, Ptr: This,
1566	Idx0: -Adjustment.getQuantity());
1567	}
1568	}
1569	setCXXABIThisValue(CGF, ThisPtr: This);
1570
1571	// If this is a function that the ABI specifies returns 'this', initialize
1572	// the return slot to 'this' at the start of the function.
1573	//
1574	// Unlike the setting of return types, this is done within the ABI
1575	// implementation instead of by clients of CGCXXABI because:
1576	// 1) getThisValue is currently protected
1577	// 2) in theory, an ABI could implement 'this' returns some other way;
1578	// HasThisReturn only specifies a contract, not the implementation
1579	if (HasThisReturn(GD: CGF.CurGD) || hasMostDerivedReturn(GD: CGF.CurGD))
1580	CGF.Builder.CreateStore(Val: getThisValue(CGF), Addr: CGF.ReturnValue);
1581
1582	if (isa<CXXConstructorDecl>(Val: MD) && MD->getParent()->getNumVBases()) {
1583	assert(getStructorImplicitParamDecl(CGF) &&
1584	"no implicit parameter for a constructor with virtual bases?");
1585	getStructorImplicitParamValue(CGF)
1586	= CGF.Builder.CreateLoad(
1587	Addr: CGF.GetAddrOfLocalVar(getStructorImplicitParamDecl(CGF)),
1588	Name: "is_most_derived");
1589	}
1590
1591	if (isDeletingDtor(GD: CGF.CurGD)) {
1592	assert(getStructorImplicitParamDecl(CGF) &&
1593	"no implicit parameter for a deleting destructor?");
1594	getStructorImplicitParamValue(CGF)
1595	= CGF.Builder.CreateLoad(
1596	Addr: CGF.GetAddrOfLocalVar(getStructorImplicitParamDecl(CGF)),
1597	Name: "should_call_delete");
1598	}
1599	}
1600
1601	CGCXXABI::AddedStructorArgs MicrosoftCXXABI::getImplicitConstructorArgs(
1602	CodeGenFunction &CGF, const CXXConstructorDecl *D, CXXCtorType Type,
1603	bool ForVirtualBase, bool Delegating) {
1604	assert(Type == Ctor_Complete || Type == Ctor_Base);
1605
1606	// Check if we need a 'most_derived' parameter.
1607	if (!D->getParent()->getNumVBases())
1608	return AddedStructorArgs{};
1609
1610	// Add the 'most_derived' argument second if we are variadic or last if not.
1611	const FunctionProtoType *FPT = D->getType()->castAs<FunctionProtoType>();
1612	llvm::Value *MostDerivedArg;
1613	if (Delegating) {
1614	MostDerivedArg = getStructorImplicitParamValue(CGF);
1615	} else {
1616	MostDerivedArg = llvm::ConstantInt::get(Ty: CGM.Int32Ty, V: Type == Ctor_Complete);
1617	}
1618	if (FPT->isVariadic()) {
1619	return AddedStructorArgs::prefix(Args: {{MostDerivedArg, getContext().IntTy}});
1620	}
1621	return AddedStructorArgs::suffix(Args: {{MostDerivedArg, getContext().IntTy}});
1622	}
1623
1624	llvm::Value *MicrosoftCXXABI::getCXXDestructorImplicitParam(
1625	CodeGenFunction &CGF, const CXXDestructorDecl *DD, CXXDtorType Type,
1626	bool ForVirtualBase, bool Delegating) {
1627	return nullptr;
1628	}
1629
1630	void MicrosoftCXXABI::EmitDestructorCall(CodeGenFunction &CGF,
1631	const CXXDestructorDecl *DD,
1632	CXXDtorType Type, bool ForVirtualBase,
1633	bool Delegating, Address This,
1634	QualType ThisTy) {
1635	// Use the base destructor variant in place of the complete destructor variant
1636	// if the class has no virtual bases. This effectively implements some of the
1637	// -mconstructor-aliases optimization, but as part of the MS C++ ABI.
1638	if (Type == Dtor_Complete && DD->getParent()->getNumVBases() == 0)
1639	Type = Dtor_Base;
1640
1641	GlobalDecl GD(DD, Type);
1642	CGCallee Callee = CGCallee::forDirect(functionPtr: CGM.getAddrOfCXXStructor(GD), abstractInfo: GD);
1643
1644	if (DD->isVirtual()) {
1645	assert(Type != CXXDtorType::Dtor_Deleting &&
1646	"The deleting destructor should only be called via a virtual call");
1647	This = adjustThisArgumentForVirtualFunctionCall(CGF, GD: GlobalDecl(DD, Type),
1648	This, VirtualCall: false);
1649	}
1650
1651	llvm::BasicBlock *BaseDtorEndBB = nullptr;
1652	if (ForVirtualBase && isa<CXXConstructorDecl>(Val: CGF.CurCodeDecl)) {
1653	BaseDtorEndBB = EmitDtorCompleteObjectHandler(CGF);
1654	}
1655
1656	llvm::Value *Implicit =
1657	getCXXDestructorImplicitParam(CGF, DD, Type, ForVirtualBase,
1658	Delegating); // = nullptr
1659	CGF.EmitCXXDestructorCall(Dtor: GD, Callee, This: CGF.getAsNaturalPointerTo(Addr: This, PointeeType: ThisTy),
1660	ThisTy,
1661	/*ImplicitParam=*/Implicit,
1662	/*ImplicitParamTy=*/QualType(), E: nullptr);
1663	if (BaseDtorEndBB) {
1664	// Complete object handler should continue to be the remaining
1665	CGF.Builder.CreateBr(Dest: BaseDtorEndBB);
1666	CGF.EmitBlock(BB: BaseDtorEndBB);
1667	}
1668	}
1669
1670	void MicrosoftCXXABI::emitVTableTypeMetadata(const VPtrInfo &Info,
1671	const CXXRecordDecl *RD,
1672	llvm::GlobalVariable *VTable) {
1673	// Emit type metadata on vtables with LTO or IR instrumentation.
1674	// In IR instrumentation, the type metadata could be used to find out vtable
1675	// definitions (for type profiling) among all global variables.
1676	if (!CGM.getCodeGenOpts().LTOUnit &&
1677	!CGM.getCodeGenOpts().hasProfileIRInstr())
1678	return;
1679
1680	// TODO: Should VirtualFunctionElimination also be supported here?
1681	// See similar handling in CodeGenModule::EmitVTableTypeMetadata.
1682	if (CGM.getCodeGenOpts().WholeProgramVTables) {
1683	llvm::DenseSet<const CXXRecordDecl *> Visited;
1684	llvm::GlobalObject::VCallVisibility TypeVis =
1685	CGM.GetVCallVisibilityLevel(RD, Visited);
1686	if (TypeVis != llvm::GlobalObject::VCallVisibilityPublic)
1687	VTable->setVCallVisibilityMetadata(TypeVis);
1688	}
1689
1690	// The location of the first virtual function pointer in the virtual table,
1691	// aka the "address point" on Itanium. This is at offset 0 if RTTI is
1692	// disabled, or sizeof(void*) if RTTI is enabled.
1693	CharUnits AddressPoint =
1694	getContext().getLangOpts().RTTIData
1695	? getContext().toCharUnitsFromBits(
1696	BitSize: getContext().getTargetInfo().getPointerWidth(AddrSpace: LangAS::Default))
1697	: CharUnits::Zero();
1698
1699	if (Info.PathToIntroducingObject.empty()) {
1700	CGM.AddVTableTypeMetadata(VTable, Offset: AddressPoint, RD);
1701	return;
1702	}
1703
1704	// Add a bitset entry for the least derived base belonging to this vftable.
1705	CGM.AddVTableTypeMetadata(VTable, Offset: AddressPoint,
1706	RD: Info.PathToIntroducingObject.back());
1707
1708	// Add a bitset entry for each derived class that is laid out at the same
1709	// offset as the least derived base.
1710	for (unsigned I = Info.PathToIntroducingObject.size() - 1; I != 0; --I) {
1711	const CXXRecordDecl *DerivedRD = Info.PathToIntroducingObject[I - 1];
1712	const CXXRecordDecl *BaseRD = Info.PathToIntroducingObject[I];
1713
1714	const ASTRecordLayout &Layout =
1715	getContext().getASTRecordLayout(DerivedRD);
1716	CharUnits Offset;
1717	auto VBI = Layout.getVBaseOffsetsMap().find(Val: BaseRD);
1718	if (VBI == Layout.getVBaseOffsetsMap().end())
1719	Offset = Layout.getBaseClassOffset(Base: BaseRD);
1720	else
1721	Offset = VBI->second.VBaseOffset;
1722	if (!Offset.isZero())
1723	return;
1724	CGM.AddVTableTypeMetadata(VTable, Offset: AddressPoint, RD: DerivedRD);
1725	}
1726
1727	// Finally do the same for the most derived class.
1728	if (Info.FullOffsetInMDC.isZero())
1729	CGM.AddVTableTypeMetadata(VTable, Offset: AddressPoint, RD);
1730	}
1731
1732	void MicrosoftCXXABI::emitVTableDefinitions(CodeGenVTables &CGVT,
1733	const CXXRecordDecl *RD) {
1734	MicrosoftVTableContext &VFTContext = CGM.getMicrosoftVTableContext();
1735	const VPtrInfoVector &VFPtrs = VFTContext.getVFPtrOffsets(RD);
1736
1737	for (const std::unique_ptr<VPtrInfo>& Info : VFPtrs) {
1738	llvm::GlobalVariable *VTable = getAddrOfVTable(RD, VPtrOffset: Info->FullOffsetInMDC);
1739	if (VTable->hasInitializer())
1740	continue;
1741
1742	const VTableLayout &VTLayout =
1743	VFTContext.getVFTableLayout(RD, VFPtrOffset: Info->FullOffsetInMDC);
1744
1745	llvm::Constant *RTTI = nullptr;
1746	if (any_of(Range: VTLayout.vtable_components(),
1747	P: [](const VTableComponent &VTC) { return VTC.isRTTIKind(); }))
1748	RTTI = getMSCompleteObjectLocator(RD, Info: *Info);
1749
1750	ConstantInitBuilder builder(CGM);
1751	auto components = builder.beginStruct();
1752	CGVT.createVTableInitializer(builder&: components, layout: VTLayout, rtti: RTTI,
1753	vtableHasLocalLinkage: VTable->hasLocalLinkage());
1754	components.finishAndSetAsInitializer(global: VTable);
1755
1756	emitVTableTypeMetadata(Info: *Info, RD, VTable);
1757	}
1758	}
1759
1760	bool MicrosoftCXXABI::isVirtualOffsetNeededForVTableField(
1761	CodeGenFunction &CGF, CodeGenFunction::VPtr Vptr) {
1762	return Vptr.NearestVBase != nullptr;
1763	}
1764
1765	llvm::Value *MicrosoftCXXABI::getVTableAddressPointInStructor(
1766	CodeGenFunction &CGF, const CXXRecordDecl *VTableClass, BaseSubobject Base,
1767	const CXXRecordDecl *NearestVBase) {
1768	llvm::Constant *VTableAddressPoint = getVTableAddressPoint(Base, VTableClass);
1769	if (!VTableAddressPoint) {
1770	assert(Base.getBase()->getNumVBases() &&
1771	!getContext().getASTRecordLayout(Base.getBase()).hasOwnVFPtr());
1772	}
1773	return VTableAddressPoint;
1774	}
1775
1776	static void mangleVFTableName(MicrosoftMangleContext &MangleContext,
1777	const CXXRecordDecl *RD, const VPtrInfo &VFPtr,
1778	SmallString<256> &Name) {
1779	llvm::raw_svector_ostream Out(Name);
1780	MangleContext.mangleCXXVFTable(Derived: RD, BasePath: VFPtr.MangledPath, Out);
1781	}
1782
1783	llvm::Constant *
1784	MicrosoftCXXABI::getVTableAddressPoint(BaseSubobject Base,
1785	const CXXRecordDecl *VTableClass) {
1786	(void)getAddrOfVTable(RD: VTableClass, VPtrOffset: Base.getBaseOffset());
1787	VFTableIdTy ID(VTableClass, Base.getBaseOffset());
1788	return VFTablesMap[ID];
1789	}
1790
1791	llvm::GlobalVariable *MicrosoftCXXABI::getAddrOfVTable(const CXXRecordDecl *RD,
1792	CharUnits VPtrOffset) {
1793	// getAddrOfVTable may return 0 if asked to get an address of a vtable which
1794	// shouldn't be used in the given record type. We want to cache this result in
1795	// VFTablesMap, thus a simple zero check is not sufficient.
1796
1797	VFTableIdTy ID(RD, VPtrOffset);
1798	VTablesMapTy::iterator I;
1799	bool Inserted;
1800	std::tie(args&: I, args&: Inserted) = VTablesMap.insert(KV: std::make_pair(x&: ID, y: nullptr));
1801	if (!Inserted)
1802	return I->second;
1803
1804	llvm::GlobalVariable *&VTable = I->second;
1805
1806	MicrosoftVTableContext &VTContext = CGM.getMicrosoftVTableContext();
1807	const VPtrInfoVector &VFPtrs = VTContext.getVFPtrOffsets(RD);
1808
1809	if (DeferredVFTables.insert(Ptr: RD).second) {
1810	// We haven't processed this record type before.
1811	// Queue up this vtable for possible deferred emission.
1812	CGM.addDeferredVTable(RD);
1813
1814	#ifndef NDEBUG
1815	// Create all the vftables at once in order to make sure each vftable has
1816	// a unique mangled name.
1817	llvm::StringSet<> ObservedMangledNames;
1818	for (size_t J = 0, F = VFPtrs.size(); J != F; ++J) {
1819	SmallString<256> Name;
1820	mangleVFTableName(MangleContext&: getMangleContext(), RD, VFPtr: *VFPtrs[J], Name);
1821	if (!ObservedMangledNames.insert(key: Name.str()).second)
1822	llvm_unreachable("Already saw this mangling before?");
1823	}
1824	#endif
1825	}
1826
1827	const std::unique_ptr<VPtrInfo> *VFPtrI =
1828	llvm::find_if(Range: VFPtrs, P: [&](const std::unique_ptr<VPtrInfo> &VPI) {
1829	return VPI->FullOffsetInMDC == VPtrOffset;
1830	});
1831	if (VFPtrI == VFPtrs.end()) {
1832	VFTablesMap[ID] = nullptr;
1833	return nullptr;
1834	}
1835	const std::unique_ptr<VPtrInfo> &VFPtr = *VFPtrI;
1836
1837	SmallString<256> VFTableName;
1838	mangleVFTableName(MangleContext&: getMangleContext(), RD, VFPtr: *VFPtr, Name&: VFTableName);
1839
1840	// Classes marked __declspec(dllimport) need vftables generated on the
1841	// import-side in order to support features like constexpr. No other
1842	// translation unit relies on the emission of the local vftable, translation
1843	// units are expected to generate them as needed.
1844	//
1845	// Because of this unique behavior, we maintain this logic here instead of
1846	// getVTableLinkage.
1847	llvm::GlobalValue::LinkageTypes VFTableLinkage =
1848	RD->hasAttr<DLLImportAttr>() ? llvm::GlobalValue::LinkOnceODRLinkage
1849	: CGM.getVTableLinkage(RD);
1850	bool VFTableComesFromAnotherTU =
1851	llvm::GlobalValue::isAvailableExternallyLinkage(Linkage: VFTableLinkage) ||
1852	llvm::GlobalValue::isExternalLinkage(Linkage: VFTableLinkage);
1853	bool VTableAliasIsRequred =
1854	!VFTableComesFromAnotherTU && getContext().getLangOpts().RTTIData;
1855
1856	if (llvm::GlobalValue *VFTable =
1857	CGM.getModule().getNamedGlobal(Name: VFTableName)) {
1858	VFTablesMap[ID] = VFTable;
1859	VTable = VTableAliasIsRequred
1860	? cast<llvm::GlobalVariable>(
1861	Val: cast<llvm::GlobalAlias>(Val: VFTable)->getAliaseeObject())
1862	: cast<llvm::GlobalVariable>(Val: VFTable);
1863	return VTable;
1864	}
1865
1866	const VTableLayout &VTLayout =
1867	VTContext.getVFTableLayout(RD, VFPtrOffset: VFPtr->FullOffsetInMDC);
1868	llvm::GlobalValue::LinkageTypes VTableLinkage =
1869	VTableAliasIsRequred ? llvm::GlobalValue::PrivateLinkage : VFTableLinkage;
1870
1871	StringRef VTableName = VTableAliasIsRequred ? StringRef() : VFTableName.str();
1872
1873	llvm::Type *VTableType = CGM.getVTables().getVTableType(layout: VTLayout);
1874
1875	// Create a backing variable for the contents of VTable. The VTable may
1876	// or may not include space for a pointer to RTTI data.
1877	llvm::GlobalValue *VFTable;
1878	VTable = new llvm::GlobalVariable(CGM.getModule(), VTableType,
1879	/*isConstant=*/true, VTableLinkage,
1880	/*Initializer=*/nullptr, VTableName);
1881	VTable->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
1882
1883	llvm::Comdat *C = nullptr;
1884	if (!VFTableComesFromAnotherTU &&
1885	llvm::GlobalValue::isWeakForLinker(Linkage: VFTableLinkage))
1886	C = CGM.getModule().getOrInsertComdat(Name: VFTableName.str());
1887
1888	// Only insert a pointer into the VFTable for RTTI data if we are not
1889	// importing it. We never reference the RTTI data directly so there is no
1890	// need to make room for it.
1891	if (VTableAliasIsRequred) {
1892	llvm::Value *GEPIndices[] = {llvm::ConstantInt::get(Ty: CGM.Int32Ty, V: 0),
1893	llvm::ConstantInt::get(Ty: CGM.Int32Ty, V: 0),
1894	llvm::ConstantInt::get(Ty: CGM.Int32Ty, V: 1)};
1895	// Create a GEP which points just after the first entry in the VFTable,
1896	// this should be the location of the first virtual method.
1897	llvm::Constant *VTableGEP = llvm::ConstantExpr::getInBoundsGetElementPtr(
1898	Ty: VTable->getValueType(), C: VTable, IdxList: GEPIndices);
1899	if (llvm::GlobalValue::isWeakForLinker(Linkage: VFTableLinkage)) {
1900	VFTableLinkage = llvm::GlobalValue::ExternalLinkage;
1901	if (C)
1902	C->setSelectionKind(llvm::Comdat::Largest);
1903	}
1904	VFTable = llvm::GlobalAlias::create(Ty: CGM.Int8PtrTy,
1905	/*AddressSpace=*/0, Linkage: VFTableLinkage,
1906	Name: VFTableName.str(), Aliasee: VTableGEP,
1907	Parent: &CGM.getModule());
1908	VFTable->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
1909	} else {
1910	// We don't need a GlobalAlias to be a symbol for the VTable if we won't
1911	// be referencing any RTTI data.
1912	// The GlobalVariable will end up being an appropriate definition of the
1913	// VFTable.
1914	VFTable = VTable;
1915	}
1916	if (C)
1917	VTable->setComdat(C);
1918
1919	if (RD->hasAttr<DLLExportAttr>())
1920	VFTable->setDLLStorageClass(llvm::GlobalValue::DLLExportStorageClass);
1921
1922	VFTablesMap[ID] = VFTable;
1923	return VTable;
1924	}
1925
1926	CGCallee MicrosoftCXXABI::getVirtualFunctionPointer(CodeGenFunction &CGF,
1927	GlobalDecl GD,
1928	Address This,
1929	llvm::Type *Ty,
1930	SourceLocation Loc) {
1931	CGBuilderTy &Builder = CGF.Builder;
1932
1933	Ty = Ty->getPointerTo();
1934	Address VPtr =
1935	adjustThisArgumentForVirtualFunctionCall(CGF, GD, This, VirtualCall: true);
1936
1937	auto *MethodDecl = cast<CXXMethodDecl>(Val: GD.getDecl());
1938	llvm::Value *VTable = CGF.GetVTablePtr(This: VPtr, VTableTy: Ty->getPointerTo(),
1939	VTableClass: MethodDecl->getParent());
1940
1941	MicrosoftVTableContext &VFTContext = CGM.getMicrosoftVTableContext();
1942	MethodVFTableLocation ML = VFTContext.getMethodVFTableLocation(GD);
1943
1944	// Compute the identity of the most derived class whose virtual table is
1945	// located at the MethodVFTableLocation ML.
1946	auto getObjectWithVPtr = [&] {
1947	return llvm::find_if(Range: VFTContext.getVFPtrOffsets(
1948	RD: ML.VBase ? ML.VBase : MethodDecl->getParent()),
1949	P: [&](const std::unique_ptr<VPtrInfo> &Info) {
1950	return Info->FullOffsetInMDC == ML.VFPtrOffset;
1951	})
1952	->get()
1953	->ObjectWithVPtr;
1954	};
1955
1956	llvm::Value *VFunc;
1957	if (CGF.ShouldEmitVTableTypeCheckedLoad(RD: MethodDecl->getParent())) {
1958	VFunc = CGF.EmitVTableTypeCheckedLoad(
1959	RD: getObjectWithVPtr(), VTable, VTableTy: Ty,
1960	VTableByteOffset: ML.Index *
1961	CGM.getContext().getTargetInfo().getPointerWidth(AddrSpace: LangAS::Default) /
1962	8);
1963	} else {
1964	if (CGM.getCodeGenOpts().PrepareForLTO)
1965	CGF.EmitTypeMetadataCodeForVCall(RD: getObjectWithVPtr(), VTable, Loc);
1966
1967	llvm::Value *VFuncPtr =
1968	Builder.CreateConstInBoundsGEP1_64(Ty, Ptr: VTable, Idx0: ML.Index, Name: "vfn");
1969	VFunc = Builder.CreateAlignedLoad(Ty, VFuncPtr, CGF.getPointerAlign());
1970	}
1971
1972	CGCallee Callee(GD, VFunc);
1973	return Callee;
1974	}
1975
1976	llvm::Value *MicrosoftCXXABI::EmitVirtualDestructorCall(
1977	CodeGenFunction &CGF, const CXXDestructorDecl *Dtor, CXXDtorType DtorType,
1978	Address This, DeleteOrMemberCallExpr E) {
1979	auto *CE = E.dyn_cast<const CXXMemberCallExpr *>();
1980	auto *D = E.dyn_cast<const CXXDeleteExpr *>();
1981	assert((CE != nullptr) ^ (D != nullptr));
1982	assert(CE == nullptr || CE->arg_begin() == CE->arg_end());
1983	assert(DtorType == Dtor_Deleting || DtorType == Dtor_Complete);
1984
1985	// We have only one destructor in the vftable but can get both behaviors
1986	// by passing an implicit int parameter.
1987	GlobalDecl GD(Dtor, Dtor_Deleting);
1988	const CGFunctionInfo *FInfo =
1989	&CGM.getTypes().arrangeCXXStructorDeclaration(GD);
1990	llvm::FunctionType *Ty = CGF.CGM.getTypes().GetFunctionType(Info: *FInfo);
1991	CGCallee Callee = CGCallee::forVirtual(CE, GD, This, Ty);
1992
1993	ASTContext &Context = getContext();
1994	llvm::Value *ImplicitParam = llvm::ConstantInt::get(
1995	Ty: llvm::IntegerType::getInt32Ty(C&: CGF.getLLVMContext()),
1996	V: DtorType == Dtor_Deleting);
1997
1998	QualType ThisTy;
1999	if (CE) {
2000	ThisTy = CE->getObjectType();
2001	} else {
2002	ThisTy = D->getDestroyedType();
2003	}
2004
2005	This = adjustThisArgumentForVirtualFunctionCall(CGF, GD, This, VirtualCall: true);
2006	RValue RV =
2007	CGF.EmitCXXDestructorCall(GD, Callee, This.emitRawPointer(CGF), ThisTy,
2008	ImplicitParam, Context.IntTy, CE);
2009	return RV.getScalarVal();
2010	}
2011
2012	const VBTableGlobals &
2013	MicrosoftCXXABI::enumerateVBTables(const CXXRecordDecl *RD) {
2014	// At this layer, we can key the cache off of a single class, which is much
2015	// easier than caching each vbtable individually.
2016	llvm::DenseMap<const CXXRecordDecl*, VBTableGlobals>::iterator Entry;
2017	bool Added;
2018	std::tie(args&: Entry, args&: Added) =
2019	VBTablesMap.insert(KV: std::make_pair(x&: RD, y: VBTableGlobals()));
2020	VBTableGlobals &VBGlobals = Entry->second;
2021	if (!Added)
2022	return VBGlobals;
2023
2024	MicrosoftVTableContext &Context = CGM.getMicrosoftVTableContext();
2025	VBGlobals.VBTables = &Context.enumerateVBTables(RD);
2026
2027	// Cache the globals for all vbtables so we don't have to recompute the
2028	// mangled names.
2029	llvm::GlobalVariable::LinkageTypes Linkage = CGM.getVTableLinkage(RD);
2030	for (VPtrInfoVector::const_iterator I = VBGlobals.VBTables->begin(),
2031	E = VBGlobals.VBTables->end();
2032	I != E; ++I) {
2033	VBGlobals.Globals.push_back(Elt: getAddrOfVBTable(VBT: **I, RD, Linkage));
2034	}
2035
2036	return VBGlobals;
2037	}
2038
2039	llvm::Function *
2040	MicrosoftCXXABI::EmitVirtualMemPtrThunk(const CXXMethodDecl *MD,
2041	const MethodVFTableLocation &ML) {
2042	assert(!isa<CXXConstructorDecl>(MD) && !isa<CXXDestructorDecl>(MD) &&
2043	"can't form pointers to ctors or virtual dtors");
2044
2045	// Calculate the mangled name.
2046	SmallString<256> ThunkName;
2047	llvm::raw_svector_ostream Out(ThunkName);
2048	getMangleContext().mangleVirtualMemPtrThunk(MD, ML, Out);
2049
2050	// If the thunk has been generated previously, just return it.
2051	if (llvm::GlobalValue *GV = CGM.getModule().getNamedValue(Name: ThunkName))
2052	return cast<llvm::Function>(Val: GV);
2053
2054	// Create the llvm::Function.
2055	const CGFunctionInfo &FnInfo =
2056	CGM.getTypes().arrangeUnprototypedMustTailThunk(MD);
2057	llvm::FunctionType *ThunkTy = CGM.getTypes().GetFunctionType(Info: FnInfo);
2058	llvm::Function *ThunkFn =
2059	llvm::Function::Create(Ty: ThunkTy, Linkage: llvm::Function::ExternalLinkage,
2060	N: ThunkName.str(), M: &CGM.getModule());
2061	assert(ThunkFn->getName() == ThunkName && "name was uniqued!");
2062
2063	ThunkFn->setLinkage(MD->isExternallyVisible()
2064	? llvm::GlobalValue::LinkOnceODRLinkage
2065	: llvm::GlobalValue::InternalLinkage);
2066	if (MD->isExternallyVisible())
2067	ThunkFn->setComdat(CGM.getModule().getOrInsertComdat(Name: ThunkFn->getName()));
2068
2069	CGM.SetLLVMFunctionAttributes(MD, FnInfo, ThunkFn, /*IsThunk=*/false);
2070	CGM.SetLLVMFunctionAttributesForDefinition(MD, ThunkFn);
2071
2072	// Add the "thunk" attribute so that LLVM knows that the return type is
2073	// meaningless. These thunks can be used to call functions with differing
2074	// return types, and the caller is required to cast the prototype
2075	// appropriately to extract the correct value.
2076	ThunkFn->addFnAttr(Kind: "thunk");
2077
2078	// These thunks can be compared, so they are not unnamed.
2079	ThunkFn->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::None);
2080
2081	// Start codegen.
2082	CodeGenFunction CGF(CGM);
2083	CGF.CurGD = GlobalDecl(MD);
2084	CGF.CurFuncIsThunk = true;
2085
2086	// Build FunctionArgs, but only include the implicit 'this' parameter
2087	// declaration.
2088	FunctionArgList FunctionArgs;
2089	buildThisParam(CGF, Params&: FunctionArgs);
2090
2091	// Start defining the function.
2092	CGF.StartFunction(GD: GlobalDecl(), RetTy: FnInfo.getReturnType(), Fn: ThunkFn, FnInfo,
2093	Args: FunctionArgs, Loc: MD->getLocation(), StartLoc: SourceLocation());
2094
2095	ApplyDebugLocation AL(CGF, MD->getLocation());
2096	setCXXABIThisValue(CGF, ThisPtr: loadIncomingCXXThis(CGF));
2097
2098	// Load the vfptr and then callee from the vftable. The callee should have
2099	// adjusted 'this' so that the vfptr is at offset zero.
2100	llvm::Type *ThunkPtrTy = ThunkTy->getPointerTo();
2101	llvm::Value *VTable = CGF.GetVTablePtr(
2102	This: getThisAddress(CGF), VTableTy: ThunkPtrTy->getPointerTo(), VTableClass: MD->getParent());
2103
2104	llvm::Value *VFuncPtr = CGF.Builder.CreateConstInBoundsGEP1_64(
2105	Ty: ThunkPtrTy, Ptr: VTable, Idx0: ML.Index, Name: "vfn");
2106	llvm::Value *Callee =
2107	CGF.Builder.CreateAlignedLoad(ThunkPtrTy, VFuncPtr, CGF.getPointerAlign());
2108
2109	CGF.EmitMustTailThunk(MD, getThisValue(CGF), {ThunkTy, Callee});
2110
2111	return ThunkFn;
2112	}
2113
2114	void MicrosoftCXXABI::emitVirtualInheritanceTables(const CXXRecordDecl *RD) {
2115	const VBTableGlobals &VBGlobals = enumerateVBTables(RD);
2116	for (unsigned I = 0, E = VBGlobals.VBTables->size(); I != E; ++I) {
2117	const std::unique_ptr<VPtrInfo>& VBT = (*VBGlobals.VBTables)[I];
2118	llvm::GlobalVariable *GV = VBGlobals.Globals[I];
2119	if (GV->isDeclaration())
2120	emitVBTableDefinition(VBT: *VBT, RD, GV);
2121	}
2122	}
2123
2124	llvm::GlobalVariable *
2125	MicrosoftCXXABI::getAddrOfVBTable(const VPtrInfo &VBT, const CXXRecordDecl *RD,
2126	llvm::GlobalVariable::LinkageTypes Linkage) {
2127	SmallString<256> OutName;
2128	llvm::raw_svector_ostream Out(OutName);
2129	getMangleContext().mangleCXXVBTable(Derived: RD, BasePath: VBT.MangledPath, Out);
2130	StringRef Name = OutName.str();
2131
2132	llvm::ArrayType *VBTableType =
2133	llvm::ArrayType::get(ElementType: CGM.IntTy, NumElements: 1 + VBT.ObjectWithVPtr->getNumVBases());
2134
2135	assert(!CGM.getModule().getNamedGlobal(Name) &&
2136	"vbtable with this name already exists: mangling bug?");
2137	CharUnits Alignment =
2138	CGM.getContext().getTypeAlignInChars(CGM.getContext().IntTy);
2139	llvm::GlobalVariable *GV = CGM.CreateOrReplaceCXXRuntimeVariable(
2140	Name, Ty: VBTableType, Linkage, Alignment: Alignment.getAsAlign());
2141	GV->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
2142
2143	if (RD->hasAttr<DLLImportAttr>())
2144	GV->setDLLStorageClass(llvm::GlobalValue::DLLImportStorageClass);
2145	else if (RD->hasAttr<DLLExportAttr>())
2146	GV->setDLLStorageClass(llvm::GlobalValue::DLLExportStorageClass);
2147
2148	if (!GV->hasExternalLinkage())
2149	emitVBTableDefinition(VBT, RD, GV);
2150
2151	return GV;
2152	}
2153
2154	void MicrosoftCXXABI::emitVBTableDefinition(const VPtrInfo &VBT,
2155	const CXXRecordDecl *RD,
2156	llvm::GlobalVariable *GV) const {
2157	const CXXRecordDecl *ObjectWithVPtr = VBT.ObjectWithVPtr;
2158
2159	assert(RD->getNumVBases() && ObjectWithVPtr->getNumVBases() &&
2160	"should only emit vbtables for classes with vbtables");
2161
2162	const ASTRecordLayout &BaseLayout =
2163	getContext().getASTRecordLayout(VBT.IntroducingObject);
2164	const ASTRecordLayout &DerivedLayout = getContext().getASTRecordLayout(RD);
2165
2166	SmallVector<llvm::Constant *, 4> Offsets(1 + ObjectWithVPtr->getNumVBases(),
2167	nullptr);
2168
2169	// The offset from ObjectWithVPtr's vbptr to itself always leads.
2170	CharUnits VBPtrOffset = BaseLayout.getVBPtrOffset();
2171	Offsets[0] = llvm::ConstantInt::get(Ty: CGM.IntTy, V: -VBPtrOffset.getQuantity());
2172
2173	MicrosoftVTableContext &Context = CGM.getMicrosoftVTableContext();
2174	for (const auto &I : ObjectWithVPtr->vbases()) {
2175	const CXXRecordDecl *VBase = I.getType()->getAsCXXRecordDecl();
2176	CharUnits Offset = DerivedLayout.getVBaseClassOffset(VBase);
2177	assert(!Offset.isNegative());
2178
2179	// Make it relative to the subobject vbptr.
2180	CharUnits CompleteVBPtrOffset = VBT.NonVirtualOffset + VBPtrOffset;
2181	if (VBT.getVBaseWithVPtr())
2182	CompleteVBPtrOffset +=
2183	DerivedLayout.getVBaseClassOffset(VBase: VBT.getVBaseWithVPtr());
2184	Offset -= CompleteVBPtrOffset;
2185
2186	unsigned VBIndex = Context.getVBTableIndex(Derived: ObjectWithVPtr, VBase);
2187	assert(Offsets[VBIndex] == nullptr && "The same vbindex seen twice?");
2188	Offsets[VBIndex] = llvm::ConstantInt::get(Ty: CGM.IntTy, V: Offset.getQuantity());
2189	}
2190
2191	assert(Offsets.size() ==
2192	cast<llvm::ArrayType>(GV->getValueType())->getNumElements());
2193	llvm::ArrayType *VBTableType =
2194	llvm::ArrayType::get(ElementType: CGM.IntTy, NumElements: Offsets.size());
2195	llvm::Constant *Init = llvm::ConstantArray::get(T: VBTableType, V: Offsets);
2196	GV->setInitializer(Init);
2197
2198	if (RD->hasAttr<DLLImportAttr>())
2199	GV->setLinkage(llvm::GlobalVariable::AvailableExternallyLinkage);
2200	}
2201
2202	llvm::Value *MicrosoftCXXABI::performThisAdjustment(CodeGenFunction &CGF,
2203	Address This,
2204	const ThisAdjustment &TA) {
2205	if (TA.isEmpty())
2206	return This.emitRawPointer(CGF);
2207
2208	This = This.withElementType(ElemTy: CGF.Int8Ty);
2209
2210	llvm::Value *V;
2211	if (TA.Virtual.isEmpty()) {
2212	V = This.emitRawPointer(CGF);
2213	} else {
2214	assert(TA.Virtual.Microsoft.VtordispOffset < 0);
2215	// Adjust the this argument based on the vtordisp value.
2216	Address VtorDispPtr =
2217	CGF.Builder.CreateConstInBoundsByteGEP(Addr: This,
2218	Offset: CharUnits::fromQuantity(Quantity: TA.Virtual.Microsoft.VtordispOffset));
2219	VtorDispPtr = VtorDispPtr.withElementType(ElemTy: CGF.Int32Ty);
2220	llvm::Value *VtorDisp = CGF.Builder.CreateLoad(Addr: VtorDispPtr, Name: "vtordisp");
2221	V = CGF.Builder.CreateGEP(Ty: This.getElementType(), Ptr: This.emitRawPointer(CGF),
2222	IdxList: CGF.Builder.CreateNeg(V: VtorDisp));
2223
2224	// Unfortunately, having applied the vtordisp means that we no
2225	// longer really have a known alignment for the vbptr step.
2226	// We'll assume the vbptr is pointer-aligned.
2227
2228	if (TA.Virtual.Microsoft.VBPtrOffset) {
2229	// If the final overrider is defined in a virtual base other than the one
2230	// that holds the vfptr, we have to use a vtordispex thunk which looks up
2231	// the vbtable of the derived class.
2232	assert(TA.Virtual.Microsoft.VBPtrOffset > 0);
2233	assert(TA.Virtual.Microsoft.VBOffsetOffset >= 0);
2234	llvm::Value *VBPtr;
2235	llvm::Value *VBaseOffset = GetVBaseOffsetFromVBPtr(
2236	CGF, Base: Address(V, CGF.Int8Ty, CGF.getPointerAlign()),
2237	VBPtrOffset: -TA.Virtual.Microsoft.VBPtrOffset,
2238	VBTableOffset: TA.Virtual.Microsoft.VBOffsetOffset, VBPtr: &VBPtr);
2239	V = CGF.Builder.CreateInBoundsGEP(Ty: CGF.Int8Ty, Ptr: VBPtr, IdxList: VBaseOffset);
2240	}
2241	}
2242
2243	if (TA.NonVirtual) {
2244	// Non-virtual adjustment might result in a pointer outside the allocated
2245	// object, e.g. if the final overrider class is laid out after the virtual
2246	// base that declares a method in the most derived class.
2247	V = CGF.Builder.CreateConstGEP1_32(Ty: CGF.Int8Ty, Ptr: V, Idx0: TA.NonVirtual);
2248	}
2249
2250	// Don't need to bitcast back, the call CodeGen will handle this.
2251	return V;
2252	}
2253
2254	llvm::Value *
2255	MicrosoftCXXABI::performReturnAdjustment(CodeGenFunction &CGF, Address Ret,
2256	const ReturnAdjustment &RA) {
2257	if (RA.isEmpty())
2258	return Ret.emitRawPointer(CGF);
2259
2260	Ret = Ret.withElementType(ElemTy: CGF.Int8Ty);
2261
2262	llvm::Value *V = Ret.emitRawPointer(CGF);
2263	if (RA.Virtual.Microsoft.VBIndex) {
2264	assert(RA.Virtual.Microsoft.VBIndex > 0);
2265	int32_t IntSize = CGF.getIntSize().getQuantity();
2266	llvm::Value *VBPtr;
2267	llvm::Value *VBaseOffset =
2268	GetVBaseOffsetFromVBPtr(CGF, Base: Ret, VBPtrOffset: RA.Virtual.Microsoft.VBPtrOffset,
2269	VBTableOffset: IntSize * RA.Virtual.Microsoft.VBIndex, VBPtr: &VBPtr);
2270	V = CGF.Builder.CreateInBoundsGEP(Ty: CGF.Int8Ty, Ptr: VBPtr, IdxList: VBaseOffset);
2271	}
2272
2273	if (RA.NonVirtual)
2274	V = CGF.Builder.CreateConstInBoundsGEP1_32(Ty: CGF.Int8Ty, Ptr: V, Idx0: RA.NonVirtual);
2275
2276	return V;
2277	}
2278
2279	bool MicrosoftCXXABI::requiresArrayCookie(const CXXDeleteExpr *expr,
2280	QualType elementType) {
2281	// Microsoft seems to completely ignore the possibility of a
2282	// two-argument usual deallocation function.
2283	return elementType.isDestructedType();
2284	}
2285
2286	bool MicrosoftCXXABI::requiresArrayCookie(const CXXNewExpr *expr) {
2287	// Microsoft seems to completely ignore the possibility of a
2288	// two-argument usual deallocation function.
2289	return expr->getAllocatedType().isDestructedType();
2290	}
2291
2292	CharUnits MicrosoftCXXABI::getArrayCookieSizeImpl(QualType type) {
2293	// The array cookie is always a size_t; we then pad that out to the
2294	// alignment of the element type.
2295	ASTContext &Ctx = getContext();
2296	return std::max(a: Ctx.getTypeSizeInChars(T: Ctx.getSizeType()),
2297	b: Ctx.getTypeAlignInChars(T: type));
2298	}
2299
2300	llvm::Value *MicrosoftCXXABI::readArrayCookieImpl(CodeGenFunction &CGF,
2301	Address allocPtr,
2302	CharUnits cookieSize) {
2303	Address numElementsPtr = allocPtr.withElementType(ElemTy: CGF.SizeTy);
2304	return CGF.Builder.CreateLoad(Addr: numElementsPtr);
2305	}
2306
2307	Address MicrosoftCXXABI::InitializeArrayCookie(CodeGenFunction &CGF,
2308	Address newPtr,
2309	llvm::Value *numElements,
2310	const CXXNewExpr *expr,
2311	QualType elementType) {
2312	assert(requiresArrayCookie(expr));
2313
2314	// The size of the cookie.
2315	CharUnits cookieSize = getArrayCookieSizeImpl(type: elementType);
2316
2317	// Compute an offset to the cookie.
2318	Address cookiePtr = newPtr;
2319
2320	// Write the number of elements into the appropriate slot.
2321	Address numElementsPtr = cookiePtr.withElementType(ElemTy: CGF.SizeTy);
2322	CGF.Builder.CreateStore(Val: numElements, Addr: numElementsPtr);
2323
2324	// Finally, compute a pointer to the actual data buffer by skipping
2325	// over the cookie completely.
2326	return CGF.Builder.CreateConstInBoundsByteGEP(Addr: newPtr, Offset: cookieSize);
2327	}
2328
2329	static void emitGlobalDtorWithTLRegDtor(CodeGenFunction &CGF, const VarDecl &VD,
2330	llvm::FunctionCallee Dtor,
2331	llvm::Constant *Addr) {
2332	// Create a function which calls the destructor.
2333	llvm::Constant *DtorStub = CGF.createAtExitStub(VD, Dtor, Addr);
2334
2335	// extern "C" int __tlregdtor(void (*f)(void));
2336	llvm::FunctionType *TLRegDtorTy = llvm::FunctionType::get(
2337	Result: CGF.IntTy, Params: DtorStub->getType(), /*isVarArg=*/false);
2338
2339	llvm::FunctionCallee TLRegDtor = CGF.CGM.CreateRuntimeFunction(
2340	Ty: TLRegDtorTy, Name: "__tlregdtor", ExtraAttrs: llvm::AttributeList(), /*Local=*/true);
2341	if (llvm::Function *TLRegDtorFn =
2342	dyn_cast<llvm::Function>(Val: TLRegDtor.getCallee()))
2343	TLRegDtorFn->setDoesNotThrow();
2344
2345	CGF.EmitNounwindRuntimeCall(callee: TLRegDtor, args: DtorStub);
2346	}
2347
2348	void MicrosoftCXXABI::registerGlobalDtor(CodeGenFunction &CGF, const VarDecl &D,
2349	llvm::FunctionCallee Dtor,
2350	llvm::Constant *Addr) {
2351	if (D.isNoDestroy(CGM.getContext()))
2352	return;
2353
2354	if (D.getTLSKind())
2355	return emitGlobalDtorWithTLRegDtor(CGF, VD: D, Dtor, Addr);
2356
2357	// HLSL doesn't support atexit.
2358	if (CGM.getLangOpts().HLSL)
2359	return CGM.AddCXXDtorEntry(DtorFn: Dtor, Object: Addr);
2360
2361	// The default behavior is to use atexit.
2362	CGF.registerGlobalDtorWithAtExit(D, fn: Dtor, addr: Addr);
2363	}
2364
2365	void MicrosoftCXXABI::EmitThreadLocalInitFuncs(
2366	CodeGenModule &CGM, ArrayRef<const VarDecl *> CXXThreadLocals,
2367	ArrayRef<llvm::Function *> CXXThreadLocalInits,
2368	ArrayRef<const VarDecl *> CXXThreadLocalInitVars) {
2369	if (CXXThreadLocalInits.empty())
2370	return;
2371
2372	CGM.AppendLinkerOptions(Opts: CGM.getTarget().getTriple().getArch() ==
2373	llvm::Triple::x86
2374	? "/include:___dyn_tls_init@12"
2375	: "/include:__dyn_tls_init");
2376
2377	// This will create a GV in the .CRT$XDU section. It will point to our
2378	// initialization function. The CRT will call all of these function
2379	// pointers at start-up time and, eventually, at thread-creation time.
2380	auto AddToXDU = [&CGM](llvm::Function *InitFunc) {
2381	llvm::GlobalVariable *InitFuncPtr = new llvm::GlobalVariable(
2382	CGM.getModule(), InitFunc->getType(), /*isConstant=*/true,
2383	llvm::GlobalVariable::InternalLinkage, InitFunc,
2384	Twine(InitFunc->getName(), "$initializer$"));
2385	InitFuncPtr->setSection(".CRT$XDU");
2386	// This variable has discardable linkage, we have to add it to @llvm.used to
2387	// ensure it won't get discarded.
2388	CGM.addUsedGlobal(GV: InitFuncPtr);
2389	return InitFuncPtr;
2390	};
2391
2392	std::vector<llvm::Function *> NonComdatInits;
2393	for (size_t I = 0, E = CXXThreadLocalInitVars.size(); I != E; ++I) {
2394	llvm::GlobalVariable *GV = cast<llvm::GlobalVariable>(
2395	Val: CGM.GetGlobalValue(Ref: CGM.getMangledName(GD: CXXThreadLocalInitVars[I])));
2396	llvm::Function *F = CXXThreadLocalInits[I];
2397
2398	// If the GV is already in a comdat group, then we have to join it.
2399	if (llvm::Comdat *C = GV->getComdat())
2400	AddToXDU(F)->setComdat(C);
2401	else
2402	NonComdatInits.push_back(x: F);
2403	}
2404
2405	if (!NonComdatInits.empty()) {
2406	llvm::FunctionType *FTy =
2407	llvm::FunctionType::get(Result: CGM.VoidTy, /*isVarArg=*/false);
2408	llvm::Function *InitFunc = CGM.CreateGlobalInitOrCleanUpFunction(
2409	ty: FTy, name: "__tls_init", FI: CGM.getTypes().arrangeNullaryFunction(),
2410	Loc: SourceLocation(), /*TLS=*/true);
2411	CodeGenFunction(CGM).GenerateCXXGlobalInitFunc(Fn: InitFunc, CXXThreadLocals: NonComdatInits);
2412
2413	AddToXDU(InitFunc);
2414	}
2415	}
2416
2417	static llvm::GlobalValue *getTlsGuardVar(CodeGenModule &CGM) {
2418	// __tls_guard comes from the MSVC runtime and reflects
2419	// whether TLS has been initialized for a particular thread.
2420	// It is set from within __dyn_tls_init by the runtime.
2421	// Every library and executable has its own variable.
2422	llvm::Type *VTy = llvm::Type::getInt8Ty(C&: CGM.getLLVMContext());
2423	llvm::Constant *TlsGuardConstant =
2424	CGM.CreateRuntimeVariable(Ty: VTy, Name: "__tls_guard");
2425	llvm::GlobalValue *TlsGuard = cast<llvm::GlobalValue>(Val: TlsGuardConstant);
2426
2427	TlsGuard->setThreadLocal(true);
2428
2429	return TlsGuard;
2430	}
2431
2432	static llvm::FunctionCallee getDynTlsOnDemandInitFn(CodeGenModule &CGM) {
2433	// __dyn_tls_on_demand_init comes from the MSVC runtime and triggers
2434	// dynamic TLS initialization by calling __dyn_tls_init internally.
2435	llvm::FunctionType *FTy =
2436	llvm::FunctionType::get(Result: llvm::Type::getVoidTy(C&: CGM.getLLVMContext()), Params: {},
2437	/*isVarArg=*/false);
2438	return CGM.CreateRuntimeFunction(
2439	FTy, "__dyn_tls_on_demand_init",
2440	llvm::AttributeList::get(CGM.getLLVMContext(),
2441	llvm::AttributeList::FunctionIndex,
2442	llvm::Attribute::NoUnwind),
2443	/*Local=*/true);
2444	}
2445
2446	static void emitTlsGuardCheck(CodeGenFunction &CGF, llvm::GlobalValue *TlsGuard,
2447	llvm::BasicBlock *DynInitBB,
2448	llvm::BasicBlock *ContinueBB) {
2449	llvm::LoadInst *TlsGuardValue =
2450	CGF.Builder.CreateLoad(Addr: Address(TlsGuard, CGF.Int8Ty, CharUnits::One()));
2451	llvm::Value *CmpResult =
2452	CGF.Builder.CreateICmpEQ(LHS: TlsGuardValue, RHS: CGF.Builder.getInt8(C: 0));
2453	CGF.Builder.CreateCondBr(Cond: CmpResult, True: DynInitBB, False: ContinueBB);
2454	}
2455
2456	static void emitDynamicTlsInitializationCall(CodeGenFunction &CGF,
2457	llvm::GlobalValue *TlsGuard,
2458	llvm::BasicBlock *ContinueBB) {
2459	llvm::FunctionCallee Initializer = getDynTlsOnDemandInitFn(CGM&: CGF.CGM);
2460	llvm::Function *InitializerFunction =
2461	cast<llvm::Function>(Val: Initializer.getCallee());
2462	llvm::CallInst *CallVal = CGF.Builder.CreateCall(Callee: InitializerFunction);
2463	CallVal->setCallingConv(InitializerFunction->getCallingConv());
2464
2465	CGF.Builder.CreateBr(Dest: ContinueBB);
2466	}
2467
2468	static void emitDynamicTlsInitialization(CodeGenFunction &CGF) {
2469	llvm::BasicBlock *DynInitBB =
2470	CGF.createBasicBlock(name: "dyntls.dyn_init", parent: CGF.CurFn);
2471	llvm::BasicBlock *ContinueBB =
2472	CGF.createBasicBlock(name: "dyntls.continue", parent: CGF.CurFn);
2473
2474	llvm::GlobalValue *TlsGuard = getTlsGuardVar(CGM&: CGF.CGM);
2475
2476	emitTlsGuardCheck(CGF, TlsGuard, DynInitBB, ContinueBB);
2477	CGF.Builder.SetInsertPoint(DynInitBB);
2478	emitDynamicTlsInitializationCall(CGF, TlsGuard, ContinueBB);
2479	CGF.Builder.SetInsertPoint(ContinueBB);
2480	}
2481
2482	LValue MicrosoftCXXABI::EmitThreadLocalVarDeclLValue(CodeGenFunction &CGF,
2483	const VarDecl *VD,
2484	QualType LValType) {
2485	// Dynamic TLS initialization works by checking the state of a
2486	// guard variable (__tls_guard) to see whether TLS initialization
2487	// for a thread has happend yet.
2488	// If not, the initialization is triggered on-demand
2489	// by calling __dyn_tls_on_demand_init.
2490	emitDynamicTlsInitialization(CGF);
2491
2492	// Emit the variable just like any regular global variable.
2493
2494	llvm::Value *V = CGF.CGM.GetAddrOfGlobalVar(D: VD);
2495	llvm::Type *RealVarTy = CGF.getTypes().ConvertTypeForMem(T: VD->getType());
2496
2497	CharUnits Alignment = CGF.getContext().getDeclAlign(VD);
2498	Address Addr(V, RealVarTy, Alignment);
2499
2500	LValue LV = VD->getType()->isReferenceType()
2501	? CGF.EmitLoadOfReferenceLValue(Addr, VD->getType(),
2502	AlignmentSource::Decl)
2503	: CGF.MakeAddrLValue(Addr, T: LValType, Source: AlignmentSource::Decl);
2504	return LV;
2505	}
2506
2507	static ConstantAddress getInitThreadEpochPtr(CodeGenModule &CGM) {
2508	StringRef VarName("_Init_thread_epoch");
2509	CharUnits Align = CGM.getIntAlign();
2510	if (auto *GV = CGM.getModule().getNamedGlobal(Name: VarName))
2511	return ConstantAddress(GV, GV->getValueType(), Align);
2512	auto *GV = new llvm::GlobalVariable(
2513	CGM.getModule(), CGM.IntTy,
2514	/*isConstant=*/false, llvm::GlobalVariable::ExternalLinkage,
2515	/*Initializer=*/nullptr, VarName,
2516	/*InsertBefore=*/nullptr, llvm::GlobalVariable::GeneralDynamicTLSModel);
2517	GV->setAlignment(Align.getAsAlign());
2518	return ConstantAddress(GV, GV->getValueType(), Align);
2519	}
2520
2521	static llvm::FunctionCallee getInitThreadHeaderFn(CodeGenModule &CGM) {
2522	llvm::FunctionType *FTy =
2523	llvm::FunctionType::get(Result: llvm::Type::getVoidTy(C&: CGM.getLLVMContext()),
2524	Params: CGM.IntTy->getPointerTo(), /*isVarArg=*/false);
2525	return CGM.CreateRuntimeFunction(
2526	FTy, "_Init_thread_header",
2527	llvm::AttributeList::get(CGM.getLLVMContext(),
2528	llvm::AttributeList::FunctionIndex,
2529	llvm::Attribute::NoUnwind),
2530	/*Local=*/true);
2531	}
2532
2533	static llvm::FunctionCallee getInitThreadFooterFn(CodeGenModule &CGM) {
2534	llvm::FunctionType *FTy =
2535	llvm::FunctionType::get(Result: llvm::Type::getVoidTy(C&: CGM.getLLVMContext()),
2536	Params: CGM.IntTy->getPointerTo(), /*isVarArg=*/false);
2537	return CGM.CreateRuntimeFunction(
2538	FTy, "_Init_thread_footer",
2539	llvm::AttributeList::get(CGM.getLLVMContext(),
2540	llvm::AttributeList::FunctionIndex,
2541	llvm::Attribute::NoUnwind),
2542	/*Local=*/true);
2543	}
2544
2545	static llvm::FunctionCallee getInitThreadAbortFn(CodeGenModule &CGM) {
2546	llvm::FunctionType *FTy =
2547	llvm::FunctionType::get(Result: llvm::Type::getVoidTy(C&: CGM.getLLVMContext()),
2548	Params: CGM.IntTy->getPointerTo(), /*isVarArg=*/false);
2549	return CGM.CreateRuntimeFunction(
2550	FTy, "_Init_thread_abort",
2551	llvm::AttributeList::get(CGM.getLLVMContext(),
2552	llvm::AttributeList::FunctionIndex,
2553	llvm::Attribute::NoUnwind),
2554	/*Local=*/true);
2555	}
2556
2557	namespace {
2558	struct ResetGuardBit final : EHScopeStack::Cleanup {
2559	Address Guard;
2560	unsigned GuardNum;
2561	ResetGuardBit(Address Guard, unsigned GuardNum)
2562	: Guard(Guard), GuardNum(GuardNum) {}
2563
2564	void Emit(CodeGenFunction &CGF, Flags flags) override {
2565	// Reset the bit in the mask so that the static variable may be
2566	// reinitialized.
2567	CGBuilderTy &Builder = CGF.Builder;
2568	llvm::LoadInst *LI = Builder.CreateLoad(Addr: Guard);
2569	llvm::ConstantInt *Mask =
2570	llvm::ConstantInt::get(Ty: CGF.IntTy, V: ~(1ULL << GuardNum));
2571	Builder.CreateStore(Val: Builder.CreateAnd(LHS: LI, RHS: Mask), Addr: Guard);
2572	}
2573	};
2574
2575	struct CallInitThreadAbort final : EHScopeStack::Cleanup {
2576	llvm::Value *Guard;
2577	CallInitThreadAbort(RawAddress Guard) : Guard(Guard.getPointer()) {}
2578
2579	void Emit(CodeGenFunction &CGF, Flags flags) override {
2580	// Calling _Init_thread_abort will reset the guard's state.
2581	CGF.EmitNounwindRuntimeCall(callee: getInitThreadAbortFn(CGM&: CGF.CGM), args: Guard);
2582	}
2583	};
2584	}
2585
2586	void MicrosoftCXXABI::EmitGuardedInit(CodeGenFunction &CGF, const VarDecl &D,
2587	llvm::GlobalVariable *GV,
2588	bool PerformInit) {
2589	// MSVC only uses guards for static locals.
2590	if (!D.isStaticLocal()) {
2591	assert(GV->hasWeakLinkage() || GV->hasLinkOnceLinkage());
2592	// GlobalOpt is allowed to discard the initializer, so use linkonce_odr.
2593	llvm::Function *F = CGF.CurFn;
2594	F->setLinkage(llvm::GlobalValue::LinkOnceODRLinkage);
2595	F->setComdat(CGM.getModule().getOrInsertComdat(Name: F->getName()));
2596	CGF.EmitCXXGlobalVarDeclInit(D, GV, PerformInit);
2597	return;
2598	}
2599
2600	bool ThreadlocalStatic = D.getTLSKind();
2601	bool ThreadsafeStatic = getContext().getLangOpts().ThreadsafeStatics;
2602
2603	// Thread-safe static variables which aren't thread-specific have a
2604	// per-variable guard.
2605	bool HasPerVariableGuard = ThreadsafeStatic && !ThreadlocalStatic;
2606
2607	CGBuilderTy &Builder = CGF.Builder;
2608	llvm::IntegerType *GuardTy = CGF.Int32Ty;
2609	llvm::ConstantInt *Zero = llvm::ConstantInt::get(Ty: GuardTy, V: 0);
2610	CharUnits GuardAlign = CharUnits::fromQuantity(Quantity: 4);
2611
2612	// Get the guard variable for this function if we have one already.
2613	GuardInfo *GI = nullptr;
2614	if (ThreadlocalStatic)
2615	GI = &ThreadLocalGuardVariableMap[D.getDeclContext()];
2616	else if (!ThreadsafeStatic)
2617	GI = &GuardVariableMap[D.getDeclContext()];
2618
2619	llvm::GlobalVariable *GuardVar = GI ? GI->Guard : nullptr;
2620	unsigned GuardNum;
2621	if (D.isExternallyVisible()) {
2622	// Externally visible variables have to be numbered in Sema to properly
2623	// handle unreachable VarDecls.
2624	GuardNum = getContext().getStaticLocalNumber(VD: &D);
2625	assert(GuardNum > 0);
2626	GuardNum--;
2627	} else if (HasPerVariableGuard) {
2628	GuardNum = ThreadSafeGuardNumMap[D.getDeclContext()]++;
2629	} else {
2630	// Non-externally visible variables are numbered here in CodeGen.
2631	GuardNum = GI->BitIndex++;
2632	}
2633
2634	if (!HasPerVariableGuard && GuardNum >= 32) {
2635	if (D.isExternallyVisible())
2636	ErrorUnsupportedABI(CGF, S: "more than 32 guarded initializations");
2637	GuardNum %= 32;
2638	GuardVar = nullptr;
2639	}
2640
2641	if (!GuardVar) {
2642	// Mangle the name for the guard.
2643	SmallString<256> GuardName;
2644	{
2645	llvm::raw_svector_ostream Out(GuardName);
2646	if (HasPerVariableGuard)
2647	getMangleContext().mangleThreadSafeStaticGuardVariable(VD: &D, GuardNum,
2648	Out);
2649	else
2650	getMangleContext().mangleStaticGuardVariable(D: &D, Out);
2651	}
2652
2653	// Create the guard variable with a zero-initializer. Just absorb linkage,
2654	// visibility and dll storage class from the guarded variable.
2655	GuardVar =
2656	new llvm::GlobalVariable(CGM.getModule(), GuardTy, /*isConstant=*/false,
2657	GV->getLinkage(), Zero, GuardName.str());
2658	GuardVar->setVisibility(GV->getVisibility());
2659	GuardVar->setDLLStorageClass(GV->getDLLStorageClass());
2660	GuardVar->setAlignment(GuardAlign.getAsAlign());
2661	if (GuardVar->isWeakForLinker())
2662	GuardVar->setComdat(
2663	CGM.getModule().getOrInsertComdat(Name: GuardVar->getName()));
2664	if (D.getTLSKind())
2665	CGM.setTLSMode(GV: GuardVar, D);
2666	if (GI && !HasPerVariableGuard)
2667	GI->Guard = GuardVar;
2668	}
2669
2670	ConstantAddress GuardAddr(GuardVar, GuardTy, GuardAlign);
2671
2672	assert(GuardVar->getLinkage() == GV->getLinkage() &&
2673	"static local from the same function had different linkage");
2674
2675	if (!HasPerVariableGuard) {
2676	// Pseudo code for the test:
2677	// if (!(GuardVar & MyGuardBit)) {
2678	// GuardVar |= MyGuardBit;
2679	// ... initialize the object ...;
2680	// }
2681
2682	// Test our bit from the guard variable.
2683	llvm::ConstantInt *Bit = llvm::ConstantInt::get(Ty: GuardTy, V: 1ULL << GuardNum);
2684	llvm::LoadInst *LI = Builder.CreateLoad(Addr: GuardAddr);
2685	llvm::Value *NeedsInit =
2686	Builder.CreateICmpEQ(LHS: Builder.CreateAnd(LHS: LI, RHS: Bit), RHS: Zero);
2687	llvm::BasicBlock *InitBlock = CGF.createBasicBlock(name: "init");
2688	llvm::BasicBlock *EndBlock = CGF.createBasicBlock(name: "init.end");
2689	CGF.EmitCXXGuardedInitBranch(NeedsInit, InitBlock, NoInitBlock: EndBlock,
2690	Kind: CodeGenFunction::GuardKind::VariableGuard, D: &D);
2691
2692	// Set our bit in the guard variable and emit the initializer and add a global
2693	// destructor if appropriate.
2694	CGF.EmitBlock(BB: InitBlock);
2695	Builder.CreateStore(Val: Builder.CreateOr(LHS: LI, RHS: Bit), Addr: GuardAddr);
2696	CGF.EHStack.pushCleanup<ResetGuardBit>(Kind: EHCleanup, A: GuardAddr, A: GuardNum);
2697	CGF.EmitCXXGlobalVarDeclInit(D, GV, PerformInit);
2698	CGF.PopCleanupBlock();
2699	Builder.CreateBr(Dest: EndBlock);
2700
2701	// Continue.
2702	CGF.EmitBlock(BB: EndBlock);
2703	} else {
2704	// Pseudo code for the test:
2705	// if (TSS > _Init_thread_epoch) {
2706	// _Init_thread_header(&TSS);
2707	// if (TSS == -1) {
2708	// ... initialize the object ...;
2709	// _Init_thread_footer(&TSS);
2710	// }
2711	// }
2712	//
2713	// The algorithm is almost identical to what can be found in the appendix
2714	// found in N2325.
2715
2716	// This BasicBLock determines whether or not we have any work to do.
2717	llvm::LoadInst *FirstGuardLoad = Builder.CreateLoad(Addr: GuardAddr);
2718	FirstGuardLoad->setOrdering(llvm::AtomicOrdering::Unordered);
2719	llvm::LoadInst *InitThreadEpoch =
2720	Builder.CreateLoad(Addr: getInitThreadEpochPtr(CGM));
2721	llvm::Value *IsUninitialized =
2722	Builder.CreateICmpSGT(LHS: FirstGuardLoad, RHS: InitThreadEpoch);
2723	llvm::BasicBlock *AttemptInitBlock = CGF.createBasicBlock(name: "init.attempt");
2724	llvm::BasicBlock *EndBlock = CGF.createBasicBlock(name: "init.end");
2725	CGF.EmitCXXGuardedInitBranch(NeedsInit: IsUninitialized, InitBlock: AttemptInitBlock, NoInitBlock: EndBlock,
2726	Kind: CodeGenFunction::GuardKind::VariableGuard, D: &D);
2727
2728	// This BasicBlock attempts to determine whether or not this thread is
2729	// responsible for doing the initialization.
2730	CGF.EmitBlock(BB: AttemptInitBlock);
2731	CGF.EmitNounwindRuntimeCall(callee: getInitThreadHeaderFn(CGM),
2732	args: GuardAddr.getPointer());
2733	llvm::LoadInst *SecondGuardLoad = Builder.CreateLoad(Addr: GuardAddr);
2734	SecondGuardLoad->setOrdering(llvm::AtomicOrdering::Unordered);
2735	llvm::Value *ShouldDoInit =
2736	Builder.CreateICmpEQ(LHS: SecondGuardLoad, RHS: getAllOnesInt());
2737	llvm::BasicBlock *InitBlock = CGF.createBasicBlock(name: "init");
2738	Builder.CreateCondBr(Cond: ShouldDoInit, True: InitBlock, False: EndBlock);
2739
2740	// Ok, we ended up getting selected as the initializing thread.
2741	CGF.EmitBlock(BB: InitBlock);
2742	CGF.EHStack.pushCleanup<CallInitThreadAbort>(Kind: EHCleanup, A: GuardAddr);
2743	CGF.EmitCXXGlobalVarDeclInit(D, GV, PerformInit);
2744	CGF.PopCleanupBlock();
2745	CGF.EmitNounwindRuntimeCall(callee: getInitThreadFooterFn(CGM),
2746	args: GuardAddr.getPointer());
2747	Builder.CreateBr(Dest: EndBlock);
2748
2749	CGF.EmitBlock(BB: EndBlock);
2750	}
2751	}
2752
2753	bool MicrosoftCXXABI::isZeroInitializable(const MemberPointerType *MPT) {
2754	// Null-ness for function memptrs only depends on the first field, which is
2755	// the function pointer. The rest don't matter, so we can zero initialize.
2756	if (MPT->isMemberFunctionPointer())
2757	return true;
2758
2759	// The virtual base adjustment field is always -1 for null, so if we have one
2760	// we can't zero initialize. The field offset is sometimes also -1 if 0 is a
2761	// valid field offset.
2762	const CXXRecordDecl *RD = MPT->getMostRecentCXXRecordDecl();
2763	MSInheritanceModel Inheritance = RD->getMSInheritanceModel();
2764	return (!inheritanceModelHasVBTableOffsetField(Inheritance) &&
2765	RD->nullFieldOffsetIsZero());
2766	}
2767
2768	llvm::Type *
2769	MicrosoftCXXABI::ConvertMemberPointerType(const MemberPointerType *MPT) {
2770	const CXXRecordDecl *RD = MPT->getMostRecentCXXRecordDecl();
2771	MSInheritanceModel Inheritance = RD->getMSInheritanceModel();
2772	llvm::SmallVector<llvm::Type *, 4> fields;
2773	if (MPT->isMemberFunctionPointer())
2774	fields.push_back(Elt: CGM.VoidPtrTy); // FunctionPointerOrVirtualThunk
2775	else
2776	fields.push_back(Elt: CGM.IntTy); // FieldOffset
2777
2778	if (inheritanceModelHasNVOffsetField(IsMemberFunction: MPT->isMemberFunctionPointer(),
2779	Inheritance))
2780	fields.push_back(Elt: CGM.IntTy);
2781	if (inheritanceModelHasVBPtrOffsetField(Inheritance))
2782	fields.push_back(Elt: CGM.IntTy);
2783	if (inheritanceModelHasVBTableOffsetField(Inheritance))
2784	fields.push_back(Elt: CGM.IntTy); // VirtualBaseAdjustmentOffset
2785
2786	if (fields.size() == 1)
2787	return fields[0];
2788	return llvm::StructType::get(Context&: CGM.getLLVMContext(), Elements: fields);
2789	}
2790
2791	void MicrosoftCXXABI::
2792	GetNullMemberPointerFields(const MemberPointerType *MPT,
2793	llvm::SmallVectorImpl<llvm::Constant *> &fields) {
2794	assert(fields.empty());
2795	const CXXRecordDecl *RD = MPT->getMostRecentCXXRecordDecl();
2796	MSInheritanceModel Inheritance = RD->getMSInheritanceModel();
2797	if (MPT->isMemberFunctionPointer()) {
2798	// FunctionPointerOrVirtualThunk
2799	fields.push_back(Elt: llvm::Constant::getNullValue(Ty: CGM.VoidPtrTy));
2800	} else {
2801	if (RD->nullFieldOffsetIsZero())
2802	fields.push_back(Elt: getZeroInt()); // FieldOffset
2803	else
2804	fields.push_back(Elt: getAllOnesInt()); // FieldOffset
2805	}
2806
2807	if (inheritanceModelHasNVOffsetField(IsMemberFunction: MPT->isMemberFunctionPointer(),
2808	Inheritance))
2809	fields.push_back(Elt: getZeroInt());
2810	if (inheritanceModelHasVBPtrOffsetField(Inheritance))
2811	fields.push_back(Elt: getZeroInt());
2812	if (inheritanceModelHasVBTableOffsetField(Inheritance))
2813	fields.push_back(Elt: getAllOnesInt());
2814	}
2815
2816	llvm::Constant *
2817	MicrosoftCXXABI::EmitNullMemberPointer(const MemberPointerType *MPT) {
2818	llvm::SmallVector<llvm::Constant *, 4> fields;
2819	GetNullMemberPointerFields(MPT, fields);
2820	if (fields.size() == 1)
2821	return fields[0];
2822	llvm::Constant *Res = llvm::ConstantStruct::getAnon(V: fields);
2823	assert(Res->getType() == ConvertMemberPointerType(MPT));
2824	return Res;
2825	}
2826
2827	llvm::Constant *
2828	MicrosoftCXXABI::EmitFullMemberPointer(llvm::Constant *FirstField,
2829	bool IsMemberFunction,
2830	const CXXRecordDecl *RD,
2831	CharUnits NonVirtualBaseAdjustment,
2832	unsigned VBTableIndex) {
2833	MSInheritanceModel Inheritance = RD->getMSInheritanceModel();
2834
2835	// Single inheritance class member pointer are represented as scalars instead
2836	// of aggregates.
2837	if (inheritanceModelHasOnlyOneField(IsMemberFunction, Inheritance))
2838	return FirstField;
2839
2840	llvm::SmallVector<llvm::Constant *, 4> fields;
2841	fields.push_back(Elt: FirstField);
2842
2843	if (inheritanceModelHasNVOffsetField(IsMemberFunction, Inheritance))
2844	fields.push_back(Elt: llvm::ConstantInt::get(
2845	Ty: CGM.IntTy, V: NonVirtualBaseAdjustment.getQuantity()));
2846
2847	if (inheritanceModelHasVBPtrOffsetField(Inheritance)) {
2848	CharUnits Offs = CharUnits::Zero();
2849	if (VBTableIndex)
2850	Offs = getContext().getASTRecordLayout(RD).getVBPtrOffset();
2851	fields.push_back(Elt: llvm::ConstantInt::get(Ty: CGM.IntTy, V: Offs.getQuantity()));
2852	}
2853
2854	// The rest of the fields are adjusted by conversions to a more derived class.
2855	if (inheritanceModelHasVBTableOffsetField(Inheritance))
2856	fields.push_back(Elt: llvm::ConstantInt::get(Ty: CGM.IntTy, V: VBTableIndex));
2857
2858	return llvm::ConstantStruct::getAnon(V: fields);
2859	}
2860
2861	llvm::Constant *
2862	MicrosoftCXXABI::EmitMemberDataPointer(const MemberPointerType *MPT,
2863	CharUnits offset) {
2864	return EmitMemberDataPointer(RD: MPT->getMostRecentCXXRecordDecl(), offset);
2865	}
2866
2867	llvm::Constant *MicrosoftCXXABI::EmitMemberDataPointer(const CXXRecordDecl *RD,
2868	CharUnits offset) {
2869	if (RD->getMSInheritanceModel() ==
2870	MSInheritanceModel::Virtual)
2871	offset -= getContext().getOffsetOfBaseWithVBPtr(RD);
2872	llvm::Constant *FirstField =
2873	llvm::ConstantInt::get(Ty: CGM.IntTy, V: offset.getQuantity());
2874	return EmitFullMemberPointer(FirstField, /*IsMemberFunction=*/false, RD,
2875	NonVirtualBaseAdjustment: CharUnits::Zero(), /*VBTableIndex=*/0);
2876	}
2877
2878	llvm::Constant *MicrosoftCXXABI::EmitMemberPointer(const APValue &MP,
2879	QualType MPType) {
2880	const MemberPointerType *DstTy = MPType->castAs<MemberPointerType>();
2881	const ValueDecl *MPD = MP.getMemberPointerDecl();
2882	if (!MPD)
2883	return EmitNullMemberPointer(MPT: DstTy);
2884
2885	ASTContext &Ctx = getContext();
2886	ArrayRef<const CXXRecordDecl *> MemberPointerPath = MP.getMemberPointerPath();
2887
2888	llvm::Constant *C;
2889	if (const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(Val: MPD)) {
2890	C = EmitMemberFunctionPointer(MD);
2891	} else {
2892	// For a pointer to data member, start off with the offset of the field in
2893	// the class in which it was declared, and convert from there if necessary.
2894	// For indirect field decls, get the outermost anonymous field and use the
2895	// parent class.
2896	CharUnits FieldOffset = Ctx.toCharUnitsFromBits(BitSize: Ctx.getFieldOffset(FD: MPD));
2897	const FieldDecl *FD = dyn_cast<FieldDecl>(Val: MPD);
2898	if (!FD)
2899	FD = cast<FieldDecl>(Val: *cast<IndirectFieldDecl>(Val: MPD)->chain_begin());
2900	const CXXRecordDecl *RD = cast<CXXRecordDecl>(Val: FD->getParent());
2901	RD = RD->getMostRecentNonInjectedDecl();
2902	C = EmitMemberDataPointer(RD, offset: FieldOffset);
2903	}
2904
2905	if (!MemberPointerPath.empty()) {
2906	const CXXRecordDecl *SrcRD = cast<CXXRecordDecl>(MPD->getDeclContext());
2907	const Type *SrcRecTy = Ctx.getTypeDeclType(SrcRD).getTypePtr();
2908	const MemberPointerType *SrcTy =
2909	Ctx.getMemberPointerType(T: DstTy->getPointeeType(), Cls: SrcRecTy)
2910	->castAs<MemberPointerType>();
2911
2912	bool DerivedMember = MP.isMemberPointerToDerivedMember();
2913	SmallVector<const CXXBaseSpecifier *, 4> DerivedToBasePath;
2914	const CXXRecordDecl *PrevRD = SrcRD;
2915	for (const CXXRecordDecl *PathElem : MemberPointerPath) {
2916	const CXXRecordDecl *Base = nullptr;
2917	const CXXRecordDecl *Derived = nullptr;
2918	if (DerivedMember) {
2919	Base = PathElem;
2920	Derived = PrevRD;
2921	} else {
2922	Base = PrevRD;
2923	Derived = PathElem;
2924	}
2925	for (const CXXBaseSpecifier &BS : Derived->bases())
2926	if (BS.getType()->getAsCXXRecordDecl()->getCanonicalDecl() ==
2927	Base->getCanonicalDecl())
2928	DerivedToBasePath.push_back(Elt: &BS);
2929	PrevRD = PathElem;
2930	}
2931	assert(DerivedToBasePath.size() == MemberPointerPath.size());
2932
2933	CastKind CK = DerivedMember ? CK_DerivedToBaseMemberPointer
2934	: CK_BaseToDerivedMemberPointer;
2935	C = EmitMemberPointerConversion(SrcTy, DstTy, CK, PathBegin: DerivedToBasePath.begin(),
2936	PathEnd: DerivedToBasePath.end(), Src: C);
2937	}
2938	return C;
2939	}
2940
2941	llvm::Constant *
2942	MicrosoftCXXABI::EmitMemberFunctionPointer(const CXXMethodDecl *MD) {
2943	assert(MD->isInstance() && "Member function must not be static!");
2944
2945	CharUnits NonVirtualBaseAdjustment = CharUnits::Zero();
2946	const CXXRecordDecl *RD = MD->getParent()->getMostRecentNonInjectedDecl();
2947	CodeGenTypes &Types = CGM.getTypes();
2948
2949	unsigned VBTableIndex = 0;
2950	llvm::Constant *FirstField;
2951	const FunctionProtoType *FPT = MD->getType()->castAs<FunctionProtoType>();
2952	if (!MD->isVirtual()) {
2953	llvm::Type *Ty;
2954	// Check whether the function has a computable LLVM signature.
2955	if (Types.isFuncTypeConvertible(FPT)) {
2956	// The function has a computable LLVM signature; use the correct type.
2957	Ty = Types.GetFunctionType(Info: Types.arrangeCXXMethodDeclaration(MD));
2958	} else {
2959	// Use an arbitrary non-function type to tell GetAddrOfFunction that the
2960	// function type is incomplete.
2961	Ty = CGM.PtrDiffTy;
2962	}
2963	FirstField = CGM.GetAddrOfFunction(MD, Ty);
2964	} else {
2965	auto &VTableContext = CGM.getMicrosoftVTableContext();
2966	MethodVFTableLocation ML = VTableContext.getMethodVFTableLocation(MD);
2967	FirstField = EmitVirtualMemPtrThunk(MD, ML);
2968	// Include the vfptr adjustment if the method is in a non-primary vftable.
2969	NonVirtualBaseAdjustment += ML.VFPtrOffset;
2970	if (ML.VBase)
2971	VBTableIndex = VTableContext.getVBTableIndex(Derived: RD, VBase: ML.VBase) * 4;
2972	}
2973
2974	if (VBTableIndex == 0 &&
2975	RD->getMSInheritanceModel() ==
2976	MSInheritanceModel::Virtual)
2977	NonVirtualBaseAdjustment -= getContext().getOffsetOfBaseWithVBPtr(RD);
2978
2979	// The rest of the fields are common with data member pointers.
2980	return EmitFullMemberPointer(FirstField, /*IsMemberFunction=*/true, RD,
2981	NonVirtualBaseAdjustment, VBTableIndex);
2982	}
2983
2984	/// Member pointers are the same if they're either bitwise identical *or* both
2985	/// null. Null-ness for function members is determined by the first field,
2986	/// while for data member pointers we must compare all fields.
2987	llvm::Value *
2988	MicrosoftCXXABI::EmitMemberPointerComparison(CodeGenFunction &CGF,
2989	llvm::Value *L,
2990	llvm::Value *R,
2991	const MemberPointerType *MPT,
2992	bool Inequality) {
2993	CGBuilderTy &Builder = CGF.Builder;
2994
2995	// Handle != comparisons by switching the sense of all boolean operations.
2996	llvm::ICmpInst::Predicate Eq;
2997	llvm::Instruction::BinaryOps And, Or;
2998	if (Inequality) {
2999	Eq = llvm::ICmpInst::ICMP_NE;
3000	And = llvm::Instruction::Or;
3001	Or = llvm::Instruction::And;
3002	} else {
3003	Eq = llvm::ICmpInst::ICMP_EQ;
3004	And = llvm::Instruction::And;
3005	Or = llvm::Instruction::Or;
3006	}
3007
3008	// If this is a single field member pointer (single inheritance), this is a
3009	// single icmp.
3010	const CXXRecordDecl *RD = MPT->getMostRecentCXXRecordDecl();
3011	MSInheritanceModel Inheritance = RD->getMSInheritanceModel();
3012	if (inheritanceModelHasOnlyOneField(IsMemberFunction: MPT->isMemberFunctionPointer(),
3013	Inheritance))
3014	return Builder.CreateICmp(P: Eq, LHS: L, RHS: R);
3015
3016	// Compare the first field.
3017	llvm::Value *L0 = Builder.CreateExtractValue(Agg: L, Idxs: 0, Name: "lhs.0");
3018	llvm::Value *R0 = Builder.CreateExtractValue(Agg: R, Idxs: 0, Name: "rhs.0");
3019	llvm::Value *Cmp0 = Builder.CreateICmp(P: Eq, LHS: L0, RHS: R0, Name: "memptr.cmp.first");
3020
3021	// Compare everything other than the first field.
3022	llvm::Value *Res = nullptr;
3023	llvm::StructType *LType = cast<llvm::StructType>(Val: L->getType());
3024	for (unsigned I = 1, E = LType->getNumElements(); I != E; ++I) {
3025	llvm::Value *LF = Builder.CreateExtractValue(Agg: L, Idxs: I);
3026	llvm::Value *RF = Builder.CreateExtractValue(Agg: R, Idxs: I);
3027	llvm::Value *Cmp = Builder.CreateICmp(P: Eq, LHS: LF, RHS: RF, Name: "memptr.cmp.rest");
3028	if (Res)
3029	Res = Builder.CreateBinOp(Opc: And, LHS: Res, RHS: Cmp);
3030	else
3031	Res = Cmp;
3032	}
3033
3034	// Check if the first field is 0 if this is a function pointer.
3035	if (MPT->isMemberFunctionPointer()) {
3036	// (l1 == r1 && ...) || l0 == 0
3037	llvm::Value *Zero = llvm::Constant::getNullValue(Ty: L0->getType());
3038	llvm::Value *IsZero = Builder.CreateICmp(P: Eq, LHS: L0, RHS: Zero, Name: "memptr.cmp.iszero");
3039	Res = Builder.CreateBinOp(Opc: Or, LHS: Res, RHS: IsZero);
3040	}
3041
3042	// Combine the comparison of the first field, which must always be true for
3043	// this comparison to succeeed.
3044	return Builder.CreateBinOp(Opc: And, LHS: Res, RHS: Cmp0, Name: "memptr.cmp");
3045	}
3046
3047	llvm::Value *
3048	MicrosoftCXXABI::EmitMemberPointerIsNotNull(CodeGenFunction &CGF,
3049	llvm::Value *MemPtr,
3050	const MemberPointerType *MPT) {
3051	CGBuilderTy &Builder = CGF.Builder;
3052	llvm::SmallVector<llvm::Constant *, 4> fields;
3053	// We only need one field for member functions.
3054	if (MPT->isMemberFunctionPointer())
3055	fields.push_back(Elt: llvm::Constant::getNullValue(Ty: CGM.VoidPtrTy));
3056	else
3057	GetNullMemberPointerFields(MPT, fields);
3058	assert(!fields.empty());
3059	llvm::Value *FirstField = MemPtr;
3060	if (MemPtr->getType()->isStructTy())
3061	FirstField = Builder.CreateExtractValue(Agg: MemPtr, Idxs: 0);
3062	llvm::Value *Res = Builder.CreateICmpNE(LHS: FirstField, RHS: fields[0], Name: "memptr.cmp0");
3063
3064	// For function member pointers, we only need to test the function pointer
3065	// field. The other fields if any can be garbage.
3066	if (MPT->isMemberFunctionPointer())
3067	return Res;
3068
3069	// Otherwise, emit a series of compares and combine the results.
3070	for (int I = 1, E = fields.size(); I < E; ++I) {
3071	llvm::Value *Field = Builder.CreateExtractValue(Agg: MemPtr, Idxs: I);
3072	llvm::Value *Next = Builder.CreateICmpNE(LHS: Field, RHS: fields[I], Name: "memptr.cmp");
3073	Res = Builder.CreateOr(LHS: Res, RHS: Next, Name: "memptr.tobool");
3074	}
3075	return Res;
3076	}
3077
3078	bool MicrosoftCXXABI::MemberPointerConstantIsNull(const MemberPointerType *MPT,
3079	llvm::Constant *Val) {
3080	// Function pointers are null if the pointer in the first field is null.
3081	if (MPT->isMemberFunctionPointer()) {
3082	llvm::Constant *FirstField = Val->getType()->isStructTy() ?
3083	Val->getAggregateElement(Elt: 0U) : Val;
3084	return FirstField->isNullValue();
3085	}
3086
3087	// If it's not a function pointer and it's zero initializable, we can easily
3088	// check zero.
3089	if (isZeroInitializable(MPT) && Val->isNullValue())
3090	return true;
3091
3092	// Otherwise, break down all the fields for comparison. Hopefully these
3093	// little Constants are reused, while a big null struct might not be.
3094	llvm::SmallVector<llvm::Constant *, 4> Fields;
3095	GetNullMemberPointerFields(MPT, fields&: Fields);
3096	if (Fields.size() == 1) {
3097	assert(Val->getType()->isIntegerTy());
3098	return Val == Fields[0];
3099	}
3100
3101	unsigned I, E;
3102	for (I = 0, E = Fields.size(); I != E; ++I) {
3103	if (Val->getAggregateElement(Elt: I) != Fields[I])
3104	break;
3105	}
3106	return I == E;
3107	}
3108
3109	llvm::Value *
3110	MicrosoftCXXABI::GetVBaseOffsetFromVBPtr(CodeGenFunction &CGF,
3111	Address This,
3112	llvm::Value *VBPtrOffset,
3113	llvm::Value *VBTableOffset,
3114	llvm::Value **VBPtrOut) {
3115	CGBuilderTy &Builder = CGF.Builder;
3116	// Load the vbtable pointer from the vbptr in the instance.
3117	llvm::Value *VBPtr = Builder.CreateInBoundsGEP(
3118	Ty: CGM.Int8Ty, Ptr: This.emitRawPointer(CGF), IdxList: VBPtrOffset, Name: "vbptr");
3119	if (VBPtrOut)
3120	*VBPtrOut = VBPtr;
3121
3122	CharUnits VBPtrAlign;
3123	if (auto CI = dyn_cast<llvm::ConstantInt>(Val: VBPtrOffset)) {
3124	VBPtrAlign = This.getAlignment().alignmentAtOffset(
3125	offset: CharUnits::fromQuantity(Quantity: CI->getSExtValue()));
3126	} else {
3127	VBPtrAlign = CGF.getPointerAlign();
3128	}
3129
3130	llvm::Value *VBTable = Builder.CreateAlignedLoad(
3131	Ty: CGM.Int32Ty->getPointerTo(AddrSpace: 0), Addr: VBPtr, Align: VBPtrAlign, Name: "vbtable");
3132
3133	// Translate from byte offset to table index. It improves analyzability.
3134	llvm::Value *VBTableIndex = Builder.CreateAShr(
3135	LHS: VBTableOffset, RHS: llvm::ConstantInt::get(Ty: VBTableOffset->getType(), V: 2),
3136	Name: "vbtindex", /*isExact=*/true);
3137
3138	// Load an i32 offset from the vb-table.
3139	llvm::Value *VBaseOffs =
3140	Builder.CreateInBoundsGEP(Ty: CGM.Int32Ty, Ptr: VBTable, IdxList: VBTableIndex);
3141	return Builder.CreateAlignedLoad(Ty: CGM.Int32Ty, Addr: VBaseOffs,
3142	Align: CharUnits::fromQuantity(Quantity: 4), Name: "vbase_offs");
3143	}
3144
3145	// Returns an adjusted base cast to i8*, since we do more address arithmetic on
3146	// it.
3147	llvm::Value *MicrosoftCXXABI::AdjustVirtualBase(
3148	CodeGenFunction &CGF, const Expr *E, const CXXRecordDecl *RD,
3149	Address Base, llvm::Value *VBTableOffset, llvm::Value *VBPtrOffset) {
3150	CGBuilderTy &Builder = CGF.Builder;
3151	Base = Base.withElementType(ElemTy: CGM.Int8Ty);
3152	llvm::BasicBlock *OriginalBB = nullptr;
3153	llvm::BasicBlock *SkipAdjustBB = nullptr;
3154	llvm::BasicBlock *VBaseAdjustBB = nullptr;
3155
3156	// In the unspecified inheritance model, there might not be a vbtable at all,
3157	// in which case we need to skip the virtual base lookup. If there is a
3158	// vbtable, the first entry is a no-op entry that gives back the original
3159	// base, so look for a virtual base adjustment offset of zero.
3160	if (VBPtrOffset) {
3161	OriginalBB = Builder.GetInsertBlock();
3162	VBaseAdjustBB = CGF.createBasicBlock(name: "memptr.vadjust");
3163	SkipAdjustBB = CGF.createBasicBlock(name: "memptr.skip_vadjust");
3164	llvm::Value *IsVirtual =
3165	Builder.CreateICmpNE(LHS: VBTableOffset, RHS: getZeroInt(),
3166	Name: "memptr.is_vbase");
3167	Builder.CreateCondBr(Cond: IsVirtual, True: VBaseAdjustBB, False: SkipAdjustBB);
3168	CGF.EmitBlock(BB: VBaseAdjustBB);
3169	}
3170
3171	// If we weren't given a dynamic vbptr offset, RD should be complete and we'll
3172	// know the vbptr offset.
3173	if (!VBPtrOffset) {
3174	CharUnits offs = CharUnits::Zero();
3175	if (!RD->hasDefinition()) {
3176	DiagnosticsEngine &Diags = CGF.CGM.getDiags();
3177	unsigned DiagID = Diags.getCustomDiagID(
3178	L: DiagnosticsEngine::Error,
3179	FormatString: "member pointer representation requires a "
3180	"complete class type for %0 to perform this expression");
3181	Diags.Report(Loc: E->getExprLoc(), DiagID) << RD << E->getSourceRange();
3182	} else if (RD->getNumVBases())
3183	offs = getContext().getASTRecordLayout(RD).getVBPtrOffset();
3184	VBPtrOffset = llvm::ConstantInt::get(Ty: CGM.IntTy, V: offs.getQuantity());
3185	}
3186	llvm::Value *VBPtr = nullptr;
3187	llvm::Value *VBaseOffs =
3188	GetVBaseOffsetFromVBPtr(CGF, This: Base, VBPtrOffset, VBTableOffset, VBPtrOut: &VBPtr);
3189	llvm::Value *AdjustedBase =
3190	Builder.CreateInBoundsGEP(Ty: CGM.Int8Ty, Ptr: VBPtr, IdxList: VBaseOffs);
3191
3192	// Merge control flow with the case where we didn't have to adjust.
3193	if (VBaseAdjustBB) {
3194	Builder.CreateBr(Dest: SkipAdjustBB);
3195	CGF.EmitBlock(BB: SkipAdjustBB);
3196	llvm::PHINode *Phi = Builder.CreatePHI(Ty: CGM.Int8PtrTy, NumReservedValues: 2, Name: "memptr.base");
3197	Phi->addIncoming(V: Base.emitRawPointer(CGF), BB: OriginalBB);
3198	Phi->addIncoming(V: AdjustedBase, BB: VBaseAdjustBB);
3199	return Phi;
3200	}
3201	return AdjustedBase;
3202	}
3203
3204	llvm::Value *MicrosoftCXXABI::EmitMemberDataPointerAddress(
3205	CodeGenFunction &CGF, const Expr *E, Address Base, llvm::Value *MemPtr,
3206	const MemberPointerType *MPT) {
3207	assert(MPT->isMemberDataPointer());
3208	CGBuilderTy &Builder = CGF.Builder;
3209	const CXXRecordDecl *RD = MPT->getMostRecentCXXRecordDecl();
3210	MSInheritanceModel Inheritance = RD->getMSInheritanceModel();
3211
3212	// Extract the fields we need, regardless of model. We'll apply them if we
3213	// have them.
3214	llvm::Value *FieldOffset = MemPtr;
3215	llvm::Value *VirtualBaseAdjustmentOffset = nullptr;
3216	llvm::Value *VBPtrOffset = nullptr;
3217	if (MemPtr->getType()->isStructTy()) {
3218	// We need to extract values.
3219	unsigned I = 0;
3220	FieldOffset = Builder.CreateExtractValue(Agg: MemPtr, Idxs: I++);
3221	if (inheritanceModelHasVBPtrOffsetField(Inheritance))
3222	VBPtrOffset = Builder.CreateExtractValue(Agg: MemPtr, Idxs: I++);
3223	if (inheritanceModelHasVBTableOffsetField(Inheritance))
3224	VirtualBaseAdjustmentOffset = Builder.CreateExtractValue(Agg: MemPtr, Idxs: I++);
3225	}
3226
3227	llvm::Value *Addr;
3228	if (VirtualBaseAdjustmentOffset) {
3229	Addr = AdjustVirtualBase(CGF, E, RD, Base, VBTableOffset: VirtualBaseAdjustmentOffset,
3230	VBPtrOffset);
3231	} else {
3232	Addr = Base.emitRawPointer(CGF);
3233	}
3234
3235	// Apply the offset, which we assume is non-null.
3236	return Builder.CreateInBoundsGEP(Ty: CGF.Int8Ty, Ptr: Addr, IdxList: FieldOffset,
3237	Name: "memptr.offset");
3238	}
3239
3240	llvm::Value *
3241	MicrosoftCXXABI::EmitMemberPointerConversion(CodeGenFunction &CGF,
3242	const CastExpr *E,
3243	llvm::Value *Src) {
3244	assert(E->getCastKind() == CK_DerivedToBaseMemberPointer ||
3245	E->getCastKind() == CK_BaseToDerivedMemberPointer ||
3246	E->getCastKind() == CK_ReinterpretMemberPointer);
3247
3248	// Use constant emission if we can.
3249	if (isa<llvm::Constant>(Val: Src))
3250	return EmitMemberPointerConversion(E, Src: cast<llvm::Constant>(Val: Src));
3251
3252	// We may be adding or dropping fields from the member pointer, so we need
3253	// both types and the inheritance models of both records.
3254	const MemberPointerType *SrcTy =
3255	E->getSubExpr()->getType()->castAs<MemberPointerType>();
3256	const MemberPointerType *DstTy = E->getType()->castAs<MemberPointerType>();
3257	bool IsFunc = SrcTy->isMemberFunctionPointer();
3258
3259	// If the classes use the same null representation, reinterpret_cast is a nop.
3260	bool IsReinterpret = E->getCastKind() == CK_ReinterpretMemberPointer;
3261	if (IsReinterpret && IsFunc)
3262	return Src;
3263
3264	CXXRecordDecl *SrcRD = SrcTy->getMostRecentCXXRecordDecl();
3265	CXXRecordDecl *DstRD = DstTy->getMostRecentCXXRecordDecl();
3266	if (IsReinterpret &&
3267	SrcRD->nullFieldOffsetIsZero() == DstRD->nullFieldOffsetIsZero())
3268	return Src;
3269
3270	CGBuilderTy &Builder = CGF.Builder;
3271
3272	// Branch past the conversion if Src is null.
3273	llvm::Value *IsNotNull = EmitMemberPointerIsNotNull(CGF, MemPtr: Src, MPT: SrcTy);
3274	llvm::Constant *DstNull = EmitNullMemberPointer(MPT: DstTy);
3275
3276	// C++ 5.2.10p9: The null member pointer value is converted to the null member
3277	// pointer value of the destination type.
3278	if (IsReinterpret) {
3279	// For reinterpret casts, sema ensures that src and dst are both functions
3280	// or data and have the same size, which means the LLVM types should match.
3281	assert(Src->getType() == DstNull->getType());
3282	return Builder.CreateSelect(C: IsNotNull, True: Src, False: DstNull);
3283	}
3284
3285	llvm::BasicBlock *OriginalBB = Builder.GetInsertBlock();
3286	llvm::BasicBlock *ConvertBB = CGF.createBasicBlock(name: "memptr.convert");
3287	llvm::BasicBlock *ContinueBB = CGF.createBasicBlock(name: "memptr.converted");
3288	Builder.CreateCondBr(Cond: IsNotNull, True: ConvertBB, False: ContinueBB);
3289	CGF.EmitBlock(BB: ConvertBB);
3290
3291	llvm::Value *Dst = EmitNonNullMemberPointerConversion(
3292	SrcTy, DstTy, CK: E->getCastKind(), PathBegin: E->path_begin(), PathEnd: E->path_end(), Src,
3293	Builder);
3294
3295	Builder.CreateBr(Dest: ContinueBB);
3296
3297	// In the continuation, choose between DstNull and Dst.
3298	CGF.EmitBlock(BB: ContinueBB);
3299	llvm::PHINode *Phi = Builder.CreatePHI(Ty: DstNull->getType(), NumReservedValues: 2, Name: "memptr.converted");
3300	Phi->addIncoming(V: DstNull, BB: OriginalBB);
3301	Phi->addIncoming(V: Dst, BB: ConvertBB);
3302	return Phi;
3303	}
3304
3305	llvm::Value *MicrosoftCXXABI::EmitNonNullMemberPointerConversion(
3306	const MemberPointerType *SrcTy, const MemberPointerType *DstTy, CastKind CK,
3307	CastExpr::path_const_iterator PathBegin,
3308	CastExpr::path_const_iterator PathEnd, llvm::Value *Src,
3309	CGBuilderTy &Builder) {
3310	const CXXRecordDecl *SrcRD = SrcTy->getMostRecentCXXRecordDecl();
3311	const CXXRecordDecl *DstRD = DstTy->getMostRecentCXXRecordDecl();
3312	MSInheritanceModel SrcInheritance = SrcRD->getMSInheritanceModel();
3313	MSInheritanceModel DstInheritance = DstRD->getMSInheritanceModel();
3314	bool IsFunc = SrcTy->isMemberFunctionPointer();
3315	bool IsConstant = isa<llvm::Constant>(Val: Src);
3316
3317	// Decompose src.
3318	llvm::Value *FirstField = Src;
3319	llvm::Value *NonVirtualBaseAdjustment = getZeroInt();
3320	llvm::Value *VirtualBaseAdjustmentOffset = getZeroInt();
3321	llvm::Value *VBPtrOffset = getZeroInt();
3322	if (!inheritanceModelHasOnlyOneField(IsMemberFunction: IsFunc, Inheritance: SrcInheritance)) {
3323	// We need to extract values.
3324	unsigned I = 0;
3325	FirstField = Builder.CreateExtractValue(Agg: Src, Idxs: I++);
3326	if (inheritanceModelHasNVOffsetField(IsMemberFunction: IsFunc, Inheritance: SrcInheritance))
3327	NonVirtualBaseAdjustment = Builder.CreateExtractValue(Agg: Src, Idxs: I++);
3328	if (inheritanceModelHasVBPtrOffsetField(Inheritance: SrcInheritance))
3329	VBPtrOffset = Builder.CreateExtractValue(Agg: Src, Idxs: I++);
3330	if (inheritanceModelHasVBTableOffsetField(Inheritance: SrcInheritance))
3331	VirtualBaseAdjustmentOffset = Builder.CreateExtractValue(Agg: Src, Idxs: I++);
3332	}
3333
3334	bool IsDerivedToBase = (CK == CK_DerivedToBaseMemberPointer);
3335	const MemberPointerType *DerivedTy = IsDerivedToBase ? SrcTy : DstTy;
3336	const CXXRecordDecl *DerivedClass = DerivedTy->getMostRecentCXXRecordDecl();
3337
3338	// For data pointers, we adjust the field offset directly. For functions, we
3339	// have a separate field.
3340	llvm::Value *&NVAdjustField = IsFunc ? NonVirtualBaseAdjustment : FirstField;
3341
3342	// The virtual inheritance model has a quirk: the virtual base table is always
3343	// referenced when dereferencing a member pointer even if the member pointer
3344	// is non-virtual. This is accounted for by adjusting the non-virtual offset
3345	// to point backwards to the top of the MDC from the first VBase. Undo this
3346	// adjustment to normalize the member pointer.
3347	llvm::Value *SrcVBIndexEqZero =
3348	Builder.CreateICmpEQ(LHS: VirtualBaseAdjustmentOffset, RHS: getZeroInt());
3349	if (SrcInheritance == MSInheritanceModel::Virtual) {
3350	if (int64_t SrcOffsetToFirstVBase =
3351	getContext().getOffsetOfBaseWithVBPtr(RD: SrcRD).getQuantity()) {
3352	llvm::Value *UndoSrcAdjustment = Builder.CreateSelect(
3353	C: SrcVBIndexEqZero,
3354	True: llvm::ConstantInt::get(Ty: CGM.IntTy, V: SrcOffsetToFirstVBase),
3355	False: getZeroInt());
3356	NVAdjustField = Builder.CreateNSWAdd(LHS: NVAdjustField, RHS: UndoSrcAdjustment);
3357	}
3358	}
3359
3360	// A non-zero vbindex implies that we are dealing with a source member in a
3361	// floating virtual base in addition to some non-virtual offset. If the
3362	// vbindex is zero, we are dealing with a source that exists in a non-virtual,
3363	// fixed, base. The difference between these two cases is that the vbindex +
3364	// nvoffset *always* point to the member regardless of what context they are
3365	// evaluated in so long as the vbindex is adjusted. A member inside a fixed
3366	// base requires explicit nv adjustment.
3367	llvm::Constant *BaseClassOffset = llvm::ConstantInt::get(
3368	Ty: CGM.IntTy,
3369	V: CGM.computeNonVirtualBaseClassOffset(DerivedClass, Start: PathBegin, End: PathEnd)
3370	.getQuantity());
3371
3372	llvm::Value *NVDisp;
3373	if (IsDerivedToBase)
3374	NVDisp = Builder.CreateNSWSub(LHS: NVAdjustField, RHS: BaseClassOffset, Name: "adj");
3375	else
3376	NVDisp = Builder.CreateNSWAdd(LHS: NVAdjustField, RHS: BaseClassOffset, Name: "adj");
3377
3378	NVAdjustField = Builder.CreateSelect(C: SrcVBIndexEqZero, True: NVDisp, False: getZeroInt());
3379
3380	// Update the vbindex to an appropriate value in the destination because
3381	// SrcRD's vbtable might not be a strict prefix of the one in DstRD.
3382	llvm::Value *DstVBIndexEqZero = SrcVBIndexEqZero;
3383	if (inheritanceModelHasVBTableOffsetField(Inheritance: DstInheritance) &&
3384	inheritanceModelHasVBTableOffsetField(Inheritance: SrcInheritance)) {
3385	if (llvm::GlobalVariable *VDispMap =
3386	getAddrOfVirtualDisplacementMap(SrcRD, DstRD)) {
3387	llvm::Value *VBIndex = Builder.CreateExactUDiv(
3388	LHS: VirtualBaseAdjustmentOffset, RHS: llvm::ConstantInt::get(Ty: CGM.IntTy, V: 4));
3389	if (IsConstant) {
3390	llvm::Constant *Mapping = VDispMap->getInitializer();
3391	VirtualBaseAdjustmentOffset =
3392	Mapping->getAggregateElement(Elt: cast<llvm::Constant>(Val: VBIndex));
3393	} else {
3394	llvm::Value *Idxs[] = {getZeroInt(), VBIndex};
3395	VirtualBaseAdjustmentOffset = Builder.CreateAlignedLoad(
3396	Ty: CGM.IntTy, Addr: Builder.CreateInBoundsGEP(Ty: VDispMap->getValueType(),
3397	Ptr: VDispMap, IdxList: Idxs),
3398	Align: CharUnits::fromQuantity(Quantity: 4));
3399	}
3400
3401	DstVBIndexEqZero =
3402	Builder.CreateICmpEQ(LHS: VirtualBaseAdjustmentOffset, RHS: getZeroInt());
3403	}
3404	}
3405
3406	// Set the VBPtrOffset to zero if the vbindex is zero. Otherwise, initialize
3407	// it to the offset of the vbptr.
3408	if (inheritanceModelHasVBPtrOffsetField(Inheritance: DstInheritance)) {
3409	llvm::Value *DstVBPtrOffset = llvm::ConstantInt::get(
3410	Ty: CGM.IntTy,
3411	V: getContext().getASTRecordLayout(DstRD).getVBPtrOffset().getQuantity());
3412	VBPtrOffset =
3413	Builder.CreateSelect(C: DstVBIndexEqZero, True: getZeroInt(), False: DstVBPtrOffset);
3414	}
3415
3416	// Likewise, apply a similar adjustment so that dereferencing the member
3417	// pointer correctly accounts for the distance between the start of the first
3418	// virtual base and the top of the MDC.
3419	if (DstInheritance == MSInheritanceModel::Virtual) {
3420	if (int64_t DstOffsetToFirstVBase =
3421	getContext().getOffsetOfBaseWithVBPtr(RD: DstRD).getQuantity()) {
3422	llvm::Value *DoDstAdjustment = Builder.CreateSelect(
3423	C: DstVBIndexEqZero,
3424	True: llvm::ConstantInt::get(Ty: CGM.IntTy, V: DstOffsetToFirstVBase),
3425	False: getZeroInt());
3426	NVAdjustField = Builder.CreateNSWSub(LHS: NVAdjustField, RHS: DoDstAdjustment);
3427	}
3428	}
3429
3430	// Recompose dst from the null struct and the adjusted fields from src.
3431	llvm::Value *Dst;
3432	if (inheritanceModelHasOnlyOneField(IsMemberFunction: IsFunc, Inheritance: DstInheritance)) {
3433	Dst = FirstField;
3434	} else {
3435	Dst = llvm::UndefValue::get(T: ConvertMemberPointerType(MPT: DstTy));
3436	unsigned Idx = 0;
3437	Dst = Builder.CreateInsertValue(Agg: Dst, Val: FirstField, Idxs: Idx++);
3438	if (inheritanceModelHasNVOffsetField(IsMemberFunction: IsFunc, Inheritance: DstInheritance))
3439	Dst = Builder.CreateInsertValue(Agg: Dst, Val: NonVirtualBaseAdjustment, Idxs: Idx++);
3440	if (inheritanceModelHasVBPtrOffsetField(Inheritance: DstInheritance))
3441	Dst = Builder.CreateInsertValue(Agg: Dst, Val: VBPtrOffset, Idxs: Idx++);
3442	if (inheritanceModelHasVBTableOffsetField(Inheritance: DstInheritance))
3443	Dst = Builder.CreateInsertValue(Agg: Dst, Val: VirtualBaseAdjustmentOffset, Idxs: Idx++);
3444	}
3445	return Dst;
3446	}
3447
3448	llvm::Constant *
3449	MicrosoftCXXABI::EmitMemberPointerConversion(const CastExpr *E,
3450	llvm::Constant *Src) {
3451	const MemberPointerType *SrcTy =
3452	E->getSubExpr()->getType()->castAs<MemberPointerType>();
3453	const MemberPointerType *DstTy = E->getType()->castAs<MemberPointerType>();
3454
3455	CastKind CK = E->getCastKind();
3456
3457	return EmitMemberPointerConversion(SrcTy, DstTy, CK, PathBegin: E->path_begin(),
3458	PathEnd: E->path_end(), Src);
3459	}
3460
3461	llvm::Constant *MicrosoftCXXABI::EmitMemberPointerConversion(
3462	const MemberPointerType *SrcTy, const MemberPointerType *DstTy, CastKind CK,
3463	CastExpr::path_const_iterator PathBegin,
3464	CastExpr::path_const_iterator PathEnd, llvm::Constant *Src) {
3465	assert(CK == CK_DerivedToBaseMemberPointer ||
3466	CK == CK_BaseToDerivedMemberPointer ||
3467	CK == CK_ReinterpretMemberPointer);
3468	// If src is null, emit a new null for dst. We can't return src because dst
3469	// might have a new representation.
3470	if (MemberPointerConstantIsNull(MPT: SrcTy, Val: Src))
3471	return EmitNullMemberPointer(MPT: DstTy);
3472
3473	// We don't need to do anything for reinterpret_casts of non-null member
3474	// pointers. We should only get here when the two type representations have
3475	// the same size.
3476	if (CK == CK_ReinterpretMemberPointer)
3477	return Src;
3478
3479	CGBuilderTy Builder(CGM, CGM.getLLVMContext());
3480	auto *Dst = cast<llvm::Constant>(Val: EmitNonNullMemberPointerConversion(
3481	SrcTy, DstTy, CK, PathBegin, PathEnd, Src, Builder));
3482
3483	return Dst;
3484	}
3485
3486	CGCallee MicrosoftCXXABI::EmitLoadOfMemberFunctionPointer(
3487	CodeGenFunction &CGF, const Expr *E, Address This,
3488	llvm::Value *&ThisPtrForCall, llvm::Value *MemPtr,
3489	const MemberPointerType *MPT) {
3490	assert(MPT->isMemberFunctionPointer());
3491	const FunctionProtoType *FPT =
3492	MPT->getPointeeType()->castAs<FunctionProtoType>();
3493	const CXXRecordDecl *RD = MPT->getMostRecentCXXRecordDecl();
3494	CGBuilderTy &Builder = CGF.Builder;
3495
3496	MSInheritanceModel Inheritance = RD->getMSInheritanceModel();
3497
3498	// Extract the fields we need, regardless of model. We'll apply them if we
3499	// have them.
3500	llvm::Value *FunctionPointer = MemPtr;
3501	llvm::Value *NonVirtualBaseAdjustment = nullptr;
3502	llvm::Value *VirtualBaseAdjustmentOffset = nullptr;
3503	llvm::Value *VBPtrOffset = nullptr;
3504	if (MemPtr->getType()->isStructTy()) {
3505	// We need to extract values.
3506	unsigned I = 0;
3507	FunctionPointer = Builder.CreateExtractValue(Agg: MemPtr, Idxs: I++);
3508	if (inheritanceModelHasNVOffsetField(IsMemberFunction: MPT, Inheritance))
3509	NonVirtualBaseAdjustment = Builder.CreateExtractValue(Agg: MemPtr, Idxs: I++);
3510	if (inheritanceModelHasVBPtrOffsetField(Inheritance))
3511	VBPtrOffset = Builder.CreateExtractValue(Agg: MemPtr, Idxs: I++);
3512	if (inheritanceModelHasVBTableOffsetField(Inheritance))
3513	VirtualBaseAdjustmentOffset = Builder.CreateExtractValue(Agg: MemPtr, Idxs: I++);
3514	}
3515
3516	if (VirtualBaseAdjustmentOffset) {
3517	ThisPtrForCall = AdjustVirtualBase(CGF, E, RD, Base: This,
3518	VBTableOffset: VirtualBaseAdjustmentOffset, VBPtrOffset);
3519	} else {
3520	ThisPtrForCall = This.emitRawPointer(CGF);
3521	}
3522
3523	if (NonVirtualBaseAdjustment)
3524	ThisPtrForCall = Builder.CreateInBoundsGEP(Ty: CGF.Int8Ty, Ptr: ThisPtrForCall,
3525	IdxList: NonVirtualBaseAdjustment);
3526
3527	CGCallee Callee(FPT, FunctionPointer);
3528	return Callee;
3529	}
3530
3531	CGCXXABI *clang::CodeGen::CreateMicrosoftCXXABI(CodeGenModule &CGM) {
3532	return new MicrosoftCXXABI(CGM);
3533	}
3534
3535	// MS RTTI Overview:
3536	// The run time type information emitted by cl.exe contains 5 distinct types of
3537	// structures. Many of them reference each other.
3538	//
3539	// TypeInfo: Static classes that are returned by typeid.
3540	//
3541	// CompleteObjectLocator: Referenced by vftables. They contain information
3542	// required for dynamic casting, including OffsetFromTop. They also contain
3543	// a reference to the TypeInfo for the type and a reference to the
3544	// CompleteHierarchyDescriptor for the type.
3545	//
3546	// ClassHierarchyDescriptor: Contains information about a class hierarchy.
3547	// Used during dynamic_cast to walk a class hierarchy. References a base
3548	// class array and the size of said array.
3549	//
3550	// BaseClassArray: Contains a list of classes in a hierarchy. BaseClassArray is
3551	// somewhat of a misnomer because the most derived class is also in the list
3552	// as well as multiple copies of virtual bases (if they occur multiple times
3553	// in the hierarchy.) The BaseClassArray contains one BaseClassDescriptor for
3554	// every path in the hierarchy, in pre-order depth first order. Note, we do
3555	// not declare a specific llvm type for BaseClassArray, it's merely an array
3556	// of BaseClassDescriptor pointers.
3557	//
3558	// BaseClassDescriptor: Contains information about a class in a class hierarchy.
3559	// BaseClassDescriptor is also somewhat of a misnomer for the same reason that
3560	// BaseClassArray is. It contains information about a class within a
3561	// hierarchy such as: is this base is ambiguous and what is its offset in the
3562	// vbtable. The names of the BaseClassDescriptors have all of their fields
3563	// mangled into them so they can be aggressively deduplicated by the linker.
3564
3565	static llvm::GlobalVariable *getTypeInfoVTable(CodeGenModule &CGM) {
3566	StringRef MangledName("??_7type_info@@6B@");
3567	if (auto VTable = CGM.getModule().getNamedGlobal(Name: MangledName))
3568	return VTable;
3569	return new llvm::GlobalVariable(CGM.getModule(), CGM.Int8PtrTy,
3570	/*isConstant=*/true,
3571	llvm::GlobalVariable::ExternalLinkage,
3572	/*Initializer=*/nullptr, MangledName);
3573	}
3574
3575	namespace {
3576
3577	/// A Helper struct that stores information about a class in a class
3578	/// hierarchy. The information stored in these structs struct is used during
3579	/// the generation of ClassHierarchyDescriptors and BaseClassDescriptors.
3580	// During RTTI creation, MSRTTIClasses are stored in a contiguous array with
3581	// implicit depth first pre-order tree connectivity. getFirstChild and
3582	// getNextSibling allow us to walk the tree efficiently.
3583	struct MSRTTIClass {
3584	enum {
3585	IsPrivateOnPath = 1 | 8,
3586	IsAmbiguous = 2,
3587	IsPrivate = 4,
3588	IsVirtual = 16,
3589	HasHierarchyDescriptor = 64
3590	};
3591	MSRTTIClass(const CXXRecordDecl *RD) : RD(RD) {}
3592	uint32_t initialize(const MSRTTIClass *Parent,
3593	const CXXBaseSpecifier *Specifier);
3594
3595	MSRTTIClass *getFirstChild() { return this + 1; }
3596	static MSRTTIClass *getNextChild(MSRTTIClass *Child) {
3597	return Child + 1 + Child->NumBases;
3598	}
3599
3600	const CXXRecordDecl *RD, *VirtualRoot;
3601	uint32_t Flags, NumBases, OffsetInVBase;
3602	};
3603
3604	/// Recursively initialize the base class array.
3605	uint32_t MSRTTIClass::initialize(const MSRTTIClass *Parent,
3606	const CXXBaseSpecifier *Specifier) {
3607	Flags = HasHierarchyDescriptor;
3608	if (!Parent) {
3609	VirtualRoot = nullptr;
3610	OffsetInVBase = 0;
3611	} else {
3612	if (Specifier->getAccessSpecifier() != AS_public)
3613	Flags |= IsPrivate | IsPrivateOnPath;
3614	if (Specifier->isVirtual()) {
3615	Flags |= IsVirtual;
3616	VirtualRoot = RD;
3617	OffsetInVBase = 0;
3618	} else {
3619	if (Parent->Flags & IsPrivateOnPath)
3620	Flags |= IsPrivateOnPath;
3621	VirtualRoot = Parent->VirtualRoot;
3622	OffsetInVBase = Parent->OffsetInVBase + RD->getASTContext()
3623	.getASTRecordLayout(Parent->RD).getBaseClassOffset(RD).getQuantity();
3624	}
3625	}
3626	NumBases = 0;
3627	MSRTTIClass *Child = getFirstChild();
3628	for (const CXXBaseSpecifier &Base : RD->bases()) {
3629	NumBases += Child->initialize(Parent: this, Specifier: &Base) + 1;
3630	Child = getNextChild(Child);
3631	}
3632	return NumBases;
3633	}
3634
3635	static llvm::GlobalValue::LinkageTypes getLinkageForRTTI(QualType Ty) {
3636	switch (Ty->getLinkage()) {
3637	case Linkage::Invalid:
3638	llvm_unreachable("Linkage hasn't been computed!");
3639
3640	case Linkage::None:
3641	case Linkage::Internal:
3642	case Linkage::UniqueExternal:
3643	return llvm::GlobalValue::InternalLinkage;
3644
3645	case Linkage::VisibleNone:
3646	case Linkage::Module:
3647	case Linkage::External:
3648	return llvm::GlobalValue::LinkOnceODRLinkage;
3649	}
3650	llvm_unreachable("Invalid linkage!");
3651	}
3652
3653	/// An ephemeral helper class for building MS RTTI types. It caches some
3654	/// calls to the module and information about the most derived class in a
3655	/// hierarchy.
3656	struct MSRTTIBuilder {
3657	enum {
3658	HasBranchingHierarchy = 1,
3659	HasVirtualBranchingHierarchy = 2,
3660	HasAmbiguousBases = 4
3661	};
3662
3663	MSRTTIBuilder(MicrosoftCXXABI &ABI, const CXXRecordDecl *RD)
3664	: CGM(ABI.CGM), Context(CGM.getContext()),
3665	VMContext(CGM.getLLVMContext()), Module(CGM.getModule()), RD(RD),
3666	Linkage(getLinkageForRTTI(Ty: CGM.getContext().getTagDeclType(RD))),
3667	ABI(ABI) {}
3668
3669	llvm::GlobalVariable *getBaseClassDescriptor(const MSRTTIClass &Classes);
3670	llvm::GlobalVariable *
3671	getBaseClassArray(SmallVectorImpl<MSRTTIClass> &Classes);
3672	llvm::GlobalVariable *getClassHierarchyDescriptor();
3673	llvm::GlobalVariable *getCompleteObjectLocator(const VPtrInfo &Info);
3674
3675	CodeGenModule &CGM;
3676	ASTContext &Context;
3677	llvm::LLVMContext &VMContext;
3678	llvm::Module &Module;
3679	const CXXRecordDecl *RD;
3680	llvm::GlobalVariable::LinkageTypes Linkage;
3681	MicrosoftCXXABI &ABI;
3682	};
3683
3684	} // namespace
3685
3686	/// Recursively serializes a class hierarchy in pre-order depth first
3687	/// order.
3688	static void serializeClassHierarchy(SmallVectorImpl<MSRTTIClass> &Classes,
3689	const CXXRecordDecl *RD) {
3690	Classes.push_back(Elt: MSRTTIClass(RD));
3691	for (const CXXBaseSpecifier &Base : RD->bases())
3692	serializeClassHierarchy(Classes, RD: Base.getType()->getAsCXXRecordDecl());
3693	}
3694
3695	/// Find ambiguity among base classes.
3696	static void
3697	detectAmbiguousBases(SmallVectorImpl<MSRTTIClass> &Classes) {
3698	llvm::SmallPtrSet<const CXXRecordDecl *, 8> VirtualBases;
3699	llvm::SmallPtrSet<const CXXRecordDecl *, 8> UniqueBases;
3700	llvm::SmallPtrSet<const CXXRecordDecl *, 8> AmbiguousBases;
3701	for (MSRTTIClass *Class = &Classes.front(); Class <= &Classes.back();) {
3702	if ((Class->Flags & MSRTTIClass::IsVirtual) &&
3703	!VirtualBases.insert(Ptr: Class->RD).second) {
3704	Class = MSRTTIClass::getNextChild(Child: Class);
3705	continue;
3706	}
3707	if (!UniqueBases.insert(Ptr: Class->RD).second)
3708	AmbiguousBases.insert(Ptr: Class->RD);
3709	Class++;
3710	}
3711	if (AmbiguousBases.empty())
3712	return;
3713	for (MSRTTIClass &Class : Classes)
3714	if (AmbiguousBases.count(Ptr: Class.RD))
3715	Class.Flags |= MSRTTIClass::IsAmbiguous;
3716	}
3717
3718	llvm::GlobalVariable *MSRTTIBuilder::getClassHierarchyDescriptor() {
3719	SmallString<256> MangledName;
3720	{
3721	llvm::raw_svector_ostream Out(MangledName);
3722	ABI.getMangleContext().mangleCXXRTTIClassHierarchyDescriptor(Derived: RD, Out);
3723	}
3724
3725	// Check to see if we've already declared this ClassHierarchyDescriptor.
3726	if (auto CHD = Module.getNamedGlobal(Name: MangledName))
3727	return CHD;
3728
3729	// Serialize the class hierarchy and initialize the CHD Fields.
3730	SmallVector<MSRTTIClass, 8> Classes;
3731	serializeClassHierarchy(Classes, RD);
3732	Classes.front().initialize(/*Parent=*/nullptr, /*Specifier=*/nullptr);
3733	detectAmbiguousBases(Classes);
3734	int Flags = 0;
3735	for (const MSRTTIClass &Class : Classes) {
3736	if (Class.RD->getNumBases() > 1)
3737	Flags |= HasBranchingHierarchy;
3738	// Note: cl.exe does not calculate "HasAmbiguousBases" correctly. We
3739	// believe the field isn't actually used.
3740	if (Class.Flags & MSRTTIClass::IsAmbiguous)
3741	Flags |= HasAmbiguousBases;
3742	}
3743	if ((Flags & HasBranchingHierarchy) && RD->getNumVBases() != 0)
3744	Flags |= HasVirtualBranchingHierarchy;
3745	// These gep indices are used to get the address of the first element of the
3746	// base class array.
3747	llvm::Value *GEPIndices[] = {llvm::ConstantInt::get(Ty: CGM.IntTy, V: 0),
3748	llvm::ConstantInt::get(Ty: CGM.IntTy, V: 0)};
3749
3750	// Forward-declare the class hierarchy descriptor
3751	auto Type = ABI.getClassHierarchyDescriptorType();
3752	auto CHD = new llvm::GlobalVariable(Module, Type, /*isConstant=*/true, Linkage,
3753	/*Initializer=*/nullptr,
3754	MangledName);
3755	if (CHD->isWeakForLinker())
3756	CHD->setComdat(CGM.getModule().getOrInsertComdat(Name: CHD->getName()));
3757
3758	auto *Bases = getBaseClassArray(Classes);
3759
3760	// Initialize the base class ClassHierarchyDescriptor.
3761	llvm::Constant *Fields[] = {
3762	llvm::ConstantInt::get(Ty: CGM.IntTy, V: 0), // reserved by the runtime
3763	llvm::ConstantInt::get(Ty: CGM.IntTy, V: Flags),
3764	llvm::ConstantInt::get(Ty: CGM.IntTy, V: Classes.size()),
3765	ABI.getImageRelativeConstant(PtrVal: llvm::ConstantExpr::getInBoundsGetElementPtr(
3766	Ty: Bases->getValueType(), C: Bases,
3767	IdxList: llvm::ArrayRef<llvm::Value *>(GEPIndices))),
3768	};
3769	CHD->setInitializer(llvm::ConstantStruct::get(T: Type, V: Fields));
3770	return CHD;
3771	}
3772
3773	llvm::GlobalVariable *
3774	MSRTTIBuilder::getBaseClassArray(SmallVectorImpl<MSRTTIClass> &Classes) {
3775	SmallString<256> MangledName;
3776	{
3777	llvm::raw_svector_ostream Out(MangledName);
3778	ABI.getMangleContext().mangleCXXRTTIBaseClassArray(Derived: RD, Out);
3779	}
3780
3781	// Forward-declare the base class array.
3782	// cl.exe pads the base class array with 1 (in 32 bit mode) or 4 (in 64 bit
3783	// mode) bytes of padding. We provide a pointer sized amount of padding by
3784	// adding +1 to Classes.size(). The sections have pointer alignment and are
3785	// marked pick-any so it shouldn't matter.
3786	llvm::Type *PtrType = ABI.getImageRelativeType(
3787	PtrType: ABI.getBaseClassDescriptorType()->getPointerTo());
3788	auto *ArrType = llvm::ArrayType::get(ElementType: PtrType, NumElements: Classes.size() + 1);
3789	auto *BCA =
3790	new llvm::GlobalVariable(Module, ArrType,
3791	/*isConstant=*/true, Linkage,
3792	/*Initializer=*/nullptr, MangledName);
3793	if (BCA->isWeakForLinker())
3794	BCA->setComdat(CGM.getModule().getOrInsertComdat(Name: BCA->getName()));
3795
3796	// Initialize the BaseClassArray.
3797	SmallVector<llvm::Constant *, 8> BaseClassArrayData;
3798	for (MSRTTIClass &Class : Classes)
3799	BaseClassArrayData.push_back(
3800	Elt: ABI.getImageRelativeConstant(PtrVal: getBaseClassDescriptor(Classes: Class)));
3801	BaseClassArrayData.push_back(Elt: llvm::Constant::getNullValue(Ty: PtrType));
3802	BCA->setInitializer(llvm::ConstantArray::get(T: ArrType, V: BaseClassArrayData));
3803	return BCA;
3804	}
3805
3806	llvm::GlobalVariable *
3807	MSRTTIBuilder::getBaseClassDescriptor(const MSRTTIClass &Class) {
3808	// Compute the fields for the BaseClassDescriptor. They are computed up front
3809	// because they are mangled into the name of the object.
3810	uint32_t OffsetInVBTable = 0;
3811	int32_t VBPtrOffset = -1;
3812	if (Class.VirtualRoot) {
3813	auto &VTableContext = CGM.getMicrosoftVTableContext();
3814	OffsetInVBTable = VTableContext.getVBTableIndex(Derived: RD, VBase: Class.VirtualRoot) * 4;
3815	VBPtrOffset = Context.getASTRecordLayout(RD).getVBPtrOffset().getQuantity();
3816	}
3817
3818	SmallString<256> MangledName;
3819	{
3820	llvm::raw_svector_ostream Out(MangledName);
3821	ABI.getMangleContext().mangleCXXRTTIBaseClassDescriptor(
3822	Derived: Class.RD, NVOffset: Class.OffsetInVBase, VBPtrOffset, VBTableOffset: OffsetInVBTable,
3823	Flags: Class.Flags, Out);
3824	}
3825
3826	// Check to see if we've already declared this object.
3827	if (auto BCD = Module.getNamedGlobal(Name: MangledName))
3828	return BCD;
3829
3830	// Forward-declare the base class descriptor.
3831	auto Type = ABI.getBaseClassDescriptorType();
3832	auto BCD =
3833	new llvm::GlobalVariable(Module, Type, /*isConstant=*/true, Linkage,
3834	/*Initializer=*/nullptr, MangledName);
3835	if (BCD->isWeakForLinker())
3836	BCD->setComdat(CGM.getModule().getOrInsertComdat(Name: BCD->getName()));
3837
3838	// Initialize the BaseClassDescriptor.
3839	llvm::Constant *Fields[] = {
3840	ABI.getImageRelativeConstant(
3841	PtrVal: ABI.getAddrOfRTTIDescriptor(Ty: Context.getTypeDeclType(Class.RD))),
3842	llvm::ConstantInt::get(Ty: CGM.IntTy, V: Class.NumBases),
3843	llvm::ConstantInt::get(Ty: CGM.IntTy, V: Class.OffsetInVBase),
3844	llvm::ConstantInt::get(Ty: CGM.IntTy, V: VBPtrOffset),
3845	llvm::ConstantInt::get(Ty: CGM.IntTy, V: OffsetInVBTable),
3846	llvm::ConstantInt::get(Ty: CGM.IntTy, V: Class.Flags),
3847	ABI.getImageRelativeConstant(
3848	PtrVal: MSRTTIBuilder(ABI, Class.RD).getClassHierarchyDescriptor()),
3849	};
3850	BCD->setInitializer(llvm::ConstantStruct::get(T: Type, V: Fields));
3851	return BCD;
3852	}
3853
3854	llvm::GlobalVariable *
3855	MSRTTIBuilder::getCompleteObjectLocator(const VPtrInfo &Info) {
3856	SmallString<256> MangledName;
3857	{
3858	llvm::raw_svector_ostream Out(MangledName);
3859	ABI.getMangleContext().mangleCXXRTTICompleteObjectLocator(Derived: RD, BasePath: Info.MangledPath, Out);
3860	}
3861
3862	// Check to see if we've already computed this complete object locator.
3863	if (auto COL = Module.getNamedGlobal(Name: MangledName))
3864	return COL;
3865
3866	// Compute the fields of the complete object locator.
3867	int OffsetToTop = Info.FullOffsetInMDC.getQuantity();
3868	int VFPtrOffset = 0;
3869	// The offset includes the vtordisp if one exists.
3870	if (const CXXRecordDecl *VBase = Info.getVBaseWithVPtr())
3871	if (Context.getASTRecordLayout(RD)
3872	.getVBaseOffsetsMap()
3873	.find(Val: VBase)
3874	->second.hasVtorDisp())
3875	VFPtrOffset = Info.NonVirtualOffset.getQuantity() + 4;
3876
3877	// Forward-declare the complete object locator.
3878	llvm::StructType *Type = ABI.getCompleteObjectLocatorType();
3879	auto COL = new llvm::GlobalVariable(Module, Type, /*isConstant=*/true, Linkage,
3880	/*Initializer=*/nullptr, MangledName);
3881
3882	// Initialize the CompleteObjectLocator.
3883	llvm::Constant *Fields[] = {
3884	llvm::ConstantInt::get(Ty: CGM.IntTy, V: ABI.isImageRelative()),
3885	llvm::ConstantInt::get(Ty: CGM.IntTy, V: OffsetToTop),
3886	llvm::ConstantInt::get(Ty: CGM.IntTy, V: VFPtrOffset),
3887	ABI.getImageRelativeConstant(
3888	PtrVal: CGM.GetAddrOfRTTIDescriptor(Ty: Context.getTypeDeclType(RD))),
3889	ABI.getImageRelativeConstant(PtrVal: getClassHierarchyDescriptor()),
3890	ABI.getImageRelativeConstant(PtrVal: COL),
3891	};
3892	llvm::ArrayRef<llvm::Constant *> FieldsRef(Fields);
3893	if (!ABI.isImageRelative())
3894	FieldsRef = FieldsRef.drop_back();
3895	COL->setInitializer(llvm::ConstantStruct::get(T: Type, V: FieldsRef));
3896	if (COL->isWeakForLinker())
3897	COL->setComdat(CGM.getModule().getOrInsertComdat(Name: COL->getName()));
3898	return COL;
3899	}
3900
3901	static QualType decomposeTypeForEH(ASTContext &Context, QualType T,
3902	bool &IsConst, bool &IsVolatile,
3903	bool &IsUnaligned) {
3904	T = Context.getExceptionObjectType(T);
3905
3906	// C++14 [except.handle]p3:
3907	// A handler is a match for an exception object of type E if [...]
3908	// - the handler is of type cv T or const T& where T is a pointer type and
3909	// E is a pointer type that can be converted to T by [...]
3910	// - a qualification conversion
3911	IsConst = false;
3912	IsVolatile = false;
3913	IsUnaligned = false;
3914	QualType PointeeType = T->getPointeeType();
3915	if (!PointeeType.isNull()) {
3916	IsConst = PointeeType.isConstQualified();
3917	IsVolatile = PointeeType.isVolatileQualified();
3918	IsUnaligned = PointeeType.getQualifiers().hasUnaligned();
3919	}
3920
3921	// Member pointer types like "const int A::*" are represented by having RTTI
3922	// for "int A::*" and separately storing the const qualifier.
3923	if (const auto *MPTy = T->getAs<MemberPointerType>())
3924	T = Context.getMemberPointerType(T: PointeeType.getUnqualifiedType(),
3925	Cls: MPTy->getClass());
3926
3927	// Pointer types like "const int * const *" are represented by having RTTI
3928	// for "const int **" and separately storing the const qualifier.
3929	if (T->isPointerType())
3930	T = Context.getPointerType(T: PointeeType.getUnqualifiedType());
3931
3932	return T;
3933	}
3934
3935	CatchTypeInfo
3936	MicrosoftCXXABI::getAddrOfCXXCatchHandlerType(QualType Type,
3937	QualType CatchHandlerType) {
3938	// TypeDescriptors for exceptions never have qualified pointer types,
3939	// qualifiers are stored separately in order to support qualification
3940	// conversions.
3941	bool IsConst, IsVolatile, IsUnaligned;
3942	Type =
3943	decomposeTypeForEH(Context&: getContext(), T: Type, IsConst, IsVolatile, IsUnaligned);
3944
3945	bool IsReference = CatchHandlerType->isReferenceType();
3946
3947	uint32_t Flags = 0;
3948	if (IsConst)
3949	Flags |= 1;
3950	if (IsVolatile)
3951	Flags |= 2;
3952	if (IsUnaligned)
3953	Flags |= 4;
3954	if (IsReference)
3955	Flags |= 8;
3956
3957	return CatchTypeInfo{.RTTI: getAddrOfRTTIDescriptor(Ty: Type)->stripPointerCasts(),
3958	.Flags: Flags};
3959	}
3960
3961	/// Gets a TypeDescriptor. Returns a llvm::Constant * rather than a
3962	/// llvm::GlobalVariable * because different type descriptors have different
3963	/// types, and need to be abstracted. They are abstracting by casting the
3964	/// address to an Int8PtrTy.
3965	llvm::Constant *MicrosoftCXXABI::getAddrOfRTTIDescriptor(QualType Type) {
3966	SmallString<256> MangledName;
3967	{
3968	llvm::raw_svector_ostream Out(MangledName);
3969	getMangleContext().mangleCXXRTTI(T: Type, Out);
3970	}
3971
3972	// Check to see if we've already declared this TypeDescriptor.
3973	if (llvm::GlobalVariable *GV = CGM.getModule().getNamedGlobal(Name: MangledName))
3974	return GV;
3975
3976	// Note for the future: If we would ever like to do deferred emission of
3977	// RTTI, check if emitting vtables opportunistically need any adjustment.
3978
3979	// Compute the fields for the TypeDescriptor.
3980	SmallString<256> TypeInfoString;
3981	{
3982	llvm::raw_svector_ostream Out(TypeInfoString);
3983	getMangleContext().mangleCXXRTTIName(T: Type, Out);
3984	}
3985
3986	// Declare and initialize the TypeDescriptor.
3987	llvm::Constant *Fields[] = {
3988	getTypeInfoVTable(CGM), // VFPtr
3989	llvm::ConstantPointerNull::get(T: CGM.Int8PtrTy), // Runtime data
3990	llvm::ConstantDataArray::getString(Context&: CGM.getLLVMContext(), Initializer: TypeInfoString)};
3991	llvm::StructType *TypeDescriptorType =
3992	getTypeDescriptorType(TypeInfoString);
3993	auto *Var = new llvm::GlobalVariable(
3994	CGM.getModule(), TypeDescriptorType, /*isConstant=*/false,
3995	getLinkageForRTTI(Ty: Type),
3996	llvm::ConstantStruct::get(T: TypeDescriptorType, V: Fields),
3997	MangledName);
3998	if (Var->isWeakForLinker())
3999	Var->setComdat(CGM.getModule().getOrInsertComdat(Name: Var->getName()));
4000	return Var;
4001	}
4002
4003	/// Gets or a creates a Microsoft CompleteObjectLocator.
4004	llvm::GlobalVariable *
4005	MicrosoftCXXABI::getMSCompleteObjectLocator(const CXXRecordDecl *RD,
4006	const VPtrInfo &Info) {
4007	return MSRTTIBuilder(*this, RD).getCompleteObjectLocator(Info);
4008	}
4009
4010	void MicrosoftCXXABI::emitCXXStructor(GlobalDecl GD) {
4011	if (auto *ctor = dyn_cast<CXXConstructorDecl>(Val: GD.getDecl())) {
4012	// There are no constructor variants, always emit the complete destructor.
4013	llvm::Function *Fn =
4014	CGM.codegenCXXStructor(GD: GD.getWithCtorType(Type: Ctor_Complete));
4015	CGM.maybeSetTrivialComdat(*ctor, *Fn);
4016	return;
4017	}
4018
4019	auto *dtor = cast<CXXDestructorDecl>(Val: GD.getDecl());
4020
4021	// Emit the base destructor if the base and complete (vbase) destructors are
4022	// equivalent. This effectively implements -mconstructor-aliases as part of
4023	// the ABI.
4024	if (GD.getDtorType() == Dtor_Complete &&
4025	dtor->getParent()->getNumVBases() == 0)
4026	GD = GD.getWithDtorType(Type: Dtor_Base);
4027
4028	// The base destructor is equivalent to the base destructor of its
4029	// base class if there is exactly one non-virtual base class with a
4030	// non-trivial destructor, there are no fields with a non-trivial
4031	// destructor, and the body of the destructor is trivial.
4032	if (GD.getDtorType() == Dtor_Base && !CGM.TryEmitBaseDestructorAsAlias(D: dtor))
4033	return;
4034
4035	llvm::Function *Fn = CGM.codegenCXXStructor(GD);
4036	if (Fn->isWeakForLinker())
4037	Fn->setComdat(CGM.getModule().getOrInsertComdat(Name: Fn->getName()));
4038	}
4039
4040	llvm::Function *
4041	MicrosoftCXXABI::getAddrOfCXXCtorClosure(const CXXConstructorDecl *CD,
4042	CXXCtorType CT) {
4043	assert(CT == Ctor_CopyingClosure || CT == Ctor_DefaultClosure);
4044
4045	// Calculate the mangled name.
4046	SmallString<256> ThunkName;
4047	llvm::raw_svector_ostream Out(ThunkName);
4048	getMangleContext().mangleName(GD: GlobalDecl(CD, CT), Out);
4049
4050	// If the thunk has been generated previously, just return it.
4051	if (llvm::GlobalValue *GV = CGM.getModule().getNamedValue(Name: ThunkName))
4052	return cast<llvm::Function>(Val: GV);
4053
4054	// Create the llvm::Function.
4055	const CGFunctionInfo &FnInfo = CGM.getTypes().arrangeMSCtorClosure(CD, CT);
4056	llvm::FunctionType *ThunkTy = CGM.getTypes().GetFunctionType(Info: FnInfo);
4057	const CXXRecordDecl *RD = CD->getParent();
4058	QualType RecordTy = getContext().getRecordType(RD);
4059	llvm::Function *ThunkFn = llvm::Function::Create(
4060	Ty: ThunkTy, Linkage: getLinkageForRTTI(Ty: RecordTy), N: ThunkName.str(), M: &CGM.getModule());
4061	ThunkFn->setCallingConv(static_cast<llvm::CallingConv::ID>(
4062	FnInfo.getEffectiveCallingConvention()));
4063	if (ThunkFn->isWeakForLinker())
4064	ThunkFn->setComdat(CGM.getModule().getOrInsertComdat(Name: ThunkFn->getName()));
4065	bool IsCopy = CT == Ctor_CopyingClosure;
4066
4067	// Start codegen.
4068	CodeGenFunction CGF(CGM);
4069	CGF.CurGD = GlobalDecl(CD, Ctor_Complete);
4070
4071	// Build FunctionArgs.
4072	FunctionArgList FunctionArgs;
4073
4074	// A constructor always starts with a 'this' pointer as its first argument.
4075	buildThisParam(CGF, Params&: FunctionArgs);
4076
4077	// Following the 'this' pointer is a reference to the source object that we
4078	// are copying from.
4079	ImplicitParamDecl SrcParam(
4080	getContext(), /*DC=*/nullptr, SourceLocation(),
4081	&getContext().Idents.get(Name: "src"),
4082	getContext().getLValueReferenceType(T: RecordTy,
4083	/*SpelledAsLValue=*/true),
4084	ImplicitParamKind::Other);
4085	if (IsCopy)
4086	FunctionArgs.push_back(&SrcParam);
4087
4088	// Constructors for classes which utilize virtual bases have an additional
4089	// parameter which indicates whether or not it is being delegated to by a more
4090	// derived constructor.
4091	ImplicitParamDecl IsMostDerived(getContext(), /*DC=*/nullptr,
4092	SourceLocation(),
4093	&getContext().Idents.get(Name: "is_most_derived"),
4094	getContext().IntTy, ImplicitParamKind::Other);
4095	// Only add the parameter to the list if the class has virtual bases.
4096	if (RD->getNumVBases() > 0)
4097	FunctionArgs.push_back(&IsMostDerived);
4098
4099	// Start defining the function.
4100	auto NL = ApplyDebugLocation::CreateEmpty(CGF);
4101	CGF.StartFunction(GD: GlobalDecl(), RetTy: FnInfo.getReturnType(), Fn: ThunkFn, FnInfo,
4102	Args: FunctionArgs, Loc: CD->getLocation(), StartLoc: SourceLocation());
4103	// Create a scope with an artificial location for the body of this function.
4104	auto AL = ApplyDebugLocation::CreateArtificial(CGF);
4105	setCXXABIThisValue(CGF, ThisPtr: loadIncomingCXXThis(CGF));
4106	llvm::Value *This = getThisValue(CGF);
4107
4108	llvm::Value *SrcVal =
4109	IsCopy ? CGF.Builder.CreateLoad(Addr: CGF.GetAddrOfLocalVar(&SrcParam), Name: "src")
4110	: nullptr;
4111
4112	CallArgList Args;
4113
4114	// Push the this ptr.
4115	Args.add(rvalue: RValue::get(V: This), type: CD->getThisType());
4116
4117	// Push the src ptr.
4118	if (SrcVal)
4119	Args.add(rvalue: RValue::get(V: SrcVal), type: SrcParam.getType());
4120
4121	// Add the rest of the default arguments.
4122	SmallVector<const Stmt *, 4> ArgVec;
4123	ArrayRef<ParmVarDecl *> params = CD->parameters().drop_front(IsCopy ? 1 : 0);
4124	for (const ParmVarDecl *PD : params) {
4125	assert(PD->hasDefaultArg() && "ctor closure lacks default args");
4126	ArgVec.push_back(PD->getDefaultArg());
4127	}
4128
4129	CodeGenFunction::RunCleanupsScope Cleanups(CGF);
4130
4131	const auto *FPT = CD->getType()->castAs<FunctionProtoType>();
4132	CGF.EmitCallArgs(Args, Prototype: FPT, ArgRange: llvm::ArrayRef(ArgVec), AC: CD, ParamsToSkip: IsCopy ? 1 : 0);
4133
4134	// Insert any ABI-specific implicit constructor arguments.
4135	AddedStructorArgCounts ExtraArgs =
4136	addImplicitConstructorArgs(CGF, D: CD, Type: Ctor_Complete,
4137	/*ForVirtualBase=*/false,
4138	/*Delegating=*/false, Args);
4139	// Call the destructor with our arguments.
4140	llvm::Constant *CalleePtr =
4141	CGM.getAddrOfCXXStructor(GD: GlobalDecl(CD, Ctor_Complete));
4142	CGCallee Callee =
4143	CGCallee::forDirect(functionPtr: CalleePtr, abstractInfo: GlobalDecl(CD, Ctor_Complete));
4144	const CGFunctionInfo &CalleeInfo = CGM.getTypes().arrangeCXXConstructorCall(
4145	Args, D: CD, CtorKind: Ctor_Complete, ExtraPrefixArgs: ExtraArgs.Prefix, ExtraSuffixArgs: ExtraArgs.Suffix);
4146	CGF.EmitCall(CallInfo: CalleeInfo, Callee, ReturnValue: ReturnValueSlot(), Args);
4147
4148	Cleanups.ForceCleanup();
4149
4150	// Emit the ret instruction, remove any temporary instructions created for the
4151	// aid of CodeGen.
4152	CGF.FinishFunction(EndLoc: SourceLocation());
4153
4154	return ThunkFn;
4155	}
4156
4157	llvm::Constant *MicrosoftCXXABI::getCatchableType(QualType T,
4158	uint32_t NVOffset,
4159	int32_t VBPtrOffset,
4160	uint32_t VBIndex) {
4161	assert(!T->isReferenceType());
4162
4163	CXXRecordDecl *RD = T->getAsCXXRecordDecl();
4164	const CXXConstructorDecl *CD =
4165	RD ? CGM.getContext().getCopyConstructorForExceptionObject(RD) : nullptr;
4166	CXXCtorType CT = Ctor_Complete;
4167	if (CD)
4168	if (!hasDefaultCXXMethodCC(getContext(), CD) || CD->getNumParams() != 1)
4169	CT = Ctor_CopyingClosure;
4170
4171	uint32_t Size = getContext().getTypeSizeInChars(T).getQuantity();
4172	SmallString<256> MangledName;
4173	{
4174	llvm::raw_svector_ostream Out(MangledName);
4175	getMangleContext().mangleCXXCatchableType(T, CD, CT, Size, NVOffset,
4176	VBPtrOffset, VBIndex, Out);
4177	}
4178	if (llvm::GlobalVariable *GV = CGM.getModule().getNamedGlobal(Name: MangledName))
4179	return getImageRelativeConstant(PtrVal: GV);
4180
4181	// The TypeDescriptor is used by the runtime to determine if a catch handler
4182	// is appropriate for the exception object.
4183	llvm::Constant *TD = getImageRelativeConstant(PtrVal: getAddrOfRTTIDescriptor(Type: T));
4184
4185	// The runtime is responsible for calling the copy constructor if the
4186	// exception is caught by value.
4187	llvm::Constant *CopyCtor;
4188	if (CD) {
4189	if (CT == Ctor_CopyingClosure)
4190	CopyCtor = getAddrOfCXXCtorClosure(CD, CT: Ctor_CopyingClosure);
4191	else
4192	CopyCtor = CGM.getAddrOfCXXStructor(GD: GlobalDecl(CD, Ctor_Complete));
4193	} else {
4194	CopyCtor = llvm::Constant::getNullValue(Ty: CGM.Int8PtrTy);
4195	}
4196	CopyCtor = getImageRelativeConstant(PtrVal: CopyCtor);
4197
4198	bool IsScalar = !RD;
4199	bool HasVirtualBases = false;
4200	bool IsStdBadAlloc = false; // std::bad_alloc is special for some reason.
4201	QualType PointeeType = T;
4202	if (T->isPointerType())
4203	PointeeType = T->getPointeeType();
4204	if (const CXXRecordDecl *RD = PointeeType->getAsCXXRecordDecl()) {
4205	HasVirtualBases = RD->getNumVBases() > 0;
4206	if (IdentifierInfo *II = RD->getIdentifier())
4207	IsStdBadAlloc = II->isStr(Str: "bad_alloc") && RD->isInStdNamespace();
4208	}
4209
4210	// Encode the relevant CatchableType properties into the Flags bitfield.
4211	// FIXME: Figure out how bits 2 or 8 can get set.
4212	uint32_t Flags = 0;
4213	if (IsScalar)
4214	Flags |= 1;
4215	if (HasVirtualBases)
4216	Flags |= 4;
4217	if (IsStdBadAlloc)
4218	Flags |= 16;
4219
4220	llvm::Constant *Fields[] = {
4221	llvm::ConstantInt::get(Ty: CGM.IntTy, V: Flags), // Flags
4222	TD, // TypeDescriptor
4223	llvm::ConstantInt::get(Ty: CGM.IntTy, V: NVOffset), // NonVirtualAdjustment
4224	llvm::ConstantInt::get(Ty: CGM.IntTy, V: VBPtrOffset), // OffsetToVBPtr
4225	llvm::ConstantInt::get(Ty: CGM.IntTy, V: VBIndex), // VBTableIndex
4226	llvm::ConstantInt::get(Ty: CGM.IntTy, V: Size), // Size
4227	CopyCtor // CopyCtor
4228	};
4229	llvm::StructType *CTType = getCatchableTypeType();
4230	auto *GV = new llvm::GlobalVariable(
4231	CGM.getModule(), CTType, /*isConstant=*/true, getLinkageForRTTI(Ty: T),
4232	llvm::ConstantStruct::get(T: CTType, V: Fields), MangledName);
4233	GV->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
4234	GV->setSection(".xdata");
4235	if (GV->isWeakForLinker())
4236	GV->setComdat(CGM.getModule().getOrInsertComdat(Name: GV->getName()));
4237	return getImageRelativeConstant(PtrVal: GV);
4238	}
4239
4240	llvm::GlobalVariable *MicrosoftCXXABI::getCatchableTypeArray(QualType T) {
4241	assert(!T->isReferenceType());
4242
4243	// See if we've already generated a CatchableTypeArray for this type before.
4244	llvm::GlobalVariable *&CTA = CatchableTypeArrays[T];
4245	if (CTA)
4246	return CTA;
4247
4248	// Ensure that we don't have duplicate entries in our CatchableTypeArray by
4249	// using a SmallSetVector. Duplicates may arise due to virtual bases
4250	// occurring more than once in the hierarchy.
4251	llvm::SmallSetVector<llvm::Constant *, 2> CatchableTypes;
4252
4253	// C++14 [except.handle]p3:
4254	// A handler is a match for an exception object of type E if [...]
4255	// - the handler is of type cv T or cv T& and T is an unambiguous public
4256	// base class of E, or
4257	// - the handler is of type cv T or const T& where T is a pointer type and
4258	// E is a pointer type that can be converted to T by [...]
4259	// - a standard pointer conversion (4.10) not involving conversions to
4260	// pointers to private or protected or ambiguous classes
4261	const CXXRecordDecl *MostDerivedClass = nullptr;
4262	bool IsPointer = T->isPointerType();
4263	if (IsPointer)
4264	MostDerivedClass = T->getPointeeType()->getAsCXXRecordDecl();
4265	else
4266	MostDerivedClass = T->getAsCXXRecordDecl();
4267
4268	// Collect all the unambiguous public bases of the MostDerivedClass.
4269	if (MostDerivedClass) {
4270	const ASTContext &Context = getContext();
4271	const ASTRecordLayout &MostDerivedLayout =
4272	Context.getASTRecordLayout(MostDerivedClass);
4273	MicrosoftVTableContext &VTableContext = CGM.getMicrosoftVTableContext();
4274	SmallVector<MSRTTIClass, 8> Classes;
4275	serializeClassHierarchy(Classes, RD: MostDerivedClass);
4276	Classes.front().initialize(/*Parent=*/nullptr, /*Specifier=*/nullptr);
4277	detectAmbiguousBases(Classes);
4278	for (const MSRTTIClass &Class : Classes) {
4279	// Skip any ambiguous or private bases.
4280	if (Class.Flags &
4281	(MSRTTIClass::IsPrivateOnPath | MSRTTIClass::IsAmbiguous))
4282	continue;
4283	// Write down how to convert from a derived pointer to a base pointer.
4284	uint32_t OffsetInVBTable = 0;
4285	int32_t VBPtrOffset = -1;
4286	if (Class.VirtualRoot) {
4287	OffsetInVBTable =
4288	VTableContext.getVBTableIndex(Derived: MostDerivedClass, VBase: Class.VirtualRoot)*4;
4289	VBPtrOffset = MostDerivedLayout.getVBPtrOffset().getQuantity();
4290	}
4291
4292	// Turn our record back into a pointer if the exception object is a
4293	// pointer.
4294	QualType RTTITy = QualType(Class.RD->getTypeForDecl(), 0);
4295	if (IsPointer)
4296	RTTITy = Context.getPointerType(T: RTTITy);
4297	CatchableTypes.insert(X: getCatchableType(T: RTTITy, NVOffset: Class.OffsetInVBase,
4298	VBPtrOffset, VBIndex: OffsetInVBTable));
4299	}
4300	}
4301
4302	// C++14 [except.handle]p3:
4303	// A handler is a match for an exception object of type E if
4304	// - The handler is of type cv T or cv T& and E and T are the same type
4305	// (ignoring the top-level cv-qualifiers)
4306	CatchableTypes.insert(X: getCatchableType(T));
4307
4308	// C++14 [except.handle]p3:
4309	// A handler is a match for an exception object of type E if
4310	// - the handler is of type cv T or const T& where T is a pointer type and
4311	// E is a pointer type that can be converted to T by [...]
4312	// - a standard pointer conversion (4.10) not involving conversions to
4313	// pointers to private or protected or ambiguous classes
4314	//
4315	// C++14 [conv.ptr]p2:
4316	// A prvalue of type "pointer to cv T," where T is an object type, can be
4317	// converted to a prvalue of type "pointer to cv void".
4318	if (IsPointer && T->getPointeeType()->isObjectType())
4319	CatchableTypes.insert(getCatchableType(T: getContext().VoidPtrTy));
4320
4321	// C++14 [except.handle]p3:
4322	// A handler is a match for an exception object of type E if [...]
4323	// - the handler is of type cv T or const T& where T is a pointer or
4324	// pointer to member type and E is std::nullptr_t.
4325	//
4326	// We cannot possibly list all possible pointer types here, making this
4327	// implementation incompatible with the standard. However, MSVC includes an
4328	// entry for pointer-to-void in this case. Let's do the same.
4329	if (T->isNullPtrType())
4330	CatchableTypes.insert(getCatchableType(T: getContext().VoidPtrTy));
4331
4332	uint32_t NumEntries = CatchableTypes.size();
4333	llvm::Type *CTType =
4334	getImageRelativeType(PtrType: getCatchableTypeType()->getPointerTo());
4335	llvm::ArrayType *AT = llvm::ArrayType::get(ElementType: CTType, NumElements: NumEntries);
4336	llvm::StructType *CTAType = getCatchableTypeArrayType(NumEntries);
4337	llvm::Constant *Fields[] = {
4338	llvm::ConstantInt::get(Ty: CGM.IntTy, V: NumEntries), // NumEntries
4339	llvm::ConstantArray::get(
4340	T: AT, V: llvm::ArrayRef(CatchableTypes.begin(),
4341	CatchableTypes.end())) // CatchableTypes
4342	};
4343	SmallString<256> MangledName;
4344	{
4345	llvm::raw_svector_ostream Out(MangledName);
4346	getMangleContext().mangleCXXCatchableTypeArray(T, NumEntries, Out);
4347	}
4348	CTA = new llvm::GlobalVariable(
4349	CGM.getModule(), CTAType, /*isConstant=*/true, getLinkageForRTTI(Ty: T),
4350	llvm::ConstantStruct::get(T: CTAType, V: Fields), MangledName);
4351	CTA->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
4352	CTA->setSection(".xdata");
4353	if (CTA->isWeakForLinker())
4354	CTA->setComdat(CGM.getModule().getOrInsertComdat(Name: CTA->getName()));
4355	return CTA;
4356	}
4357
4358	llvm::GlobalVariable *MicrosoftCXXABI::getThrowInfo(QualType T) {
4359	bool IsConst, IsVolatile, IsUnaligned;
4360	T = decomposeTypeForEH(Context&: getContext(), T, IsConst, IsVolatile, IsUnaligned);
4361
4362	// The CatchableTypeArray enumerates the various (CV-unqualified) types that
4363	// the exception object may be caught as.
4364	llvm::GlobalVariable *CTA = getCatchableTypeArray(T);
4365	// The first field in a CatchableTypeArray is the number of CatchableTypes.
4366	// This is used as a component of the mangled name which means that we need to
4367	// know what it is in order to see if we have previously generated the
4368	// ThrowInfo.
4369	uint32_t NumEntries =
4370	cast<llvm::ConstantInt>(Val: CTA->getInitializer()->getAggregateElement(Elt: 0U))
4371	->getLimitedValue();
4372
4373	SmallString<256> MangledName;
4374	{
4375	llvm::raw_svector_ostream Out(MangledName);
4376	getMangleContext().mangleCXXThrowInfo(T, IsConst, IsVolatile, IsUnaligned,
4377	NumEntries, Out);
4378	}
4379
4380	// Reuse a previously generated ThrowInfo if we have generated an appropriate
4381	// one before.
4382	if (llvm::GlobalVariable *GV = CGM.getModule().getNamedGlobal(Name: MangledName))
4383	return GV;
4384
4385	// The RTTI TypeDescriptor uses an unqualified type but catch clauses must
4386	// be at least as CV qualified. Encode this requirement into the Flags
4387	// bitfield.
4388	uint32_t Flags = 0;
4389	if (IsConst)
4390	Flags |= 1;
4391	if (IsVolatile)
4392	Flags |= 2;
4393	if (IsUnaligned)
4394	Flags |= 4;
4395
4396	// The cleanup-function (a destructor) must be called when the exception
4397	// object's lifetime ends.
4398	llvm::Constant *CleanupFn = llvm::Constant::getNullValue(Ty: CGM.Int8PtrTy);
4399	if (const CXXRecordDecl *RD = T->getAsCXXRecordDecl())
4400	if (CXXDestructorDecl *DtorD = RD->getDestructor())
4401	if (!DtorD->isTrivial())
4402	CleanupFn = CGM.getAddrOfCXXStructor(GD: GlobalDecl(DtorD, Dtor_Complete));
4403	// This is unused as far as we can tell, initialize it to null.
4404	llvm::Constant *ForwardCompat =
4405	getImageRelativeConstant(PtrVal: llvm::Constant::getNullValue(Ty: CGM.Int8PtrTy));
4406	llvm::Constant *PointerToCatchableTypes = getImageRelativeConstant(PtrVal: CTA);
4407	llvm::StructType *TIType = getThrowInfoType();
4408	llvm::Constant *Fields[] = {
4409	llvm::ConstantInt::get(Ty: CGM.IntTy, V: Flags), // Flags
4410	getImageRelativeConstant(PtrVal: CleanupFn), // CleanupFn
4411	ForwardCompat, // ForwardCompat
4412	PointerToCatchableTypes // CatchableTypeArray
4413	};
4414	auto *GV = new llvm::GlobalVariable(
4415	CGM.getModule(), TIType, /*isConstant=*/true, getLinkageForRTTI(Ty: T),
4416	llvm::ConstantStruct::get(T: TIType, V: Fields), MangledName.str());
4417	GV->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
4418	GV->setSection(".xdata");
4419	if (GV->isWeakForLinker())
4420	GV->setComdat(CGM.getModule().getOrInsertComdat(Name: GV->getName()));
4421	return GV;
4422	}
4423
4424	void MicrosoftCXXABI::emitThrow(CodeGenFunction &CGF, const CXXThrowExpr *E) {
4425	const Expr *SubExpr = E->getSubExpr();
4426	assert(SubExpr && "SubExpr cannot be null");
4427	QualType ThrowType = SubExpr->getType();
4428	// The exception object lives on the stack and it's address is passed to the
4429	// runtime function.
4430	Address AI = CGF.CreateMemTemp(T: ThrowType);
4431	CGF.EmitAnyExprToMem(E: SubExpr, Location: AI, Quals: ThrowType.getQualifiers(),
4432	/*IsInit=*/IsInitializer: true);
4433
4434	// The so-called ThrowInfo is used to describe how the exception object may be
4435	// caught.
4436	llvm::GlobalVariable *TI = getThrowInfo(T: ThrowType);
4437
4438	// Call into the runtime to throw the exception.
4439	llvm::Value *Args[] = {AI.emitRawPointer(CGF), TI};
4440	CGF.EmitNoreturnRuntimeCallOrInvoke(callee: getThrowFn(), args: Args);
4441	}
4442
4443	std::pair<llvm::Value *, const CXXRecordDecl *>
4444	MicrosoftCXXABI::LoadVTablePtr(CodeGenFunction &CGF, Address This,
4445	const CXXRecordDecl *RD) {
4446	std::tie(args&: This, args: std::ignore, args&: RD) =
4447	performBaseAdjustment(CGF, Value: This, SrcRecordTy: QualType(RD->getTypeForDecl(), 0));
4448	return {CGF.GetVTablePtr(This, VTableTy: CGM.Int8PtrTy, VTableClass: RD), RD};
4449	}
4450
4451	bool MicrosoftCXXABI::isPermittedToBeHomogeneousAggregate(
4452	const CXXRecordDecl *RD) const {
4453	// All aggregates are permitted to be HFA on non-ARM platforms, which mostly
4454	// affects vectorcall on x64/x86.
4455	if (!CGM.getTarget().getTriple().isAArch64())
4456	return true;
4457	// MSVC Windows on Arm64 has its own rules for determining if a type is HFA
4458	// that are inconsistent with the AAPCS64 ABI. The following are our best
4459	// determination of those rules so far, based on observation of MSVC's
4460	// behavior.
4461	if (RD->isEmpty())
4462	return false;
4463	if (RD->isPolymorphic())
4464	return false;
4465	if (RD->hasNonTrivialCopyAssignment())
4466	return false;
4467	if (RD->hasNonTrivialDestructor())
4468	return false;
4469	if (RD->hasNonTrivialDefaultConstructor())
4470	return false;
4471	// These two are somewhat redundant given the caller
4472	// (ABIInfo::isHomogeneousAggregate) checks the bases and fields, but that
4473	// caller doesn't consider empty bases/fields to be non-homogenous, but it
4474	// looks like Microsoft's AArch64 ABI does care about these empty types &
4475	// anything containing/derived from one is non-homogeneous.
4476	// Instead we could add another CXXABI entry point to query this property and
4477	// have ABIInfo::isHomogeneousAggregate use that property.
4478	// I don't think any other of the features listed above could be true of a
4479	// base/field while not true of the outer struct. For example, if you have a
4480	// base/field that has an non-trivial copy assignment/dtor/default ctor, then
4481	// the outer struct's corresponding operation must be non-trivial.
4482	for (const CXXBaseSpecifier &B : RD->bases()) {
4483	if (const CXXRecordDecl *FRD = B.getType()->getAsCXXRecordDecl()) {
4484	if (!isPermittedToBeHomogeneousAggregate(RD: FRD))
4485	return false;
4486	}
4487	}
4488	// empty fields seem to be caught by the ABIInfo::isHomogeneousAggregate
4489	// checking for padding - but maybe there are ways to end up with an empty
4490	// field without padding? Not that I know of, so don't check fields here &
4491	// rely on the padding check.
4492	return true;
4493	}
4494