1	//===------- ItaniumCXXABI.cpp - Emit LLVM Code from ASTs for a Module ----===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This provides C++ code generation targeting the Itanium C++ ABI. The class
10	// in this file generates structures that follow the Itanium C++ ABI, which is
11	// documented at:
12	// https://itanium-cxx-abi.github.io/cxx-abi/abi.html
13	// https://itanium-cxx-abi.github.io/cxx-abi/abi-eh.html
14	//
15	// It also supports the closely-related ARM ABI, documented at:
16	// https://developer.arm.com/documentation/ihi0041/g/
17	//
18	//===----------------------------------------------------------------------===//
19
20	#include "CGCXXABI.h"
21	#include "CGCleanup.h"
22	#include "CGRecordLayout.h"
23	#include "CGVTables.h"
24	#include "CodeGenFunction.h"
25	#include "CodeGenModule.h"
26	#include "TargetInfo.h"
27	#include "clang/AST/Attr.h"
28	#include "clang/AST/Mangle.h"
29	#include "clang/AST/StmtCXX.h"
30	#include "clang/AST/Type.h"
31	#include "clang/CodeGen/ConstantInitBuilder.h"
32	#include "llvm/IR/DataLayout.h"
33	#include "llvm/IR/GlobalValue.h"
34	#include "llvm/IR/Instructions.h"
35	#include "llvm/IR/Intrinsics.h"
36	#include "llvm/IR/Value.h"
37	#include "llvm/Support/ScopedPrinter.h"
38
39	#include <optional>
40
41	using namespace clang;
42	using namespace CodeGen;
43
44	namespace {
45	class ItaniumCXXABI : public CodeGen::CGCXXABI {
46	/// VTables - All the vtables which have been defined.
47	llvm::DenseMap<const CXXRecordDecl *, llvm::GlobalVariable *> VTables;
48
49	/// All the thread wrapper functions that have been used.
50	llvm::SmallVector<std::pair<const VarDecl *, llvm::Function *>, 8>
51	ThreadWrappers;
52
53	protected:
54	bool UseARMMethodPtrABI;
55	bool UseARMGuardVarABI;
56	bool Use32BitVTableOffsetABI;
57
58	ItaniumMangleContext &getMangleContext() {
59	return cast<ItaniumMangleContext>(Val&: CodeGen::CGCXXABI::getMangleContext());
60	}
61
62	public:
63	ItaniumCXXABI(CodeGen::CodeGenModule &CGM,
64	bool UseARMMethodPtrABI = false,
65	bool UseARMGuardVarABI = false) :
66	CGCXXABI(CGM), UseARMMethodPtrABI(UseARMMethodPtrABI),
67	UseARMGuardVarABI(UseARMGuardVarABI),
68	Use32BitVTableOffsetABI(false) { }
69
70	bool classifyReturnType(CGFunctionInfo &FI) const override;
71
72	RecordArgABI getRecordArgABI(const CXXRecordDecl *RD) const override {
73	// If C++ prohibits us from making a copy, pass by address.
74	if (!RD->canPassInRegisters())
75	return RAA_Indirect;
76	return RAA_Default;
77	}
78
79	bool isThisCompleteObject(GlobalDecl GD) const override {
80	// The Itanium ABI has separate complete-object vs. base-object
81	// variants of both constructors and destructors.
82	if (isa<CXXDestructorDecl>(Val: GD.getDecl())) {
83	switch (GD.getDtorType()) {
84	case Dtor_Complete:
85	case Dtor_Deleting:
86	return true;
87
88	case Dtor_Base:
89	return false;
90
91	case Dtor_Comdat:
92	llvm_unreachable("emitting dtor comdat as function?");
93	}
94	llvm_unreachable("bad dtor kind");
95	}
96	if (isa<CXXConstructorDecl>(Val: GD.getDecl())) {
97	switch (GD.getCtorType()) {
98	case Ctor_Complete:
99	return true;
100
101	case Ctor_Base:
102	return false;
103
104	case Ctor_CopyingClosure:
105	case Ctor_DefaultClosure:
106	llvm_unreachable("closure ctors in Itanium ABI?");
107
108	case Ctor_Comdat:
109	llvm_unreachable("emitting ctor comdat as function?");
110	}
111	llvm_unreachable("bad dtor kind");
112	}
113
114	// No other kinds.
115	return false;
116	}
117
118	bool isZeroInitializable(const MemberPointerType *MPT) override;
119
120	llvm::Type *ConvertMemberPointerType(const MemberPointerType *MPT) override;
121
122	CGCallee
123	EmitLoadOfMemberFunctionPointer(CodeGenFunction &CGF,
124	const Expr *E,
125	Address This,
126	llvm::Value *&ThisPtrForCall,
127	llvm::Value *MemFnPtr,
128	const MemberPointerType *MPT) override;
129
130	llvm::Value *
131	EmitMemberDataPointerAddress(CodeGenFunction &CGF, const Expr *E,
132	Address Base,
133	llvm::Value *MemPtr,
134	const MemberPointerType *MPT) override;
135
136	llvm::Value *EmitMemberPointerConversion(CodeGenFunction &CGF,
137	const CastExpr *E,
138	llvm::Value *Src) override;
139	llvm::Constant *EmitMemberPointerConversion(const CastExpr *E,
140	llvm::Constant *Src) override;
141
142	llvm::Constant *EmitNullMemberPointer(const MemberPointerType *MPT) override;
143
144	llvm::Constant *EmitMemberFunctionPointer(const CXXMethodDecl *MD) override;
145	llvm::Constant *EmitMemberDataPointer(const MemberPointerType *MPT,
146	CharUnits offset) override;
147	llvm::Constant *EmitMemberPointer(const APValue &MP, QualType MPT) override;
148	llvm::Constant *BuildMemberPointer(const CXXMethodDecl *MD,
149	CharUnits ThisAdjustment);
150
151	llvm::Value *EmitMemberPointerComparison(CodeGenFunction &CGF,
152	llvm::Value *L, llvm::Value *R,
153	const MemberPointerType *MPT,
154	bool Inequality) override;
155
156	llvm::Value *EmitMemberPointerIsNotNull(CodeGenFunction &CGF,
157	llvm::Value *Addr,
158	const MemberPointerType *MPT) override;
159
160	void emitVirtualObjectDelete(CodeGenFunction &CGF, const CXXDeleteExpr *DE,
161	Address Ptr, QualType ElementType,
162	const CXXDestructorDecl *Dtor) override;
163
164	void emitRethrow(CodeGenFunction &CGF, bool isNoReturn) override;
165	void emitThrow(CodeGenFunction &CGF, const CXXThrowExpr *E) override;
166
167	void emitBeginCatch(CodeGenFunction &CGF, const CXXCatchStmt *C) override;
168
169	llvm::CallInst *
170	emitTerminateForUnexpectedException(CodeGenFunction &CGF,
171	llvm::Value *Exn) override;
172
173	void EmitFundamentalRTTIDescriptors(const CXXRecordDecl *RD);
174	llvm::Constant *getAddrOfRTTIDescriptor(QualType Ty) override;
175	CatchTypeInfo
176	getAddrOfCXXCatchHandlerType(QualType Ty,
177	QualType CatchHandlerType) override {
178	return CatchTypeInfo{.RTTI: getAddrOfRTTIDescriptor(Ty), .Flags: 0};
179	}
180
181	bool shouldTypeidBeNullChecked(bool IsDeref, QualType SrcRecordTy) override;
182	void EmitBadTypeidCall(CodeGenFunction &CGF) override;
183	llvm::Value *EmitTypeid(CodeGenFunction &CGF, QualType SrcRecordTy,
184	Address ThisPtr,
185	llvm::Type *StdTypeInfoPtrTy) override;
186
187	bool shouldDynamicCastCallBeNullChecked(bool SrcIsPtr,
188	QualType SrcRecordTy) override;
189
190	/// Determine whether we know that all instances of type RecordTy will have
191	/// the same vtable pointer values, that is distinct from all other vtable
192	/// pointers. While this is required by the Itanium ABI, it doesn't happen in
193	/// practice in some cases due to language extensions.
194	bool hasUniqueVTablePointer(QualType RecordTy) {
195	const CXXRecordDecl *RD = RecordTy->getAsCXXRecordDecl();
196
197	// Under -fapple-kext, multiple definitions of the same vtable may be
198	// emitted.
199	if (!CGM.getCodeGenOpts().AssumeUniqueVTables ||
200	getContext().getLangOpts().AppleKext)
201	return false;
202
203	// If the type_info* would be null, the vtable might be merged with that of
204	// another type.
205	if (!CGM.shouldEmitRTTI())
206	return false;
207
208	// If there's only one definition of the vtable in the program, it has a
209	// unique address.
210	if (!llvm::GlobalValue::isWeakForLinker(Linkage: CGM.getVTableLinkage(RD)))
211	return true;
212
213	// Even if there are multiple definitions of the vtable, they are required
214	// by the ABI to use the same symbol name, so should be merged at load
215	// time. However, if the class has hidden visibility, there can be
216	// different versions of the class in different modules, and the ABI
217	// library might treat them as being the same.
218	if (CGM.GetLLVMVisibility(V: RD->getVisibility()) !=
219	llvm::GlobalValue::DefaultVisibility)
220	return false;
221
222	return true;
223	}
224
225	bool shouldEmitExactDynamicCast(QualType DestRecordTy) override {
226	return hasUniqueVTablePointer(RecordTy: DestRecordTy);
227	}
228
229	llvm::Value *emitDynamicCastCall(CodeGenFunction &CGF, Address Value,
230	QualType SrcRecordTy, QualType DestTy,
231	QualType DestRecordTy,
232	llvm::BasicBlock *CastEnd) override;
233
234	llvm::Value *emitExactDynamicCast(CodeGenFunction &CGF, Address ThisAddr,
235	QualType SrcRecordTy, QualType DestTy,
236	QualType DestRecordTy,
237	llvm::BasicBlock *CastSuccess,
238	llvm::BasicBlock *CastFail) override;
239
240	llvm::Value *emitDynamicCastToVoid(CodeGenFunction &CGF, Address Value,
241	QualType SrcRecordTy) override;
242
243	bool EmitBadCastCall(CodeGenFunction &CGF) override;
244
245	llvm::Value *
246	GetVirtualBaseClassOffset(CodeGenFunction &CGF, Address This,
247	const CXXRecordDecl *ClassDecl,
248	const CXXRecordDecl *BaseClassDecl) override;
249
250	void EmitCXXConstructors(const CXXConstructorDecl *D) override;
251
252	AddedStructorArgCounts
253	buildStructorSignature(GlobalDecl GD,
254	SmallVectorImpl<CanQualType> &ArgTys) override;
255
256	bool useThunkForDtorVariant(const CXXDestructorDecl *Dtor,
257	CXXDtorType DT) const override {
258	// Itanium does not emit any destructor variant as an inline thunk.
259	// Delegating may occur as an optimization, but all variants are either
260	// emitted with external linkage or as linkonce if they are inline and used.
261	return false;
262	}
263
264	void EmitCXXDestructors(const CXXDestructorDecl *D) override;
265
266	void addImplicitStructorParams(CodeGenFunction &CGF, QualType &ResTy,
267	FunctionArgList &Params) override;
268
269	void EmitInstanceFunctionProlog(CodeGenFunction &CGF) override;
270
271	AddedStructorArgs getImplicitConstructorArgs(CodeGenFunction &CGF,
272	const CXXConstructorDecl *D,
273	CXXCtorType Type,
274	bool ForVirtualBase,
275	bool Delegating) override;
276
277	llvm::Value *getCXXDestructorImplicitParam(CodeGenFunction &CGF,
278	const CXXDestructorDecl *DD,
279	CXXDtorType Type,
280	bool ForVirtualBase,
281	bool Delegating) override;
282
283	void EmitDestructorCall(CodeGenFunction &CGF, const CXXDestructorDecl *DD,
284	CXXDtorType Type, bool ForVirtualBase,
285	bool Delegating, Address This,
286	QualType ThisTy) override;
287
288	void emitVTableDefinitions(CodeGenVTables &CGVT,
289	const CXXRecordDecl *RD) override;
290
291	bool isVirtualOffsetNeededForVTableField(CodeGenFunction &CGF,
292	CodeGenFunction::VPtr Vptr) override;
293
294	bool doStructorsInitializeVPtrs(const CXXRecordDecl *VTableClass) override {
295	return true;
296	}
297
298	llvm::Constant *
299	getVTableAddressPoint(BaseSubobject Base,
300	const CXXRecordDecl *VTableClass) override;
301
302	llvm::Value *getVTableAddressPointInStructor(
303	CodeGenFunction &CGF, const CXXRecordDecl *VTableClass,
304	BaseSubobject Base, const CXXRecordDecl *NearestVBase) override;
305
306	llvm::Value *getVTableAddressPointInStructorWithVTT(
307	CodeGenFunction &CGF, const CXXRecordDecl *VTableClass,
308	BaseSubobject Base, const CXXRecordDecl *NearestVBase);
309
310	llvm::GlobalVariable *getAddrOfVTable(const CXXRecordDecl *RD,
311	CharUnits VPtrOffset) override;
312
313	CGCallee getVirtualFunctionPointer(CodeGenFunction &CGF, GlobalDecl GD,
314	Address This, llvm::Type *Ty,
315	SourceLocation Loc) override;
316
317	llvm::Value *EmitVirtualDestructorCall(CodeGenFunction &CGF,
318	const CXXDestructorDecl *Dtor,
319	CXXDtorType DtorType, Address This,
320	DeleteOrMemberCallExpr E) override;
321
322	void emitVirtualInheritanceTables(const CXXRecordDecl *RD) override;
323
324	bool canSpeculativelyEmitVTable(const CXXRecordDecl *RD) const override;
325	bool canSpeculativelyEmitVTableAsBaseClass(const CXXRecordDecl *RD) const;
326
327	void setThunkLinkage(llvm::Function *Thunk, bool ForVTable, GlobalDecl GD,
328	bool ReturnAdjustment) override {
329	// Allow inlining of thunks by emitting them with available_externally
330	// linkage together with vtables when needed.
331	if (ForVTable && !Thunk->hasLocalLinkage())
332	Thunk->setLinkage(llvm::GlobalValue::AvailableExternallyLinkage);
333	CGM.setGVProperties(GV: Thunk, GD);
334	}
335
336	bool exportThunk() override { return true; }
337
338	llvm::Value *performThisAdjustment(CodeGenFunction &CGF, Address This,
339	const ThisAdjustment &TA) override;
340
341	llvm::Value *performReturnAdjustment(CodeGenFunction &CGF, Address Ret,
342	const ReturnAdjustment &RA) override;
343
344	size_t getSrcArgforCopyCtor(const CXXConstructorDecl *,
345	FunctionArgList &Args) const override {
346	assert(!Args.empty() && "expected the arglist to not be empty!");
347	return Args.size() - 1;
348	}
349
350	StringRef GetPureVirtualCallName() override { return "__cxa_pure_virtual"; }
351	StringRef GetDeletedVirtualCallName() override
352	{ return "__cxa_deleted_virtual"; }
353
354	CharUnits getArrayCookieSizeImpl(QualType elementType) override;
355	Address InitializeArrayCookie(CodeGenFunction &CGF,
356	Address NewPtr,
357	llvm::Value *NumElements,
358	const CXXNewExpr *expr,
359	QualType ElementType) override;
360	llvm::Value *readArrayCookieImpl(CodeGenFunction &CGF,
361	Address allocPtr,
362	CharUnits cookieSize) override;
363
364	void EmitGuardedInit(CodeGenFunction &CGF, const VarDecl &D,
365	llvm::GlobalVariable *DeclPtr,
366	bool PerformInit) override;
367	void registerGlobalDtor(CodeGenFunction &CGF, const VarDecl &D,
368	llvm::FunctionCallee dtor,
369	llvm::Constant *addr) override;
370
371	llvm::Function *getOrCreateThreadLocalWrapper(const VarDecl *VD,
372	llvm::Value *Val);
373	void EmitThreadLocalInitFuncs(
374	CodeGenModule &CGM,
375	ArrayRef<const VarDecl *> CXXThreadLocals,
376	ArrayRef<llvm::Function *> CXXThreadLocalInits,
377	ArrayRef<const VarDecl *> CXXThreadLocalInitVars) override;
378
379	bool usesThreadWrapperFunction(const VarDecl *VD) const override {
380	return !isEmittedWithConstantInitializer(VD) ||
381	mayNeedDestruction(VD);
382	}
383	LValue EmitThreadLocalVarDeclLValue(CodeGenFunction &CGF, const VarDecl *VD,
384	QualType LValType) override;
385
386	bool NeedsVTTParameter(GlobalDecl GD) override;
387
388	/**************************** RTTI Uniqueness ******************************/
389
390	protected:
391	/// Returns true if the ABI requires RTTI type_info objects to be unique
392	/// across a program.
393	virtual bool shouldRTTIBeUnique() const { return true; }
394
395	public:
396	/// What sort of unique-RTTI behavior should we use?
397	enum RTTIUniquenessKind {
398	/// We are guaranteeing, or need to guarantee, that the RTTI string
399	/// is unique.
400	RUK_Unique,
401
402	/// We are not guaranteeing uniqueness for the RTTI string, so we
403	/// can demote to hidden visibility but must use string comparisons.
404	RUK_NonUniqueHidden,
405
406	/// We are not guaranteeing uniqueness for the RTTI string, so we
407	/// have to use string comparisons, but we also have to emit it with
408	/// non-hidden visibility.
409	RUK_NonUniqueVisible
410	};
411
412	/// Return the required visibility status for the given type and linkage in
413	/// the current ABI.
414	RTTIUniquenessKind
415	classifyRTTIUniqueness(QualType CanTy,
416	llvm::GlobalValue::LinkageTypes Linkage) const;
417	friend class ItaniumRTTIBuilder;
418
419	void emitCXXStructor(GlobalDecl GD) override;
420
421	std::pair<llvm::Value *, const CXXRecordDecl *>
422	LoadVTablePtr(CodeGenFunction &CGF, Address This,
423	const CXXRecordDecl *RD) override;
424
425	private:
426	bool hasAnyUnusedVirtualInlineFunction(const CXXRecordDecl *RD) const {
427	const auto &VtableLayout =
428	CGM.getItaniumVTableContext().getVTableLayout(RD);
429
430	for (const auto &VtableComponent : VtableLayout.vtable_components()) {
431	// Skip empty slot.
432	if (!VtableComponent.isUsedFunctionPointerKind())
433	continue;
434
435	const CXXMethodDecl *Method = VtableComponent.getFunctionDecl();
436	if (!Method->getCanonicalDecl()->isInlined())
437	continue;
438
439	StringRef Name = CGM.getMangledName(GD: VtableComponent.getGlobalDecl());
440	auto *Entry = CGM.GetGlobalValue(Ref: Name);
441	// This checks if virtual inline function has already been emitted.
442	// Note that it is possible that this inline function would be emitted
443	// after trying to emit vtable speculatively. Because of this we do
444	// an extra pass after emitting all deferred vtables to find and emit
445	// these vtables opportunistically.
446	if (!Entry || Entry->isDeclaration())
447	return true;
448	}
449	return false;
450	}
451
452	bool isVTableHidden(const CXXRecordDecl *RD) const {
453	const auto &VtableLayout =
454	CGM.getItaniumVTableContext().getVTableLayout(RD);
455
456	for (const auto &VtableComponent : VtableLayout.vtable_components()) {
457	if (VtableComponent.isRTTIKind()) {
458	const CXXRecordDecl *RTTIDecl = VtableComponent.getRTTIDecl();
459	if (RTTIDecl->getVisibility() == Visibility::HiddenVisibility)
460	return true;
461	} else if (VtableComponent.isUsedFunctionPointerKind()) {
462	const CXXMethodDecl *Method = VtableComponent.getFunctionDecl();
463	if (Method->getVisibility() == Visibility::HiddenVisibility &&
464	!Method->isDefined())
465	return true;
466	}
467	}
468	return false;
469	}
470	};
471
472	class ARMCXXABI : public ItaniumCXXABI {
473	public:
474	ARMCXXABI(CodeGen::CodeGenModule &CGM) :
475	ItaniumCXXABI(CGM, /*UseARMMethodPtrABI=*/true,
476	/*UseARMGuardVarABI=*/true) {}
477
478	bool constructorsAndDestructorsReturnThis() const override { return true; }
479
480	void EmitReturnFromThunk(CodeGenFunction &CGF, RValue RV,
481	QualType ResTy) override;
482
483	CharUnits getArrayCookieSizeImpl(QualType elementType) override;
484	Address InitializeArrayCookie(CodeGenFunction &CGF,
485	Address NewPtr,
486	llvm::Value *NumElements,
487	const CXXNewExpr *expr,
488	QualType ElementType) override;
489	llvm::Value *readArrayCookieImpl(CodeGenFunction &CGF, Address allocPtr,
490	CharUnits cookieSize) override;
491	};
492
493	class AppleARM64CXXABI : public ARMCXXABI {
494	public:
495	AppleARM64CXXABI(CodeGen::CodeGenModule &CGM) : ARMCXXABI(CGM) {
496	Use32BitVTableOffsetABI = true;
497	}
498
499	// ARM64 libraries are prepared for non-unique RTTI.
500	bool shouldRTTIBeUnique() const override { return false; }
501	};
502
503	class FuchsiaCXXABI final : public ItaniumCXXABI {
504	public:
505	explicit FuchsiaCXXABI(CodeGen::CodeGenModule &CGM)
506	: ItaniumCXXABI(CGM) {}
507
508	private:
509	bool constructorsAndDestructorsReturnThis() const override { return true; }
510	};
511
512	class WebAssemblyCXXABI final : public ItaniumCXXABI {
513	public:
514	explicit WebAssemblyCXXABI(CodeGen::CodeGenModule &CGM)
515	: ItaniumCXXABI(CGM, /*UseARMMethodPtrABI=*/true,
516	/*UseARMGuardVarABI=*/true) {}
517	void emitBeginCatch(CodeGenFunction &CGF, const CXXCatchStmt *C) override;
518	llvm::CallInst *
519	emitTerminateForUnexpectedException(CodeGenFunction &CGF,
520	llvm::Value *Exn) override;
521
522	private:
523	bool constructorsAndDestructorsReturnThis() const override { return true; }
524	bool canCallMismatchedFunctionType() const override { return false; }
525	};
526
527	class XLCXXABI final : public ItaniumCXXABI {
528	public:
529	explicit XLCXXABI(CodeGen::CodeGenModule &CGM)
530	: ItaniumCXXABI(CGM) {}
531
532	void registerGlobalDtor(CodeGenFunction &CGF, const VarDecl &D,
533	llvm::FunctionCallee dtor,
534	llvm::Constant *addr) override;
535
536	bool useSinitAndSterm() const override { return true; }
537
538	private:
539	void emitCXXStermFinalizer(const VarDecl &D, llvm::Function *dtorStub,
540	llvm::Constant *addr);
541	};
542	}
543
544	CodeGen::CGCXXABI *CodeGen::CreateItaniumCXXABI(CodeGenModule &CGM) {
545	switch (CGM.getContext().getCXXABIKind()) {
546	// For IR-generation purposes, there's no significant difference
547	// between the ARM and iOS ABIs.
548	case TargetCXXABI::GenericARM:
549	case TargetCXXABI::iOS:
550	case TargetCXXABI::WatchOS:
551	return new ARMCXXABI(CGM);
552
553	case TargetCXXABI::AppleARM64:
554	return new AppleARM64CXXABI(CGM);
555
556	case TargetCXXABI::Fuchsia:
557	return new FuchsiaCXXABI(CGM);
558
559	// Note that AArch64 uses the generic ItaniumCXXABI class since it doesn't
560	// include the other 32-bit ARM oddities: constructor/destructor return values
561	// and array cookies.
562	case TargetCXXABI::GenericAArch64:
563	return new ItaniumCXXABI(CGM, /*UseARMMethodPtrABI=*/true,
564	/*UseARMGuardVarABI=*/true);
565
566	case TargetCXXABI::GenericMIPS:
567	return new ItaniumCXXABI(CGM, /*UseARMMethodPtrABI=*/true);
568
569	case TargetCXXABI::WebAssembly:
570	return new WebAssemblyCXXABI(CGM);
571
572	case TargetCXXABI::XL:
573	return new XLCXXABI(CGM);
574
575	case TargetCXXABI::GenericItanium:
576	if (CGM.getContext().getTargetInfo().getTriple().getArch()
577	== llvm::Triple::le32) {
578	// For PNaCl, use ARM-style method pointers so that PNaCl code
579	// does not assume anything about the alignment of function
580	// pointers.
581	return new ItaniumCXXABI(CGM, /*UseARMMethodPtrABI=*/true);
582	}
583	return new ItaniumCXXABI(CGM);
584
585	case TargetCXXABI::Microsoft:
586	llvm_unreachable("Microsoft ABI is not Itanium-based");
587	}
588	llvm_unreachable("bad ABI kind");
589	}
590
591	llvm::Type *
592	ItaniumCXXABI::ConvertMemberPointerType(const MemberPointerType *MPT) {
593	if (MPT->isMemberDataPointer())
594	return CGM.PtrDiffTy;
595	return llvm::StructType::get(elt1: CGM.PtrDiffTy, elts: CGM.PtrDiffTy);
596	}
597
598	/// In the Itanium and ARM ABIs, method pointers have the form:
599	/// struct { ptrdiff_t ptr; ptrdiff_t adj; } memptr;
600	///
601	/// In the Itanium ABI:
602	/// - method pointers are virtual if (memptr.ptr & 1) is nonzero
603	/// - the this-adjustment is (memptr.adj)
604	/// - the virtual offset is (memptr.ptr - 1)
605	///
606	/// In the ARM ABI:
607	/// - method pointers are virtual if (memptr.adj & 1) is nonzero
608	/// - the this-adjustment is (memptr.adj >> 1)
609	/// - the virtual offset is (memptr.ptr)
610	/// ARM uses 'adj' for the virtual flag because Thumb functions
611	/// may be only single-byte aligned.
612	///
613	/// If the member is virtual, the adjusted 'this' pointer points
614	/// to a vtable pointer from which the virtual offset is applied.
615	///
616	/// If the member is non-virtual, memptr.ptr is the address of
617	/// the function to call.
618	CGCallee ItaniumCXXABI::EmitLoadOfMemberFunctionPointer(
619	CodeGenFunction &CGF, const Expr *E, Address ThisAddr,
620	llvm::Value *&ThisPtrForCall,
621	llvm::Value *MemFnPtr, const MemberPointerType *MPT) {
622	CGBuilderTy &Builder = CGF.Builder;
623
624	const FunctionProtoType *FPT =
625	MPT->getPointeeType()->castAs<FunctionProtoType>();
626	auto *RD =
627	cast<CXXRecordDecl>(Val: MPT->getClass()->castAs<RecordType>()->getDecl());
628
629	llvm::Constant *ptrdiff_1 = llvm::ConstantInt::get(Ty: CGM.PtrDiffTy, V: 1);
630
631	llvm::BasicBlock *FnVirtual = CGF.createBasicBlock(name: "memptr.virtual");
632	llvm::BasicBlock *FnNonVirtual = CGF.createBasicBlock(name: "memptr.nonvirtual");
633	llvm::BasicBlock *FnEnd = CGF.createBasicBlock(name: "memptr.end");
634
635	// Extract memptr.adj, which is in the second field.
636	llvm::Value *RawAdj = Builder.CreateExtractValue(Agg: MemFnPtr, Idxs: 1, Name: "memptr.adj");
637
638	// Compute the true adjustment.
639	llvm::Value *Adj = RawAdj;
640	if (UseARMMethodPtrABI)
641	Adj = Builder.CreateAShr(LHS: Adj, RHS: ptrdiff_1, Name: "memptr.adj.shifted");
642
643	// Apply the adjustment and cast back to the original struct type
644	// for consistency.
645	llvm::Value *This = ThisAddr.emitRawPointer(CGF);
646	This = Builder.CreateInBoundsGEP(Ty: Builder.getInt8Ty(), Ptr: This, IdxList: Adj);
647	ThisPtrForCall = This;
648
649	// Load the function pointer.
650	llvm::Value *FnAsInt = Builder.CreateExtractValue(Agg: MemFnPtr, Idxs: 0, Name: "memptr.ptr");
651
652	// If the LSB in the function pointer is 1, the function pointer points to
653	// a virtual function.
654	llvm::Value *IsVirtual;
655	if (UseARMMethodPtrABI)
656	IsVirtual = Builder.CreateAnd(LHS: RawAdj, RHS: ptrdiff_1);
657	else
658	IsVirtual = Builder.CreateAnd(LHS: FnAsInt, RHS: ptrdiff_1);
659	IsVirtual = Builder.CreateIsNotNull(Arg: IsVirtual, Name: "memptr.isvirtual");
660	Builder.CreateCondBr(Cond: IsVirtual, True: FnVirtual, False: FnNonVirtual);
661
662	// In the virtual path, the adjustment left 'This' pointing to the
663	// vtable of the correct base subobject. The "function pointer" is an
664	// offset within the vtable (+1 for the virtual flag on non-ARM).
665	CGF.EmitBlock(BB: FnVirtual);
666
667	// Cast the adjusted this to a pointer to vtable pointer and load.
668	llvm::Type *VTableTy = CGF.CGM.GlobalsInt8PtrTy;
669	CharUnits VTablePtrAlign =
670	CGF.CGM.getDynamicOffsetAlignment(ActualAlign: ThisAddr.getAlignment(), Class: RD,
671	ExpectedTargetAlign: CGF.getPointerAlign());
672	llvm::Value *VTable = CGF.GetVTablePtr(
673	This: Address(This, ThisAddr.getElementType(), VTablePtrAlign), VTableTy, VTableClass: RD);
674
675	// Apply the offset.
676	// On ARM64, to reserve extra space in virtual member function pointers,
677	// we only pay attention to the low 32 bits of the offset.
678	llvm::Value *VTableOffset = FnAsInt;
679	if (!UseARMMethodPtrABI)
680	VTableOffset = Builder.CreateSub(LHS: VTableOffset, RHS: ptrdiff_1);
681	if (Use32BitVTableOffsetABI) {
682	VTableOffset = Builder.CreateTrunc(V: VTableOffset, DestTy: CGF.Int32Ty);
683	VTableOffset = Builder.CreateZExt(V: VTableOffset, DestTy: CGM.PtrDiffTy);
684	}
685
686	// Check the address of the function pointer if CFI on member function
687	// pointers is enabled.
688	llvm::Constant *CheckSourceLocation;
689	llvm::Constant *CheckTypeDesc;
690	bool ShouldEmitCFICheck = CGF.SanOpts.has(K: SanitizerKind::CFIMFCall) &&
691	CGM.HasHiddenLTOVisibility(RD);
692	bool ShouldEmitVFEInfo = CGM.getCodeGenOpts().VirtualFunctionElimination &&
693	CGM.HasHiddenLTOVisibility(RD);
694	bool ShouldEmitWPDInfo =
695	CGM.getCodeGenOpts().WholeProgramVTables &&
696	// Don't insert type tests if we are forcing public visibility.
697	!CGM.AlwaysHasLTOVisibilityPublic(RD);
698	llvm::Value *VirtualFn = nullptr;
699
700	{
701	CodeGenFunction::SanitizerScope SanScope(&CGF);
702	llvm::Value *TypeId = nullptr;
703	llvm::Value *CheckResult = nullptr;
704
705	if (ShouldEmitCFICheck || ShouldEmitVFEInfo || ShouldEmitWPDInfo) {
706	// If doing CFI, VFE or WPD, we will need the metadata node to check
707	// against.
708	llvm::Metadata *MD =
709	CGM.CreateMetadataIdentifierForVirtualMemPtrType(T: QualType(MPT, 0));
710	TypeId = llvm::MetadataAsValue::get(Context&: CGF.getLLVMContext(), MD);
711	}
712
713	if (ShouldEmitVFEInfo) {
714	llvm::Value *VFPAddr =
715	Builder.CreateGEP(Ty: CGF.Int8Ty, Ptr: VTable, IdxList: VTableOffset);
716
717	// If doing VFE, load from the vtable with a type.checked.load intrinsic
718	// call. Note that we use the GEP to calculate the address to load from
719	// and pass 0 as the offset to the intrinsic. This is because every
720	// vtable slot of the correct type is marked with matching metadata, and
721	// we know that the load must be from one of these slots.
722	llvm::Value *CheckedLoad = Builder.CreateCall(
723	CGM.getIntrinsic(llvm::Intrinsic::type_checked_load),
724	{VFPAddr, llvm::ConstantInt::get(CGM.Int32Ty, 0), TypeId});
725	CheckResult = Builder.CreateExtractValue(Agg: CheckedLoad, Idxs: 1);
726	VirtualFn = Builder.CreateExtractValue(Agg: CheckedLoad, Idxs: 0);
727	} else {
728	// When not doing VFE, emit a normal load, as it allows more
729	// optimisations than type.checked.load.
730	if (ShouldEmitCFICheck || ShouldEmitWPDInfo) {
731	llvm::Value *VFPAddr =
732	Builder.CreateGEP(Ty: CGF.Int8Ty, Ptr: VTable, IdxList: VTableOffset);
733	llvm::Intrinsic::ID IID = CGM.HasHiddenLTOVisibility(RD)
734	? llvm::Intrinsic::type_test
735	: llvm::Intrinsic::public_type_test;
736
737	CheckResult =
738	Builder.CreateCall(Callee: CGM.getIntrinsic(IID), Args: {VFPAddr, TypeId});
739	}
740
741	if (CGM.getItaniumVTableContext().isRelativeLayout()) {
742	VirtualFn = CGF.Builder.CreateCall(
743	CGM.getIntrinsic(llvm::Intrinsic::load_relative,
744	{VTableOffset->getType()}),
745	{VTable, VTableOffset});
746	} else {
747	llvm::Value *VFPAddr =
748	CGF.Builder.CreateGEP(Ty: CGF.Int8Ty, Ptr: VTable, IdxList: VTableOffset);
749	VirtualFn = CGF.Builder.CreateAlignedLoad(CGF.UnqualPtrTy, VFPAddr,
750	CGF.getPointerAlign(),
751	"memptr.virtualfn");
752	}
753	}
754	assert(VirtualFn && "Virtual fuction pointer not created!");
755	assert((!ShouldEmitCFICheck || !ShouldEmitVFEInfo || !ShouldEmitWPDInfo ||
756	CheckResult) &&
757	"Check result required but not created!");
758
759	if (ShouldEmitCFICheck) {
760	// If doing CFI, emit the check.
761	CheckSourceLocation = CGF.EmitCheckSourceLocation(Loc: E->getBeginLoc());
762	CheckTypeDesc = CGF.EmitCheckTypeDescriptor(T: QualType(MPT, 0));
763	llvm::Constant *StaticData[] = {
764	llvm::ConstantInt::get(Ty: CGF.Int8Ty, V: CodeGenFunction::CFITCK_VMFCall),
765	CheckSourceLocation,
766	CheckTypeDesc,
767	};
768
769	if (CGM.getCodeGenOpts().SanitizeTrap.has(K: SanitizerKind::CFIMFCall)) {
770	CGF.EmitTrapCheck(Checked: CheckResult, CheckHandlerID: SanitizerHandler::CFICheckFail);
771	} else {
772	llvm::Value *AllVtables = llvm::MetadataAsValue::get(
773	Context&: CGM.getLLVMContext(),
774	MD: llvm::MDString::get(Context&: CGM.getLLVMContext(), Str: "all-vtables"));
775	llvm::Value *ValidVtable = Builder.CreateCall(
776	CGM.getIntrinsic(llvm::Intrinsic::type_test), {VTable, AllVtables});
777	CGF.EmitCheck(Checked: std::make_pair(x&: CheckResult, y: SanitizerKind::CFIMFCall),
778	Check: SanitizerHandler::CFICheckFail, StaticArgs: StaticData,
779	DynamicArgs: {VTable, ValidVtable});
780	}
781
782	FnVirtual = Builder.GetInsertBlock();
783	}
784	} // End of sanitizer scope
785
786	CGF.EmitBranch(Block: FnEnd);
787
788	// In the non-virtual path, the function pointer is actually a
789	// function pointer.
790	CGF.EmitBlock(BB: FnNonVirtual);
791	llvm::Value *NonVirtualFn =
792	Builder.CreateIntToPtr(V: FnAsInt, DestTy: CGF.UnqualPtrTy, Name: "memptr.nonvirtualfn");
793
794	// Check the function pointer if CFI on member function pointers is enabled.
795	if (ShouldEmitCFICheck) {
796	CXXRecordDecl *RD = MPT->getClass()->getAsCXXRecordDecl();
797	if (RD->hasDefinition()) {
798	CodeGenFunction::SanitizerScope SanScope(&CGF);
799
800	llvm::Constant *StaticData[] = {
801	llvm::ConstantInt::get(Ty: CGF.Int8Ty, V: CodeGenFunction::CFITCK_NVMFCall),
802	CheckSourceLocation,
803	CheckTypeDesc,
804	};
805
806	llvm::Value *Bit = Builder.getFalse();
807	for (const CXXRecordDecl *Base : CGM.getMostBaseClasses(RD)) {
808	llvm::Metadata *MD = CGM.CreateMetadataIdentifierForType(
809	T: getContext().getMemberPointerType(
810	T: MPT->getPointeeType(),
811	Cls: getContext().getRecordType(Base).getTypePtr()));
812	llvm::Value *TypeId =
813	llvm::MetadataAsValue::get(Context&: CGF.getLLVMContext(), MD);
814
815	llvm::Value *TypeTest =
816	Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::type_test),
817	{NonVirtualFn, TypeId});
818	Bit = Builder.CreateOr(LHS: Bit, RHS: TypeTest);
819	}
820
821	CGF.EmitCheck(Checked: std::make_pair(x&: Bit, y: SanitizerKind::CFIMFCall),
822	Check: SanitizerHandler::CFICheckFail, StaticArgs: StaticData,
823	DynamicArgs: {NonVirtualFn, llvm::UndefValue::get(T: CGF.IntPtrTy)});
824
825	FnNonVirtual = Builder.GetInsertBlock();
826	}
827	}
828
829	// We're done.
830	CGF.EmitBlock(BB: FnEnd);
831	llvm::PHINode *CalleePtr = Builder.CreatePHI(Ty: CGF.UnqualPtrTy, NumReservedValues: 2);
832	CalleePtr->addIncoming(V: VirtualFn, BB: FnVirtual);
833	CalleePtr->addIncoming(V: NonVirtualFn, BB: FnNonVirtual);
834
835	CGCallee Callee(FPT, CalleePtr);
836	return Callee;
837	}
838
839	/// Compute an l-value by applying the given pointer-to-member to a
840	/// base object.
841	llvm::Value *ItaniumCXXABI::EmitMemberDataPointerAddress(
842	CodeGenFunction &CGF, const Expr *E, Address Base, llvm::Value *MemPtr,
843	const MemberPointerType *MPT) {
844	assert(MemPtr->getType() == CGM.PtrDiffTy);
845
846	CGBuilderTy &Builder = CGF.Builder;
847
848	// Apply the offset, which we assume is non-null.
849	return Builder.CreateInBoundsGEP(Ty: CGF.Int8Ty, Ptr: Base.emitRawPointer(CGF), IdxList: MemPtr,
850	Name: "memptr.offset");
851	}
852
853	/// Perform a bitcast, derived-to-base, or base-to-derived member pointer
854	/// conversion.
855	///
856	/// Bitcast conversions are always a no-op under Itanium.
857	///
858	/// Obligatory offset/adjustment diagram:
859	/// <-- offset --> <-- adjustment -->
860	/// |--------------------------|----------------------|--------------------|
861	/// ^Derived address point ^Base address point ^Member address point
862	///
863	/// So when converting a base member pointer to a derived member pointer,
864	/// we add the offset to the adjustment because the address point has
865	/// decreased; and conversely, when converting a derived MP to a base MP
866	/// we subtract the offset from the adjustment because the address point
867	/// has increased.
868	///
869	/// The standard forbids (at compile time) conversion to and from
870	/// virtual bases, which is why we don't have to consider them here.
871	///
872	/// The standard forbids (at run time) casting a derived MP to a base
873	/// MP when the derived MP does not point to a member of the base.
874	/// This is why -1 is a reasonable choice for null data member
875	/// pointers.
876	llvm::Value *
877	ItaniumCXXABI::EmitMemberPointerConversion(CodeGenFunction &CGF,
878	const CastExpr *E,
879	llvm::Value *src) {
880	assert(E->getCastKind() == CK_DerivedToBaseMemberPointer ||
881	E->getCastKind() == CK_BaseToDerivedMemberPointer ||
882	E->getCastKind() == CK_ReinterpretMemberPointer);
883
884	// Under Itanium, reinterprets don't require any additional processing.
885	if (E->getCastKind() == CK_ReinterpretMemberPointer) return src;
886
887	// Use constant emission if we can.
888	if (isa<llvm::Constant>(Val: src))
889	return EmitMemberPointerConversion(E, Src: cast<llvm::Constant>(Val: src));
890
891	llvm::Constant *adj = getMemberPointerAdjustment(E);
892	if (!adj) return src;
893
894	CGBuilderTy &Builder = CGF.Builder;
895	bool isDerivedToBase = (E->getCastKind() == CK_DerivedToBaseMemberPointer);
896
897	const MemberPointerType *destTy =
898	E->getType()->castAs<MemberPointerType>();
899
900	// For member data pointers, this is just a matter of adding the
901	// offset if the source is non-null.
902	if (destTy->isMemberDataPointer()) {
903	llvm::Value *dst;
904	if (isDerivedToBase)
905	dst = Builder.CreateNSWSub(LHS: src, RHS: adj, Name: "adj");
906	else
907	dst = Builder.CreateNSWAdd(LHS: src, RHS: adj, Name: "adj");
908
909	// Null check.
910	llvm::Value *null = llvm::Constant::getAllOnesValue(Ty: src->getType());
911	llvm::Value *isNull = Builder.CreateICmpEQ(LHS: src, RHS: null, Name: "memptr.isnull");
912	return Builder.CreateSelect(C: isNull, True: src, False: dst);
913	}
914
915	// The this-adjustment is left-shifted by 1 on ARM.
916	if (UseARMMethodPtrABI) {
917	uint64_t offset = cast<llvm::ConstantInt>(Val: adj)->getZExtValue();
918	offset <<= 1;
919	adj = llvm::ConstantInt::get(Ty: adj->getType(), V: offset);
920	}
921
922	llvm::Value *srcAdj = Builder.CreateExtractValue(Agg: src, Idxs: 1, Name: "src.adj");
923	llvm::Value *dstAdj;
924	if (isDerivedToBase)
925	dstAdj = Builder.CreateNSWSub(LHS: srcAdj, RHS: adj, Name: "adj");
926	else
927	dstAdj = Builder.CreateNSWAdd(LHS: srcAdj, RHS: adj, Name: "adj");
928
929	return Builder.CreateInsertValue(Agg: src, Val: dstAdj, Idxs: 1);
930	}
931
932	llvm::Constant *
933	ItaniumCXXABI::EmitMemberPointerConversion(const CastExpr *E,
934	llvm::Constant *src) {
935	assert(E->getCastKind() == CK_DerivedToBaseMemberPointer ||
936	E->getCastKind() == CK_BaseToDerivedMemberPointer ||
937	E->getCastKind() == CK_ReinterpretMemberPointer);
938
939	// Under Itanium, reinterprets don't require any additional processing.
940	if (E->getCastKind() == CK_ReinterpretMemberPointer) return src;
941
942	// If the adjustment is trivial, we don't need to do anything.
943	llvm::Constant *adj = getMemberPointerAdjustment(E);
944	if (!adj) return src;
945
946	bool isDerivedToBase = (E->getCastKind() == CK_DerivedToBaseMemberPointer);
947
948	const MemberPointerType *destTy =
949	E->getType()->castAs<MemberPointerType>();
950
951	// For member data pointers, this is just a matter of adding the
952	// offset if the source is non-null.
953	if (destTy->isMemberDataPointer()) {
954	// null maps to null.
955	if (src->isAllOnesValue()) return src;
956
957	if (isDerivedToBase)
958	return llvm::ConstantExpr::getNSWSub(C1: src, C2: adj);
959	else
960	return llvm::ConstantExpr::getNSWAdd(C1: src, C2: adj);
961	}
962
963	// The this-adjustment is left-shifted by 1 on ARM.
964	if (UseARMMethodPtrABI) {
965	uint64_t offset = cast<llvm::ConstantInt>(Val: adj)->getZExtValue();
966	offset <<= 1;
967	adj = llvm::ConstantInt::get(Ty: adj->getType(), V: offset);
968	}
969
970	llvm::Constant *srcAdj = src->getAggregateElement(Elt: 1);
971	llvm::Constant *dstAdj;
972	if (isDerivedToBase)
973	dstAdj = llvm::ConstantExpr::getNSWSub(C1: srcAdj, C2: adj);
974	else
975	dstAdj = llvm::ConstantExpr::getNSWAdd(C1: srcAdj, C2: adj);
976
977	llvm::Constant *res = ConstantFoldInsertValueInstruction(Agg: src, Val: dstAdj, Idxs: 1);
978	assert(res != nullptr && "Folding must succeed");
979	return res;
980	}
981
982	llvm::Constant *
983	ItaniumCXXABI::EmitNullMemberPointer(const MemberPointerType *MPT) {
984	// Itanium C++ ABI 2.3:
985	// A NULL pointer is represented as -1.
986	if (MPT->isMemberDataPointer())
987	return llvm::ConstantInt::get(Ty: CGM.PtrDiffTy, V: -1ULL, /*isSigned=*/IsSigned: true);
988
989	llvm::Constant *Zero = llvm::ConstantInt::get(Ty: CGM.PtrDiffTy, V: 0);
990	llvm::Constant *Values[2] = { Zero, Zero };
991	return llvm::ConstantStruct::getAnon(V: Values);
992	}
993
994	llvm::Constant *
995	ItaniumCXXABI::EmitMemberDataPointer(const MemberPointerType *MPT,
996	CharUnits offset) {
997	// Itanium C++ ABI 2.3:
998	// A pointer to data member is an offset from the base address of
999	// the class object containing it, represented as a ptrdiff_t
1000	return llvm::ConstantInt::get(Ty: CGM.PtrDiffTy, V: offset.getQuantity());
1001	}
1002
1003	llvm::Constant *
1004	ItaniumCXXABI::EmitMemberFunctionPointer(const CXXMethodDecl *MD) {
1005	return BuildMemberPointer(MD, ThisAdjustment: CharUnits::Zero());
1006	}
1007
1008	llvm::Constant *ItaniumCXXABI::BuildMemberPointer(const CXXMethodDecl *MD,
1009	CharUnits ThisAdjustment) {
1010	assert(MD->isInstance() && "Member function must not be static!");
1011
1012	CodeGenTypes &Types = CGM.getTypes();
1013
1014	// Get the function pointer (or index if this is a virtual function).
1015	llvm::Constant *MemPtr[2];
1016	if (MD->isVirtual()) {
1017	uint64_t Index = CGM.getItaniumVTableContext().getMethodVTableIndex(MD);
1018	uint64_t VTableOffset;
1019	if (CGM.getItaniumVTableContext().isRelativeLayout()) {
1020	// Multiply by 4-byte relative offsets.
1021	VTableOffset = Index * 4;
1022	} else {
1023	const ASTContext &Context = getContext();
1024	CharUnits PointerWidth = Context.toCharUnitsFromBits(
1025	BitSize: Context.getTargetInfo().getPointerWidth(AddrSpace: LangAS::Default));
1026	VTableOffset = Index * PointerWidth.getQuantity();
1027	}
1028
1029	if (UseARMMethodPtrABI) {
1030	// ARM C++ ABI 3.2.1:
1031	// This ABI specifies that adj contains twice the this
1032	// adjustment, plus 1 if the member function is virtual. The
1033	// least significant bit of adj then makes exactly the same
1034	// discrimination as the least significant bit of ptr does for
1035	// Itanium.
1036	MemPtr[0] = llvm::ConstantInt::get(Ty: CGM.PtrDiffTy, V: VTableOffset);
1037	MemPtr[1] = llvm::ConstantInt::get(Ty: CGM.PtrDiffTy,
1038	V: 2 * ThisAdjustment.getQuantity() + 1);
1039	} else {
1040	// Itanium C++ ABI 2.3:
1041	// For a virtual function, [the pointer field] is 1 plus the
1042	// virtual table offset (in bytes) of the function,
1043	// represented as a ptrdiff_t.
1044	MemPtr[0] = llvm::ConstantInt::get(Ty: CGM.PtrDiffTy, V: VTableOffset + 1);
1045	MemPtr[1] = llvm::ConstantInt::get(Ty: CGM.PtrDiffTy,
1046	V: ThisAdjustment.getQuantity());
1047	}
1048	} else {
1049	const FunctionProtoType *FPT = MD->getType()->castAs<FunctionProtoType>();
1050	llvm::Type *Ty;
1051	// Check whether the function has a computable LLVM signature.
1052	if (Types.isFuncTypeConvertible(FPT)) {
1053	// The function has a computable LLVM signature; use the correct type.
1054	Ty = Types.GetFunctionType(Info: Types.arrangeCXXMethodDeclaration(MD));
1055	} else {
1056	// Use an arbitrary non-function type to tell GetAddrOfFunction that the
1057	// function type is incomplete.
1058	Ty = CGM.PtrDiffTy;
1059	}
1060	llvm::Constant *addr = CGM.GetAddrOfFunction(MD, Ty);
1061
1062	MemPtr[0] = llvm::ConstantExpr::getPtrToInt(C: addr, Ty: CGM.PtrDiffTy);
1063	MemPtr[1] = llvm::ConstantInt::get(Ty: CGM.PtrDiffTy,
1064	V: (UseARMMethodPtrABI ? 2 : 1) *
1065	ThisAdjustment.getQuantity());
1066	}
1067
1068	return llvm::ConstantStruct::getAnon(V: MemPtr);
1069	}
1070
1071	llvm::Constant *ItaniumCXXABI::EmitMemberPointer(const APValue &MP,
1072	QualType MPType) {
1073	const MemberPointerType *MPT = MPType->castAs<MemberPointerType>();
1074	const ValueDecl *MPD = MP.getMemberPointerDecl();
1075	if (!MPD)
1076	return EmitNullMemberPointer(MPT);
1077
1078	CharUnits ThisAdjustment = getContext().getMemberPointerPathAdjustment(MP);
1079
1080	if (const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(Val: MPD))
1081	return BuildMemberPointer(MD, ThisAdjustment);
1082
1083	CharUnits FieldOffset =
1084	getContext().toCharUnitsFromBits(BitSize: getContext().getFieldOffset(FD: MPD));
1085	return EmitMemberDataPointer(MPT, offset: ThisAdjustment + FieldOffset);
1086	}
1087
1088	/// The comparison algorithm is pretty easy: the member pointers are
1089	/// the same if they're either bitwise identical *or* both null.
1090	///
1091	/// ARM is different here only because null-ness is more complicated.
1092	llvm::Value *
1093	ItaniumCXXABI::EmitMemberPointerComparison(CodeGenFunction &CGF,
1094	llvm::Value *L,
1095	llvm::Value *R,
1096	const MemberPointerType *MPT,
1097	bool Inequality) {
1098	CGBuilderTy &Builder = CGF.Builder;
1099
1100	llvm::ICmpInst::Predicate Eq;
1101	llvm::Instruction::BinaryOps And, Or;
1102	if (Inequality) {
1103	Eq = llvm::ICmpInst::ICMP_NE;
1104	And = llvm::Instruction::Or;
1105	Or = llvm::Instruction::And;
1106	} else {
1107	Eq = llvm::ICmpInst::ICMP_EQ;
1108	And = llvm::Instruction::And;
1109	Or = llvm::Instruction::Or;
1110	}
1111
1112	// Member data pointers are easy because there's a unique null
1113	// value, so it just comes down to bitwise equality.
1114	if (MPT->isMemberDataPointer())
1115	return Builder.CreateICmp(P: Eq, LHS: L, RHS: R);
1116
1117	// For member function pointers, the tautologies are more complex.
1118	// The Itanium tautology is:
1119	// (L == R) <==> (L.ptr == R.ptr && (L.ptr == 0 || L.adj == R.adj))
1120	// The ARM tautology is:
1121	// (L == R) <==> (L.ptr == R.ptr &&
1122	// (L.adj == R.adj ||
1123	// (L.ptr == 0 && ((L.adj|R.adj) & 1) == 0)))
1124	// The inequality tautologies have exactly the same structure, except
1125	// applying De Morgan's laws.
1126
1127	llvm::Value *LPtr = Builder.CreateExtractValue(Agg: L, Idxs: 0, Name: "lhs.memptr.ptr");
1128	llvm::Value *RPtr = Builder.CreateExtractValue(Agg: R, Idxs: 0, Name: "rhs.memptr.ptr");
1129
1130	// This condition tests whether L.ptr == R.ptr. This must always be
1131	// true for equality to hold.
1132	llvm::Value *PtrEq = Builder.CreateICmp(P: Eq, LHS: LPtr, RHS: RPtr, Name: "cmp.ptr");
1133
1134	// This condition, together with the assumption that L.ptr == R.ptr,
1135	// tests whether the pointers are both null. ARM imposes an extra
1136	// condition.
1137	llvm::Value *Zero = llvm::Constant::getNullValue(Ty: LPtr->getType());
1138	llvm::Value *EqZero = Builder.CreateICmp(P: Eq, LHS: LPtr, RHS: Zero, Name: "cmp.ptr.null");
1139
1140	// This condition tests whether L.adj == R.adj. If this isn't
1141	// true, the pointers are unequal unless they're both null.
1142	llvm::Value *LAdj = Builder.CreateExtractValue(Agg: L, Idxs: 1, Name: "lhs.memptr.adj");
1143	llvm::Value *RAdj = Builder.CreateExtractValue(Agg: R, Idxs: 1, Name: "rhs.memptr.adj");
1144	llvm::Value *AdjEq = Builder.CreateICmp(P: Eq, LHS: LAdj, RHS: RAdj, Name: "cmp.adj");
1145
1146	// Null member function pointers on ARM clear the low bit of Adj,
1147	// so the zero condition has to check that neither low bit is set.
1148	if (UseARMMethodPtrABI) {
1149	llvm::Value *One = llvm::ConstantInt::get(Ty: LPtr->getType(), V: 1);
1150
1151	// Compute (l.adj | r.adj) & 1 and test it against zero.
1152	llvm::Value *OrAdj = Builder.CreateOr(LHS: LAdj, RHS: RAdj, Name: "or.adj");
1153	llvm::Value *OrAdjAnd1 = Builder.CreateAnd(LHS: OrAdj, RHS: One);
1154	llvm::Value *OrAdjAnd1EqZero = Builder.CreateICmp(P: Eq, LHS: OrAdjAnd1, RHS: Zero,
1155	Name: "cmp.or.adj");
1156	EqZero = Builder.CreateBinOp(Opc: And, LHS: EqZero, RHS: OrAdjAnd1EqZero);
1157	}
1158
1159	// Tie together all our conditions.
1160	llvm::Value *Result = Builder.CreateBinOp(Opc: Or, LHS: EqZero, RHS: AdjEq);
1161	Result = Builder.CreateBinOp(Opc: And, LHS: PtrEq, RHS: Result,
1162	Name: Inequality ? "memptr.ne" : "memptr.eq");
1163	return Result;
1164	}
1165
1166	llvm::Value *
1167	ItaniumCXXABI::EmitMemberPointerIsNotNull(CodeGenFunction &CGF,
1168	llvm::Value *MemPtr,
1169	const MemberPointerType *MPT) {
1170	CGBuilderTy &Builder = CGF.Builder;
1171
1172	/// For member data pointers, this is just a check against -1.
1173	if (MPT->isMemberDataPointer()) {
1174	assert(MemPtr->getType() == CGM.PtrDiffTy);
1175	llvm::Value *NegativeOne =
1176	llvm::Constant::getAllOnesValue(Ty: MemPtr->getType());
1177	return Builder.CreateICmpNE(LHS: MemPtr, RHS: NegativeOne, Name: "memptr.tobool");
1178	}
1179
1180	// In Itanium, a member function pointer is not null if 'ptr' is not null.
1181	llvm::Value *Ptr = Builder.CreateExtractValue(Agg: MemPtr, Idxs: 0, Name: "memptr.ptr");
1182
1183	llvm::Constant *Zero = llvm::ConstantInt::get(Ty: Ptr->getType(), V: 0);
1184	llvm::Value *Result = Builder.CreateICmpNE(LHS: Ptr, RHS: Zero, Name: "memptr.tobool");
1185
1186	// On ARM, a member function pointer is also non-null if the low bit of 'adj'
1187	// (the virtual bit) is set.
1188	if (UseARMMethodPtrABI) {
1189	llvm::Constant *One = llvm::ConstantInt::get(Ty: Ptr->getType(), V: 1);
1190	llvm::Value *Adj = Builder.CreateExtractValue(Agg: MemPtr, Idxs: 1, Name: "memptr.adj");
1191	llvm::Value *VirtualBit = Builder.CreateAnd(LHS: Adj, RHS: One, Name: "memptr.virtualbit");
1192	llvm::Value *IsVirtual = Builder.CreateICmpNE(LHS: VirtualBit, RHS: Zero,
1193	Name: "memptr.isvirtual");
1194	Result = Builder.CreateOr(LHS: Result, RHS: IsVirtual);
1195	}
1196
1197	return Result;
1198	}
1199
1200	bool ItaniumCXXABI::classifyReturnType(CGFunctionInfo &FI) const {
1201	const CXXRecordDecl *RD = FI.getReturnType()->getAsCXXRecordDecl();
1202	if (!RD)
1203	return false;
1204
1205	// If C++ prohibits us from making a copy, return by address.
1206	if (!RD->canPassInRegisters()) {
1207	auto Align = CGM.getContext().getTypeAlignInChars(T: FI.getReturnType());
1208	FI.getReturnInfo() = ABIArgInfo::getIndirect(Alignment: Align, /*ByVal=*/false);
1209	return true;
1210	}
1211	return false;
1212	}
1213
1214	/// The Itanium ABI requires non-zero initialization only for data
1215	/// member pointers, for which '0' is a valid offset.
1216	bool ItaniumCXXABI::isZeroInitializable(const MemberPointerType *MPT) {
1217	return MPT->isMemberFunctionPointer();
1218	}
1219
1220	/// The Itanium ABI always places an offset to the complete object
1221	/// at entry -2 in the vtable.
1222	void ItaniumCXXABI::emitVirtualObjectDelete(CodeGenFunction &CGF,
1223	const CXXDeleteExpr *DE,
1224	Address Ptr,
1225	QualType ElementType,
1226	const CXXDestructorDecl *Dtor) {
1227	bool UseGlobalDelete = DE->isGlobalDelete();
1228	if (UseGlobalDelete) {
1229	// Derive the complete-object pointer, which is what we need
1230	// to pass to the deallocation function.
1231
1232	// Grab the vtable pointer as an intptr_t*.
1233	auto *ClassDecl =
1234	cast<CXXRecordDecl>(Val: ElementType->castAs<RecordType>()->getDecl());
1235	llvm::Value *VTable = CGF.GetVTablePtr(This: Ptr, VTableTy: CGF.UnqualPtrTy, VTableClass: ClassDecl);
1236
1237	// Track back to entry -2 and pull out the offset there.
1238	llvm::Value *OffsetPtr = CGF.Builder.CreateConstInBoundsGEP1_64(
1239	Ty: CGF.IntPtrTy, Ptr: VTable, Idx0: -2, Name: "complete-offset.ptr");
1240	llvm::Value *Offset = CGF.Builder.CreateAlignedLoad(CGF.IntPtrTy, OffsetPtr,
1241	CGF.getPointerAlign());
1242
1243	// Apply the offset.
1244	llvm::Value *CompletePtr = Ptr.emitRawPointer(CGF);
1245	CompletePtr =
1246	CGF.Builder.CreateInBoundsGEP(Ty: CGF.Int8Ty, Ptr: CompletePtr, IdxList: Offset);
1247
1248	// If we're supposed to call the global delete, make sure we do so
1249	// even if the destructor throws.
1250	CGF.pushCallObjectDeleteCleanup(OperatorDelete: DE->getOperatorDelete(), CompletePtr,
1251	ElementType);
1252	}
1253
1254	// FIXME: Provide a source location here even though there's no
1255	// CXXMemberCallExpr for dtor call.
1256	CXXDtorType DtorType = UseGlobalDelete ? Dtor_Complete : Dtor_Deleting;
1257	EmitVirtualDestructorCall(CGF, Dtor, DtorType, This: Ptr, E: DE);
1258
1259	if (UseGlobalDelete)
1260	CGF.PopCleanupBlock();
1261	}
1262
1263	void ItaniumCXXABI::emitRethrow(CodeGenFunction &CGF, bool isNoReturn) {
1264	// void __cxa_rethrow();
1265
1266	llvm::FunctionType *FTy =
1267	llvm::FunctionType::get(Result: CGM.VoidTy, /*isVarArg=*/false);
1268
1269	llvm::FunctionCallee Fn = CGM.CreateRuntimeFunction(Ty: FTy, Name: "__cxa_rethrow");
1270
1271	if (isNoReturn)
1272	CGF.EmitNoreturnRuntimeCallOrInvoke(callee: Fn, args: std::nullopt);
1273	else
1274	CGF.EmitRuntimeCallOrInvoke(callee: Fn);
1275	}
1276
1277	static llvm::FunctionCallee getAllocateExceptionFn(CodeGenModule &CGM) {
1278	// void *__cxa_allocate_exception(size_t thrown_size);
1279
1280	llvm::FunctionType *FTy =
1281	llvm::FunctionType::get(Result: CGM.Int8PtrTy, Params: CGM.SizeTy, /*isVarArg=*/false);
1282
1283	return CGM.CreateRuntimeFunction(Ty: FTy, Name: "__cxa_allocate_exception");
1284	}
1285
1286	static llvm::FunctionCallee getThrowFn(CodeGenModule &CGM) {
1287	// void __cxa_throw(void *thrown_exception, std::type_info *tinfo,
1288	// void (*dest) (void *));
1289
1290	llvm::Type *Args[3] = { CGM.Int8PtrTy, CGM.GlobalsInt8PtrTy, CGM.Int8PtrTy };
1291	llvm::FunctionType *FTy =
1292	llvm::FunctionType::get(Result: CGM.VoidTy, Params: Args, /*isVarArg=*/false);
1293
1294	return CGM.CreateRuntimeFunction(Ty: FTy, Name: "__cxa_throw");
1295	}
1296
1297	void ItaniumCXXABI::emitThrow(CodeGenFunction &CGF, const CXXThrowExpr *E) {
1298	QualType ThrowType = E->getSubExpr()->getType();
1299	// Now allocate the exception object.
1300	llvm::Type *SizeTy = CGF.ConvertType(T: getContext().getSizeType());
1301	uint64_t TypeSize = getContext().getTypeSizeInChars(T: ThrowType).getQuantity();
1302
1303	llvm::FunctionCallee AllocExceptionFn = getAllocateExceptionFn(CGM);
1304	llvm::CallInst *ExceptionPtr = CGF.EmitNounwindRuntimeCall(
1305	callee: AllocExceptionFn, args: llvm::ConstantInt::get(Ty: SizeTy, V: TypeSize), name: "exception");
1306
1307	CharUnits ExnAlign = CGF.getContext().getExnObjectAlignment();
1308	CGF.EmitAnyExprToExn(
1309	E: E->getSubExpr(), Addr: Address(ExceptionPtr, CGM.Int8Ty, ExnAlign));
1310
1311	// Now throw the exception.
1312	llvm::Constant *TypeInfo = CGM.GetAddrOfRTTIDescriptor(Ty: ThrowType,
1313	/*ForEH=*/true);
1314
1315	// The address of the destructor. If the exception type has a
1316	// trivial destructor (or isn't a record), we just pass null.
1317	llvm::Constant *Dtor = nullptr;
1318	if (const RecordType *RecordTy = ThrowType->getAs<RecordType>()) {
1319	CXXRecordDecl *Record = cast<CXXRecordDecl>(Val: RecordTy->getDecl());
1320	if (!Record->hasTrivialDestructor()) {
1321	CXXDestructorDecl *DtorD = Record->getDestructor();
1322	Dtor = CGM.getAddrOfCXXStructor(GD: GlobalDecl(DtorD, Dtor_Complete));
1323	}
1324	}
1325	if (!Dtor) Dtor = llvm::Constant::getNullValue(Ty: CGM.Int8PtrTy);
1326
1327	llvm::Value *args[] = { ExceptionPtr, TypeInfo, Dtor };
1328	CGF.EmitNoreturnRuntimeCallOrInvoke(callee: getThrowFn(CGM), args);
1329	}
1330
1331	static llvm::FunctionCallee getItaniumDynamicCastFn(CodeGenFunction &CGF) {
1332	// void *__dynamic_cast(const void *sub,
1333	// global_as const abi::__class_type_info *src,
1334	// global_as const abi::__class_type_info *dst,
1335	// std::ptrdiff_t src2dst_offset);
1336
1337	llvm::Type *Int8PtrTy = CGF.Int8PtrTy;
1338	llvm::Type *GlobInt8PtrTy = CGF.GlobalsInt8PtrTy;
1339	llvm::Type *PtrDiffTy =
1340	CGF.ConvertType(T: CGF.getContext().getPointerDiffType());
1341
1342	llvm::Type *Args[4] = { Int8PtrTy, GlobInt8PtrTy, GlobInt8PtrTy, PtrDiffTy };
1343
1344	llvm::FunctionType *FTy = llvm::FunctionType::get(Result: Int8PtrTy, Params: Args, isVarArg: false);
1345
1346	// Mark the function as nounwind willreturn readonly.
1347	llvm::AttrBuilder FuncAttrs(CGF.getLLVMContext());
1348	FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
1349	FuncAttrs.addAttribute(llvm::Attribute::WillReturn);
1350	FuncAttrs.addMemoryAttr(ME: llvm::MemoryEffects::readOnly());
1351	llvm::AttributeList Attrs = llvm::AttributeList::get(
1352	C&: CGF.getLLVMContext(), Index: llvm::AttributeList::FunctionIndex, B: FuncAttrs);
1353
1354	return CGF.CGM.CreateRuntimeFunction(Ty: FTy, Name: "__dynamic_cast", ExtraAttrs: Attrs);
1355	}
1356
1357	static llvm::FunctionCallee getBadCastFn(CodeGenFunction &CGF) {
1358	// void __cxa_bad_cast();
1359	llvm::FunctionType *FTy = llvm::FunctionType::get(Result: CGF.VoidTy, isVarArg: false);
1360	return CGF.CGM.CreateRuntimeFunction(Ty: FTy, Name: "__cxa_bad_cast");
1361	}
1362
1363	/// Compute the src2dst_offset hint as described in the
1364	/// Itanium C++ ABI [2.9.7]
1365	static CharUnits computeOffsetHint(ASTContext &Context,
1366	const CXXRecordDecl *Src,
1367	const CXXRecordDecl *Dst) {
1368	CXXBasePaths Paths(/*FindAmbiguities=*/true, /*RecordPaths=*/true,
1369	/*DetectVirtual=*/false);
1370
1371	// If Dst is not derived from Src we can skip the whole computation below and
1372	// return that Src is not a public base of Dst. Record all inheritance paths.
1373	if (!Dst->isDerivedFrom(Base: Src, Paths))
1374	return CharUnits::fromQuantity(Quantity: -2ULL);
1375
1376	unsigned NumPublicPaths = 0;
1377	CharUnits Offset;
1378
1379	// Now walk all possible inheritance paths.
1380	for (const CXXBasePath &Path : Paths) {
1381	if (Path.Access != AS_public) // Ignore non-public inheritance.
1382	continue;
1383
1384	++NumPublicPaths;
1385
1386	for (const CXXBasePathElement &PathElement : Path) {
1387	// If the path contains a virtual base class we can't give any hint.
1388	// -1: no hint.
1389	if (PathElement.Base->isVirtual())
1390	return CharUnits::fromQuantity(Quantity: -1ULL);
1391
1392	if (NumPublicPaths > 1) // Won't use offsets, skip computation.
1393	continue;
1394
1395	// Accumulate the base class offsets.
1396	const ASTRecordLayout &L = Context.getASTRecordLayout(PathElement.Class);
1397	Offset += L.getBaseClassOffset(
1398	Base: PathElement.Base->getType()->getAsCXXRecordDecl());
1399	}
1400	}
1401
1402	// -2: Src is not a public base of Dst.
1403	if (NumPublicPaths == 0)
1404	return CharUnits::fromQuantity(Quantity: -2ULL);
1405
1406	// -3: Src is a multiple public base type but never a virtual base type.
1407	if (NumPublicPaths > 1)
1408	return CharUnits::fromQuantity(Quantity: -3ULL);
1409
1410	// Otherwise, the Src type is a unique public nonvirtual base type of Dst.
1411	// Return the offset of Src from the origin of Dst.
1412	return Offset;
1413	}
1414
1415	static llvm::FunctionCallee getBadTypeidFn(CodeGenFunction &CGF) {
1416	// void __cxa_bad_typeid();
1417	llvm::FunctionType *FTy = llvm::FunctionType::get(Result: CGF.VoidTy, isVarArg: false);
1418
1419	return CGF.CGM.CreateRuntimeFunction(Ty: FTy, Name: "__cxa_bad_typeid");
1420	}
1421
1422	bool ItaniumCXXABI::shouldTypeidBeNullChecked(bool IsDeref,
1423	QualType SrcRecordTy) {
1424	return IsDeref;
1425	}
1426
1427	void ItaniumCXXABI::EmitBadTypeidCall(CodeGenFunction &CGF) {
1428	llvm::FunctionCallee Fn = getBadTypeidFn(CGF);
1429	llvm::CallBase *Call = CGF.EmitRuntimeCallOrInvoke(callee: Fn);
1430	Call->setDoesNotReturn();
1431	CGF.Builder.CreateUnreachable();
1432	}
1433
1434	llvm::Value *ItaniumCXXABI::EmitTypeid(CodeGenFunction &CGF,
1435	QualType SrcRecordTy,
1436	Address ThisPtr,
1437	llvm::Type *StdTypeInfoPtrTy) {
1438	auto *ClassDecl =
1439	cast<CXXRecordDecl>(Val: SrcRecordTy->castAs<RecordType>()->getDecl());
1440	llvm::Value *Value = CGF.GetVTablePtr(This: ThisPtr, VTableTy: CGM.GlobalsInt8PtrTy,
1441	VTableClass: ClassDecl);
1442
1443	if (CGM.getItaniumVTableContext().isRelativeLayout()) {
1444	// Load the type info.
1445	Value = CGF.Builder.CreateCall(
1446	CGM.getIntrinsic(llvm::Intrinsic::load_relative, {CGM.Int32Ty}),
1447	{Value, llvm::ConstantInt::get(CGM.Int32Ty, -4)});
1448	} else {
1449	// Load the type info.
1450	Value =
1451	CGF.Builder.CreateConstInBoundsGEP1_64(Ty: StdTypeInfoPtrTy, Ptr: Value, Idx0: -1ULL);
1452	}
1453	return CGF.Builder.CreateAlignedLoad(StdTypeInfoPtrTy, Value,
1454	CGF.getPointerAlign());
1455	}
1456
1457	bool ItaniumCXXABI::shouldDynamicCastCallBeNullChecked(bool SrcIsPtr,
1458	QualType SrcRecordTy) {
1459	return SrcIsPtr;
1460	}
1461
1462	llvm::Value *ItaniumCXXABI::emitDynamicCastCall(
1463	CodeGenFunction &CGF, Address ThisAddr, QualType SrcRecordTy,
1464	QualType DestTy, QualType DestRecordTy, llvm::BasicBlock *CastEnd) {
1465	llvm::Type *PtrDiffLTy =
1466	CGF.ConvertType(T: CGF.getContext().getPointerDiffType());
1467
1468	llvm::Value *SrcRTTI =
1469	CGF.CGM.GetAddrOfRTTIDescriptor(Ty: SrcRecordTy.getUnqualifiedType());
1470	llvm::Value *DestRTTI =
1471	CGF.CGM.GetAddrOfRTTIDescriptor(Ty: DestRecordTy.getUnqualifiedType());
1472
1473	// Compute the offset hint.
1474	const CXXRecordDecl *SrcDecl = SrcRecordTy->getAsCXXRecordDecl();
1475	const CXXRecordDecl *DestDecl = DestRecordTy->getAsCXXRecordDecl();
1476	llvm::Value *OffsetHint = llvm::ConstantInt::get(
1477	Ty: PtrDiffLTy,
1478	V: computeOffsetHint(Context&: CGF.getContext(), Src: SrcDecl, Dst: DestDecl).getQuantity());
1479
1480	// Emit the call to __dynamic_cast.
1481	llvm::Value *Args[] = {ThisAddr.emitRawPointer(CGF), SrcRTTI, DestRTTI,
1482	OffsetHint};
1483	llvm::Value *Value =
1484	CGF.EmitNounwindRuntimeCall(callee: getItaniumDynamicCastFn(CGF), args: Args);
1485
1486	/// C++ [expr.dynamic.cast]p9:
1487	/// A failed cast to reference type throws std::bad_cast
1488	if (DestTy->isReferenceType()) {
1489	llvm::BasicBlock *BadCastBlock =
1490	CGF.createBasicBlock(name: "dynamic_cast.bad_cast");
1491
1492	llvm::Value *IsNull = CGF.Builder.CreateIsNull(Arg: Value);
1493	CGF.Builder.CreateCondBr(Cond: IsNull, True: BadCastBlock, False: CastEnd);
1494
1495	CGF.EmitBlock(BB: BadCastBlock);
1496	EmitBadCastCall(CGF);
1497	}
1498
1499	return Value;
1500	}
1501
1502	llvm::Value *ItaniumCXXABI::emitExactDynamicCast(
1503	CodeGenFunction &CGF, Address ThisAddr, QualType SrcRecordTy,
1504	QualType DestTy, QualType DestRecordTy, llvm::BasicBlock *CastSuccess,
1505	llvm::BasicBlock *CastFail) {
1506	ASTContext &Context = getContext();
1507
1508	// Find all the inheritance paths.
1509	const CXXRecordDecl *SrcDecl = SrcRecordTy->getAsCXXRecordDecl();
1510	const CXXRecordDecl *DestDecl = DestRecordTy->getAsCXXRecordDecl();
1511	CXXBasePaths Paths(/*FindAmbiguities=*/true, /*RecordPaths=*/true,
1512	/*DetectVirtual=*/false);
1513	(void)DestDecl->isDerivedFrom(Base: SrcDecl, Paths);
1514
1515	// Find an offset within `DestDecl` where a `SrcDecl` instance and its vptr
1516	// might appear.
1517	std::optional<CharUnits> Offset;
1518	for (const CXXBasePath &Path : Paths) {
1519	// dynamic_cast only finds public inheritance paths.
1520	if (Path.Access != AS_public)
1521	continue;
1522
1523	CharUnits PathOffset;
1524	for (const CXXBasePathElement &PathElement : Path) {
1525	// Find the offset along this inheritance step.
1526	const CXXRecordDecl *Base =
1527	PathElement.Base->getType()->getAsCXXRecordDecl();
1528	if (PathElement.Base->isVirtual()) {
1529	// For a virtual base class, we know that the derived class is exactly
1530	// DestDecl, so we can use the vbase offset from its layout.
1531	const ASTRecordLayout &L = Context.getASTRecordLayout(DestDecl);
1532	PathOffset = L.getVBaseClassOffset(VBase: Base);
1533	} else {
1534	const ASTRecordLayout &L =
1535	Context.getASTRecordLayout(PathElement.Class);
1536	PathOffset += L.getBaseClassOffset(Base);
1537	}
1538	}
1539
1540	if (!Offset)
1541	Offset = PathOffset;
1542	else if (Offset != PathOffset) {
1543	// Base appears in at least two different places. Find the most-derived
1544	// object and see if it's a DestDecl. Note that the most-derived object
1545	// must be at least as aligned as this base class subobject, and must
1546	// have a vptr at offset 0.
1547	ThisAddr = Address(emitDynamicCastToVoid(CGF, Value: ThisAddr, SrcRecordTy),
1548	CGF.VoidPtrTy, ThisAddr.getAlignment());
1549	SrcDecl = DestDecl;
1550	Offset = CharUnits::Zero();
1551	break;
1552	}
1553	}
1554
1555	if (!Offset) {
1556	// If there are no public inheritance paths, the cast always fails.
1557	CGF.EmitBranch(Block: CastFail);
1558	return llvm::PoisonValue::get(T: CGF.VoidPtrTy);
1559	}
1560
1561	// Compare the vptr against the expected vptr for the destination type at
1562	// this offset. Note that we do not know what type ThisAddr points to in
1563	// the case where the derived class multiply inherits from the base class
1564	// so we can't use GetVTablePtr, so we load the vptr directly instead.
1565	llvm::Instruction *VPtr = CGF.Builder.CreateLoad(
1566	Addr: ThisAddr.withElementType(ElemTy: CGF.VoidPtrPtrTy), Name: "vtable");
1567	CGM.DecorateInstructionWithTBAA(
1568	Inst: VPtr, TBAAInfo: CGM.getTBAAVTablePtrAccessInfo(VTablePtrType: CGF.VoidPtrPtrTy));
1569	llvm::Value *Success = CGF.Builder.CreateICmpEQ(
1570	LHS: VPtr, RHS: getVTableAddressPoint(Base: BaseSubobject(SrcDecl, *Offset), VTableClass: DestDecl));
1571	llvm::Value *Result = ThisAddr.emitRawPointer(CGF);
1572	if (!Offset->isZero())
1573	Result = CGF.Builder.CreateInBoundsGEP(
1574	Ty: CGF.CharTy, Ptr: Result,
1575	IdxList: {llvm::ConstantInt::get(Ty: CGF.PtrDiffTy, V: -Offset->getQuantity())});
1576	CGF.Builder.CreateCondBr(Cond: Success, True: CastSuccess, False: CastFail);
1577	return Result;
1578	}
1579
1580	llvm::Value *ItaniumCXXABI::emitDynamicCastToVoid(CodeGenFunction &CGF,
1581	Address ThisAddr,
1582	QualType SrcRecordTy) {
1583	auto *ClassDecl =
1584	cast<CXXRecordDecl>(Val: SrcRecordTy->castAs<RecordType>()->getDecl());
1585	llvm::Value *OffsetToTop;
1586	if (CGM.getItaniumVTableContext().isRelativeLayout()) {
1587	// Get the vtable pointer.
1588	llvm::Value *VTable =
1589	CGF.GetVTablePtr(This: ThisAddr, VTableTy: CGF.UnqualPtrTy, VTableClass: ClassDecl);
1590
1591	// Get the offset-to-top from the vtable.
1592	OffsetToTop =
1593	CGF.Builder.CreateConstInBoundsGEP1_32(Ty: CGM.Int32Ty, Ptr: VTable, Idx0: -2U);
1594	OffsetToTop = CGF.Builder.CreateAlignedLoad(
1595	Ty: CGM.Int32Ty, Addr: OffsetToTop, Align: CharUnits::fromQuantity(Quantity: 4), Name: "offset.to.top");
1596	} else {
1597	llvm::Type *PtrDiffLTy =
1598	CGF.ConvertType(T: CGF.getContext().getPointerDiffType());
1599
1600	// Get the vtable pointer.
1601	llvm::Value *VTable =
1602	CGF.GetVTablePtr(This: ThisAddr, VTableTy: CGF.UnqualPtrTy, VTableClass: ClassDecl);
1603
1604	// Get the offset-to-top from the vtable.
1605	OffsetToTop =
1606	CGF.Builder.CreateConstInBoundsGEP1_64(Ty: PtrDiffLTy, Ptr: VTable, Idx0: -2ULL);
1607	OffsetToTop = CGF.Builder.CreateAlignedLoad(
1608	PtrDiffLTy, OffsetToTop, CGF.getPointerAlign(), "offset.to.top");
1609	}
1610	// Finally, add the offset to the pointer.
1611	return CGF.Builder.CreateInBoundsGEP(Ty: CGF.Int8Ty, Ptr: ThisAddr.emitRawPointer(CGF),
1612	IdxList: OffsetToTop);
1613	}
1614
1615	bool ItaniumCXXABI::EmitBadCastCall(CodeGenFunction &CGF) {
1616	llvm::FunctionCallee Fn = getBadCastFn(CGF);
1617	llvm::CallBase *Call = CGF.EmitRuntimeCallOrInvoke(callee: Fn);
1618	Call->setDoesNotReturn();
1619	CGF.Builder.CreateUnreachable();
1620	return true;
1621	}
1622
1623	llvm::Value *
1624	ItaniumCXXABI::GetVirtualBaseClassOffset(CodeGenFunction &CGF,
1625	Address This,
1626	const CXXRecordDecl *ClassDecl,
1627	const CXXRecordDecl *BaseClassDecl) {
1628	llvm::Value *VTablePtr = CGF.GetVTablePtr(This, VTableTy: CGM.Int8PtrTy, VTableClass: ClassDecl);
1629	CharUnits VBaseOffsetOffset =
1630	CGM.getItaniumVTableContext().getVirtualBaseOffsetOffset(RD: ClassDecl,
1631	VBase: BaseClassDecl);
1632	llvm::Value *VBaseOffsetPtr =
1633	CGF.Builder.CreateConstGEP1_64(
1634	Ty: CGF.Int8Ty, Ptr: VTablePtr, Idx0: VBaseOffsetOffset.getQuantity(),
1635	Name: "vbase.offset.ptr");
1636
1637	llvm::Value *VBaseOffset;
1638	if (CGM.getItaniumVTableContext().isRelativeLayout()) {
1639	VBaseOffset = CGF.Builder.CreateAlignedLoad(
1640	Ty: CGF.Int32Ty, Addr: VBaseOffsetPtr, Align: CharUnits::fromQuantity(Quantity: 4),
1641	Name: "vbase.offset");
1642	} else {
1643	VBaseOffset = CGF.Builder.CreateAlignedLoad(
1644	CGM.PtrDiffTy, VBaseOffsetPtr, CGF.getPointerAlign(), "vbase.offset");
1645	}
1646	return VBaseOffset;
1647	}
1648
1649	void ItaniumCXXABI::EmitCXXConstructors(const CXXConstructorDecl *D) {
1650	// Just make sure we're in sync with TargetCXXABI.
1651	assert(CGM.getTarget().getCXXABI().hasConstructorVariants());
1652
1653	// The constructor used for constructing this as a base class;
1654	// ignores virtual bases.
1655	CGM.EmitGlobal(D: GlobalDecl(D, Ctor_Base));
1656
1657	// The constructor used for constructing this as a complete class;
1658	// constructs the virtual bases, then calls the base constructor.
1659	if (!D->getParent()->isAbstract()) {
1660	// We don't need to emit the complete ctor if the class is abstract.
1661	CGM.EmitGlobal(D: GlobalDecl(D, Ctor_Complete));
1662	}
1663	}
1664
1665	CGCXXABI::AddedStructorArgCounts
1666	ItaniumCXXABI::buildStructorSignature(GlobalDecl GD,
1667	SmallVectorImpl<CanQualType> &ArgTys) {
1668	ASTContext &Context = getContext();
1669
1670	// All parameters are already in place except VTT, which goes after 'this'.
1671	// These are Clang types, so we don't need to worry about sret yet.
1672
1673	// Check if we need to add a VTT parameter (which has type global void **).
1674	if ((isa<CXXConstructorDecl>(Val: GD.getDecl()) ? GD.getCtorType() == Ctor_Base
1675	: GD.getDtorType() == Dtor_Base) &&
1676	cast<CXXMethodDecl>(Val: GD.getDecl())->getParent()->getNumVBases() != 0) {
1677	LangAS AS = CGM.GetGlobalVarAddressSpace(D: nullptr);
1678	QualType Q = Context.getAddrSpaceQualType(T: Context.VoidPtrTy, AddressSpace: AS);
1679	ArgTys.insert(I: ArgTys.begin() + 1,
1680	Elt: Context.getPointerType(T: CanQualType::CreateUnsafe(Other: Q)));
1681	return AddedStructorArgCounts::prefix(N: 1);
1682	}
1683	return AddedStructorArgCounts{};
1684	}
1685
1686	void ItaniumCXXABI::EmitCXXDestructors(const CXXDestructorDecl *D) {
1687	// The destructor used for destructing this as a base class; ignores
1688	// virtual bases.
1689	CGM.EmitGlobal(D: GlobalDecl(D, Dtor_Base));
1690
1691	// The destructor used for destructing this as a most-derived class;
1692	// call the base destructor and then destructs any virtual bases.
1693	CGM.EmitGlobal(D: GlobalDecl(D, Dtor_Complete));
1694
1695	// The destructor in a virtual table is always a 'deleting'
1696	// destructor, which calls the complete destructor and then uses the
1697	// appropriate operator delete.
1698	if (D->isVirtual())
1699	CGM.EmitGlobal(D: GlobalDecl(D, Dtor_Deleting));
1700	}
1701
1702	void ItaniumCXXABI::addImplicitStructorParams(CodeGenFunction &CGF,
1703	QualType &ResTy,
1704	FunctionArgList &Params) {
1705	const CXXMethodDecl *MD = cast<CXXMethodDecl>(Val: CGF.CurGD.getDecl());
1706	assert(isa<CXXConstructorDecl>(MD) || isa<CXXDestructorDecl>(MD));
1707
1708	// Check if we need a VTT parameter as well.
1709	if (NeedsVTTParameter(GD: CGF.CurGD)) {
1710	ASTContext &Context = getContext();
1711
1712	// FIXME: avoid the fake decl
1713	LangAS AS = CGM.GetGlobalVarAddressSpace(D: nullptr);
1714	QualType Q = Context.getAddrSpaceQualType(T: Context.VoidPtrTy, AddressSpace: AS);
1715	QualType T = Context.getPointerType(T: Q);
1716	auto *VTTDecl = ImplicitParamDecl::Create(
1717	Context, /*DC=*/nullptr, MD->getLocation(), &Context.Idents.get(Name: "vtt"),
1718	T, ImplicitParamKind::CXXVTT);
1719	Params.insert(Params.begin() + 1, VTTDecl);
1720	getStructorImplicitParamDecl(CGF) = VTTDecl;
1721	}
1722	}
1723
1724	void ItaniumCXXABI::EmitInstanceFunctionProlog(CodeGenFunction &CGF) {
1725	// Naked functions have no prolog.
1726	if (CGF.CurFuncDecl && CGF.CurFuncDecl->hasAttr<NakedAttr>())
1727	return;
1728
1729	/// Initialize the 'this' slot. In the Itanium C++ ABI, no prologue
1730	/// adjustments are required, because they are all handled by thunks.
1731	setCXXABIThisValue(CGF, ThisPtr: loadIncomingCXXThis(CGF));
1732
1733	/// Initialize the 'vtt' slot if needed.
1734	if (getStructorImplicitParamDecl(CGF)) {
1735	getStructorImplicitParamValue(CGF) = CGF.Builder.CreateLoad(
1736	Addr: CGF.GetAddrOfLocalVar(getStructorImplicitParamDecl(CGF)), Name: "vtt");
1737	}
1738
1739	/// If this is a function that the ABI specifies returns 'this', initialize
1740	/// the return slot to 'this' at the start of the function.
1741	///
1742	/// Unlike the setting of return types, this is done within the ABI
1743	/// implementation instead of by clients of CGCXXABI because:
1744	/// 1) getThisValue is currently protected
1745	/// 2) in theory, an ABI could implement 'this' returns some other way;
1746	/// HasThisReturn only specifies a contract, not the implementation
1747	if (HasThisReturn(GD: CGF.CurGD))
1748	CGF.Builder.CreateStore(Val: getThisValue(CGF), Addr: CGF.ReturnValue);
1749	}
1750
1751	CGCXXABI::AddedStructorArgs ItaniumCXXABI::getImplicitConstructorArgs(
1752	CodeGenFunction &CGF, const CXXConstructorDecl *D, CXXCtorType Type,
1753	bool ForVirtualBase, bool Delegating) {
1754	if (!NeedsVTTParameter(GD: GlobalDecl(D, Type)))
1755	return AddedStructorArgs{};
1756
1757	// Insert the implicit 'vtt' argument as the second argument. Make sure to
1758	// correctly reflect its address space, which can differ from generic on
1759	// some targets.
1760	llvm::Value *VTT =
1761	CGF.GetVTTParameter(GD: GlobalDecl(D, Type), ForVirtualBase, Delegating);
1762	LangAS AS = CGM.GetGlobalVarAddressSpace(D: nullptr);
1763	QualType Q = getContext().getAddrSpaceQualType(T: getContext().VoidPtrTy, AddressSpace: AS);
1764	QualType VTTTy = getContext().getPointerType(T: Q);
1765	return AddedStructorArgs::prefix({{VTT, VTTTy}});
1766	}
1767
1768	llvm::Value *ItaniumCXXABI::getCXXDestructorImplicitParam(
1769	CodeGenFunction &CGF, const CXXDestructorDecl *DD, CXXDtorType Type,
1770	bool ForVirtualBase, bool Delegating) {
1771	GlobalDecl GD(DD, Type);
1772	return CGF.GetVTTParameter(GD, ForVirtualBase, Delegating);
1773	}
1774
1775	void ItaniumCXXABI::EmitDestructorCall(CodeGenFunction &CGF,
1776	const CXXDestructorDecl *DD,
1777	CXXDtorType Type, bool ForVirtualBase,
1778	bool Delegating, Address This,
1779	QualType ThisTy) {
1780	GlobalDecl GD(DD, Type);
1781	llvm::Value *VTT =
1782	getCXXDestructorImplicitParam(CGF, DD, Type, ForVirtualBase, Delegating);
1783	QualType VTTTy = getContext().getPointerType(getContext().VoidPtrTy);
1784
1785	CGCallee Callee;
1786	if (getContext().getLangOpts().AppleKext &&
1787	Type != Dtor_Base && DD->isVirtual())
1788	Callee = CGF.BuildAppleKextVirtualDestructorCall(DD, Type, RD: DD->getParent());
1789	else
1790	Callee = CGCallee::forDirect(functionPtr: CGM.getAddrOfCXXStructor(GD), abstractInfo: GD);
1791
1792	CGF.EmitCXXDestructorCall(Dtor: GD, Callee, This: CGF.getAsNaturalPointerTo(Addr: This, PointeeType: ThisTy),
1793	ThisTy, ImplicitParam: VTT, ImplicitParamTy: VTTTy, E: nullptr);
1794	}
1795
1796	void ItaniumCXXABI::emitVTableDefinitions(CodeGenVTables &CGVT,
1797	const CXXRecordDecl *RD) {
1798	llvm::GlobalVariable *VTable = getAddrOfVTable(RD, VPtrOffset: CharUnits());
1799	if (VTable->hasInitializer())
1800	return;
1801
1802	ItaniumVTableContext &VTContext = CGM.getItaniumVTableContext();
1803	const VTableLayout &VTLayout = VTContext.getVTableLayout(RD);
1804	llvm::GlobalVariable::LinkageTypes Linkage = CGM.getVTableLinkage(RD);
1805	llvm::Constant *RTTI =
1806	CGM.GetAddrOfRTTIDescriptor(Ty: CGM.getContext().getTagDeclType(RD));
1807
1808	// Create and set the initializer.
1809	ConstantInitBuilder builder(CGM);
1810	auto components = builder.beginStruct();
1811	CGVT.createVTableInitializer(builder&: components, layout: VTLayout, rtti: RTTI,
1812	vtableHasLocalLinkage: llvm::GlobalValue::isLocalLinkage(Linkage));
1813	components.finishAndSetAsInitializer(global: VTable);
1814
1815	// Set the correct linkage.
1816	VTable->setLinkage(Linkage);
1817
1818	if (CGM.supportsCOMDAT() && VTable->isWeakForLinker())
1819	VTable->setComdat(CGM.getModule().getOrInsertComdat(Name: VTable->getName()));
1820
1821	// Set the right visibility.
1822	CGM.setGVProperties(VTable, RD);
1823
1824	// If this is the magic class __cxxabiv1::__fundamental_type_info,
1825	// we will emit the typeinfo for the fundamental types. This is the
1826	// same behaviour as GCC.
1827	const DeclContext *DC = RD->getDeclContext();
1828	if (RD->getIdentifier() &&
1829	RD->getIdentifier()->isStr("__fundamental_type_info") &&
1830	isa<NamespaceDecl>(Val: DC) && cast<NamespaceDecl>(Val: DC)->getIdentifier() &&
1831	cast<NamespaceDecl>(Val: DC)->getIdentifier()->isStr("__cxxabiv1") &&
1832	DC->getParent()->isTranslationUnit())
1833	EmitFundamentalRTTIDescriptors(RD);
1834
1835	// Always emit type metadata on non-available_externally definitions, and on
1836	// available_externally definitions if we are performing whole program
1837	// devirtualization. For WPD we need the type metadata on all vtable
1838	// definitions to ensure we associate derived classes with base classes
1839	// defined in headers but with a strong definition only in a shared library.
1840	if (!VTable->isDeclarationForLinker() ||
1841	CGM.getCodeGenOpts().WholeProgramVTables) {
1842	CGM.EmitVTableTypeMetadata(RD, VTable, VTLayout);
1843	// For available_externally definitions, add the vtable to
1844	// @llvm.compiler.used so that it isn't deleted before whole program
1845	// analysis.
1846	if (VTable->isDeclarationForLinker()) {
1847	assert(CGM.getCodeGenOpts().WholeProgramVTables);
1848	CGM.addCompilerUsedGlobal(GV: VTable);
1849	}
1850	}
1851
1852	if (VTContext.isRelativeLayout()) {
1853	CGVT.RemoveHwasanMetadata(GV: VTable);
1854	if (!VTable->isDSOLocal())
1855	CGVT.GenerateRelativeVTableAlias(VTable, AliasNameRef: VTable->getName());
1856	}
1857	}
1858
1859	bool ItaniumCXXABI::isVirtualOffsetNeededForVTableField(
1860	CodeGenFunction &CGF, CodeGenFunction::VPtr Vptr) {
1861	if (Vptr.NearestVBase == nullptr)
1862	return false;
1863	return NeedsVTTParameter(GD: CGF.CurGD);
1864	}
1865
1866	llvm::Value *ItaniumCXXABI::getVTableAddressPointInStructor(
1867	CodeGenFunction &CGF, const CXXRecordDecl *VTableClass, BaseSubobject Base,
1868	const CXXRecordDecl *NearestVBase) {
1869
1870	if ((Base.getBase()->getNumVBases() || NearestVBase != nullptr) &&
1871	NeedsVTTParameter(GD: CGF.CurGD)) {
1872	return getVTableAddressPointInStructorWithVTT(CGF, VTableClass, Base,
1873	NearestVBase);
1874	}
1875	return getVTableAddressPoint(Base, VTableClass);
1876	}
1877
1878	llvm::Constant *
1879	ItaniumCXXABI::getVTableAddressPoint(BaseSubobject Base,
1880	const CXXRecordDecl *VTableClass) {
1881	llvm::GlobalValue *VTable = getAddrOfVTable(RD: VTableClass, VPtrOffset: CharUnits());
1882
1883	// Find the appropriate vtable within the vtable group, and the address point
1884	// within that vtable.
1885	const VTableLayout &Layout =
1886	CGM.getItaniumVTableContext().getVTableLayout(RD: VTableClass);
1887	VTableLayout::AddressPointLocation AddressPoint =
1888	Layout.getAddressPoint(Base);
1889	llvm::Value *Indices[] = {
1890	llvm::ConstantInt::get(Ty: CGM.Int32Ty, V: 0),
1891	llvm::ConstantInt::get(Ty: CGM.Int32Ty, V: AddressPoint.VTableIndex),
1892	llvm::ConstantInt::get(Ty: CGM.Int32Ty, V: AddressPoint.AddressPointIndex),
1893	};
1894
1895	// Add inrange attribute to indicate that only the VTableIndex can be
1896	// accessed.
1897	unsigned ComponentSize =
1898	CGM.getDataLayout().getTypeAllocSize(Ty: CGM.getVTableComponentType());
1899	unsigned VTableSize =
1900	ComponentSize * Layout.getVTableSize(i: AddressPoint.VTableIndex);
1901	unsigned Offset = ComponentSize * AddressPoint.AddressPointIndex;
1902	llvm::ConstantRange InRange(llvm::APInt(32, -Offset, true),
1903	llvm::APInt(32, VTableSize - Offset, true));
1904	return llvm::ConstantExpr::getGetElementPtr(
1905	Ty: VTable->getValueType(), C: VTable, IdxList: Indices, /*InBounds=*/true, InRange);
1906	}
1907
1908	// Check whether all the non-inline virtual methods for the class have the
1909	// specified attribute.
1910	template <typename T>
1911	static bool CXXRecordAllNonInlineVirtualsHaveAttr(const CXXRecordDecl *RD) {
1912	bool FoundNonInlineVirtualMethodWithAttr = false;
1913	for (const auto *D : RD->noload_decls()) {
1914	if (const auto *FD = dyn_cast<FunctionDecl>(D)) {
1915	if (!FD->isVirtualAsWritten() || FD->isInlineSpecified() ||
1916	FD->doesThisDeclarationHaveABody())
1917	continue;
1918	if (!D->hasAttr<T>())
1919	return false;
1920	FoundNonInlineVirtualMethodWithAttr = true;
1921	}
1922	}
1923
1924	// We didn't find any non-inline virtual methods missing the attribute. We
1925	// will return true when we found at least one non-inline virtual with the
1926	// attribute. (This lets our caller know that the attribute needs to be
1927	// propagated up to the vtable.)
1928	return FoundNonInlineVirtualMethodWithAttr;
1929	}
1930
1931	llvm::Value *ItaniumCXXABI::getVTableAddressPointInStructorWithVTT(
1932	CodeGenFunction &CGF, const CXXRecordDecl *VTableClass, BaseSubobject Base,
1933	const CXXRecordDecl *NearestVBase) {
1934	assert((Base.getBase()->getNumVBases() || NearestVBase != nullptr) &&
1935	NeedsVTTParameter(CGF.CurGD) && "This class doesn't have VTT");
1936
1937	// Get the secondary vpointer index.
1938	uint64_t VirtualPointerIndex =
1939	CGM.getVTables().getSecondaryVirtualPointerIndex(RD: VTableClass, Base);
1940
1941	/// Load the VTT.
1942	llvm::Value *VTT = CGF.LoadCXXVTT();
1943	if (VirtualPointerIndex)
1944	VTT = CGF.Builder.CreateConstInBoundsGEP1_64(Ty: CGF.GlobalsVoidPtrTy, Ptr: VTT,
1945	Idx0: VirtualPointerIndex);
1946
1947	// And load the address point from the VTT.
1948	return CGF.Builder.CreateAlignedLoad(CGF.GlobalsVoidPtrTy, VTT,
1949	CGF.getPointerAlign());
1950	}
1951
1952	llvm::GlobalVariable *ItaniumCXXABI::getAddrOfVTable(const CXXRecordDecl *RD,
1953	CharUnits VPtrOffset) {
1954	assert(VPtrOffset.isZero() && "Itanium ABI only supports zero vptr offsets");
1955
1956	llvm::GlobalVariable *&VTable = VTables[RD];
1957	if (VTable)
1958	return VTable;
1959
1960	// Queue up this vtable for possible deferred emission.
1961	CGM.addDeferredVTable(RD);
1962
1963	SmallString<256> Name;
1964	llvm::raw_svector_ostream Out(Name);
1965	getMangleContext().mangleCXXVTable(RD, Out);
1966
1967	const VTableLayout &VTLayout =
1968	CGM.getItaniumVTableContext().getVTableLayout(RD);
1969	llvm::Type *VTableType = CGM.getVTables().getVTableType(layout: VTLayout);
1970
1971	// Use pointer to global alignment for the vtable. Otherwise we would align
1972	// them based on the size of the initializer which doesn't make sense as only
1973	// single values are read.
1974	LangAS AS = CGM.GetGlobalVarAddressSpace(D: nullptr);
1975	unsigned PAlign = CGM.getItaniumVTableContext().isRelativeLayout()
1976	? 32
1977	: CGM.getTarget().getPointerAlign(AddrSpace: AS);
1978
1979	VTable = CGM.CreateOrReplaceCXXRuntimeVariable(
1980	Name, Ty: VTableType, Linkage: llvm::GlobalValue::ExternalLinkage,
1981	Alignment: getContext().toCharUnitsFromBits(BitSize: PAlign).getAsAlign());
1982	VTable->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
1983
1984	// In MS C++ if you have a class with virtual functions in which you are using
1985	// selective member import/export, then all virtual functions must be exported
1986	// unless they are inline, otherwise a link error will result. To match this
1987	// behavior, for such classes, we dllimport the vtable if it is defined
1988	// externally and all the non-inline virtual methods are marked dllimport, and
1989	// we dllexport the vtable if it is defined in this TU and all the non-inline
1990	// virtual methods are marked dllexport.
1991	if (CGM.getTarget().hasPS4DLLImportExport()) {
1992	if ((!RD->hasAttr<DLLImportAttr>()) && (!RD->hasAttr<DLLExportAttr>())) {
1993	if (CGM.getVTables().isVTableExternal(RD)) {
1994	if (CXXRecordAllNonInlineVirtualsHaveAttr<DLLImportAttr>(RD))
1995	VTable->setDLLStorageClass(llvm::GlobalValue::DLLImportStorageClass);
1996	} else {
1997	if (CXXRecordAllNonInlineVirtualsHaveAttr<DLLExportAttr>(RD))
1998	VTable->setDLLStorageClass(llvm::GlobalValue::DLLExportStorageClass);
1999	}
2000	}
2001	}
2002	CGM.setGVProperties(VTable, RD);
2003
2004	return VTable;
2005	}
2006
2007	CGCallee ItaniumCXXABI::getVirtualFunctionPointer(CodeGenFunction &CGF,
2008	GlobalDecl GD,
2009	Address This,
2010	llvm::Type *Ty,
2011	SourceLocation Loc) {
2012	llvm::Type *PtrTy = CGM.GlobalsInt8PtrTy;
2013	auto *MethodDecl = cast<CXXMethodDecl>(Val: GD.getDecl());
2014	llvm::Value *VTable = CGF.GetVTablePtr(This, VTableTy: PtrTy, VTableClass: MethodDecl->getParent());
2015
2016	uint64_t VTableIndex = CGM.getItaniumVTableContext().getMethodVTableIndex(GD);
2017	llvm::Value *VFunc;
2018	if (CGF.ShouldEmitVTableTypeCheckedLoad(RD: MethodDecl->getParent())) {
2019	VFunc = CGF.EmitVTableTypeCheckedLoad(
2020	RD: MethodDecl->getParent(), VTable, VTableTy: PtrTy,
2021	VTableByteOffset: VTableIndex *
2022	CGM.getContext().getTargetInfo().getPointerWidth(AddrSpace: LangAS::Default) /
2023	8);
2024	} else {
2025	CGF.EmitTypeMetadataCodeForVCall(RD: MethodDecl->getParent(), VTable, Loc);
2026
2027	llvm::Value *VFuncLoad;
2028	if (CGM.getItaniumVTableContext().isRelativeLayout()) {
2029	VFuncLoad = CGF.Builder.CreateCall(
2030	CGM.getIntrinsic(llvm::Intrinsic::load_relative, {CGM.Int32Ty}),
2031	{VTable, llvm::ConstantInt::get(CGM.Int32Ty, 4 * VTableIndex)});
2032	} else {
2033	llvm::Value *VTableSlotPtr = CGF.Builder.CreateConstInBoundsGEP1_64(
2034	Ty: PtrTy, Ptr: VTable, Idx0: VTableIndex, Name: "vfn");
2035	VFuncLoad = CGF.Builder.CreateAlignedLoad(PtrTy, VTableSlotPtr,
2036	CGF.getPointerAlign());
2037	}
2038
2039	// Add !invariant.load md to virtual function load to indicate that
2040	// function didn't change inside vtable.
2041	// It's safe to add it without -fstrict-vtable-pointers, but it would not
2042	// help in devirtualization because it will only matter if we will have 2
2043	// the same virtual function loads from the same vtable load, which won't
2044	// happen without enabled devirtualization with -fstrict-vtable-pointers.
2045	if (CGM.getCodeGenOpts().OptimizationLevel > 0 &&
2046	CGM.getCodeGenOpts().StrictVTablePointers) {
2047	if (auto *VFuncLoadInstr = dyn_cast<llvm::Instruction>(Val: VFuncLoad)) {
2048	VFuncLoadInstr->setMetadata(
2049	KindID: llvm::LLVMContext::MD_invariant_load,
2050	Node: llvm::MDNode::get(Context&: CGM.getLLVMContext(),
2051	MDs: llvm::ArrayRef<llvm::Metadata *>()));
2052	}
2053	}
2054	VFunc = VFuncLoad;
2055	}
2056
2057	CGCallee Callee(GD, VFunc);
2058	return Callee;
2059	}
2060
2061	llvm::Value *ItaniumCXXABI::EmitVirtualDestructorCall(
2062	CodeGenFunction &CGF, const CXXDestructorDecl *Dtor, CXXDtorType DtorType,
2063	Address This, DeleteOrMemberCallExpr E) {
2064	auto *CE = E.dyn_cast<const CXXMemberCallExpr *>();
2065	auto *D = E.dyn_cast<const CXXDeleteExpr *>();
2066	assert((CE != nullptr) ^ (D != nullptr));
2067	assert(CE == nullptr || CE->arg_begin() == CE->arg_end());
2068	assert(DtorType == Dtor_Deleting || DtorType == Dtor_Complete);
2069
2070	GlobalDecl GD(Dtor, DtorType);
2071	const CGFunctionInfo *FInfo =
2072	&CGM.getTypes().arrangeCXXStructorDeclaration(GD);
2073	llvm::FunctionType *Ty = CGF.CGM.getTypes().GetFunctionType(Info: *FInfo);
2074	CGCallee Callee = CGCallee::forVirtual(CE, GD, This, Ty);
2075
2076	QualType ThisTy;
2077	if (CE) {
2078	ThisTy = CE->getObjectType();
2079	} else {
2080	ThisTy = D->getDestroyedType();
2081	}
2082
2083	CGF.EmitCXXDestructorCall(Dtor: GD, Callee, This: This.emitRawPointer(CGF), ThisTy,
2084	ImplicitParam: nullptr, ImplicitParamTy: QualType(), E: nullptr);
2085	return nullptr;
2086	}
2087
2088	void ItaniumCXXABI::emitVirtualInheritanceTables(const CXXRecordDecl *RD) {
2089	CodeGenVTables &VTables = CGM.getVTables();
2090	llvm::GlobalVariable *VTT = VTables.GetAddrOfVTT(RD);
2091	VTables.EmitVTTDefinition(VTT, Linkage: CGM.getVTableLinkage(RD), RD);
2092	}
2093
2094	bool ItaniumCXXABI::canSpeculativelyEmitVTableAsBaseClass(
2095	const CXXRecordDecl *RD) const {
2096	// We don't emit available_externally vtables if we are in -fapple-kext mode
2097	// because kext mode does not permit devirtualization.
2098	if (CGM.getLangOpts().AppleKext)
2099	return false;
2100
2101	// If the vtable is hidden then it is not safe to emit an available_externally
2102	// copy of vtable.
2103	if (isVTableHidden(RD))
2104	return false;
2105
2106	if (CGM.getCodeGenOpts().ForceEmitVTables)
2107	return true;
2108
2109	// If we don't have any not emitted inline virtual function then we are safe
2110	// to emit an available_externally copy of vtable.
2111	// FIXME we can still emit a copy of the vtable if we
2112	// can emit definition of the inline functions.
2113	if (hasAnyUnusedVirtualInlineFunction(RD))
2114	return false;
2115
2116	// For a class with virtual bases, we must also be able to speculatively
2117	// emit the VTT, because CodeGen doesn't have separate notions of "can emit
2118	// the vtable" and "can emit the VTT". For a base subobject, this means we
2119	// need to be able to emit non-virtual base vtables.
2120	if (RD->getNumVBases()) {
2121	for (const auto &B : RD->bases()) {
2122	auto *BRD = B.getType()->getAsCXXRecordDecl();
2123	assert(BRD && "no class for base specifier");
2124	if (B.isVirtual() || !BRD->isDynamicClass())
2125	continue;
2126	if (!canSpeculativelyEmitVTableAsBaseClass(RD: BRD))
2127	return false;
2128	}
2129	}
2130
2131	return true;
2132	}
2133
2134	bool ItaniumCXXABI::canSpeculativelyEmitVTable(const CXXRecordDecl *RD) const {
2135	if (!canSpeculativelyEmitVTableAsBaseClass(RD))
2136	return false;
2137
2138	// For a complete-object vtable (or more specifically, for the VTT), we need
2139	// to be able to speculatively emit the vtables of all dynamic virtual bases.
2140	for (const auto &B : RD->vbases()) {
2141	auto *BRD = B.getType()->getAsCXXRecordDecl();
2142	assert(BRD && "no class for base specifier");
2143	if (!BRD->isDynamicClass())
2144	continue;
2145	if (!canSpeculativelyEmitVTableAsBaseClass(RD: BRD))
2146	return false;
2147	}
2148
2149	return true;
2150	}
2151	static llvm::Value *performTypeAdjustment(CodeGenFunction &CGF,
2152	Address InitialPtr,
2153	int64_t NonVirtualAdjustment,
2154	int64_t VirtualAdjustment,
2155	bool IsReturnAdjustment) {
2156	if (!NonVirtualAdjustment && !VirtualAdjustment)
2157	return InitialPtr.emitRawPointer(CGF);
2158
2159	Address V = InitialPtr.withElementType(ElemTy: CGF.Int8Ty);
2160
2161	// In a base-to-derived cast, the non-virtual adjustment is applied first.
2162	if (NonVirtualAdjustment && !IsReturnAdjustment) {
2163	V = CGF.Builder.CreateConstInBoundsByteGEP(Addr: V,
2164	Offset: CharUnits::fromQuantity(Quantity: NonVirtualAdjustment));
2165	}
2166
2167	// Perform the virtual adjustment if we have one.
2168	llvm::Value *ResultPtr;
2169	if (VirtualAdjustment) {
2170	Address VTablePtrPtr = V.withElementType(ElemTy: CGF.Int8PtrTy);
2171	llvm::Value *VTablePtr = CGF.Builder.CreateLoad(Addr: VTablePtrPtr);
2172
2173	llvm::Value *Offset;
2174	llvm::Value *OffsetPtr = CGF.Builder.CreateConstInBoundsGEP1_64(
2175	Ty: CGF.Int8Ty, Ptr: VTablePtr, Idx0: VirtualAdjustment);
2176	if (CGF.CGM.getItaniumVTableContext().isRelativeLayout()) {
2177	// Load the adjustment offset from the vtable as a 32-bit int.
2178	Offset =
2179	CGF.Builder.CreateAlignedLoad(Ty: CGF.Int32Ty, Addr: OffsetPtr,
2180	Align: CharUnits::fromQuantity(Quantity: 4));
2181	} else {
2182	llvm::Type *PtrDiffTy =
2183	CGF.ConvertType(T: CGF.getContext().getPointerDiffType());
2184
2185	// Load the adjustment offset from the vtable.
2186	Offset = CGF.Builder.CreateAlignedLoad(PtrDiffTy, OffsetPtr,
2187	CGF.getPointerAlign());
2188	}
2189	// Adjust our pointer.
2190	ResultPtr = CGF.Builder.CreateInBoundsGEP(Ty: V.getElementType(),
2191	Ptr: V.emitRawPointer(CGF), IdxList: Offset);
2192	} else {
2193	ResultPtr = V.emitRawPointer(CGF);
2194	}
2195
2196	// In a derived-to-base conversion, the non-virtual adjustment is
2197	// applied second.
2198	if (NonVirtualAdjustment && IsReturnAdjustment) {
2199	ResultPtr = CGF.Builder.CreateConstInBoundsGEP1_64(Ty: CGF.Int8Ty, Ptr: ResultPtr,
2200	Idx0: NonVirtualAdjustment);
2201	}
2202
2203	return ResultPtr;
2204	}
2205
2206	llvm::Value *ItaniumCXXABI::performThisAdjustment(CodeGenFunction &CGF,
2207	Address This,
2208	const ThisAdjustment &TA) {
2209	return performTypeAdjustment(CGF, InitialPtr: This, NonVirtualAdjustment: TA.NonVirtual,
2210	VirtualAdjustment: TA.Virtual.Itanium.VCallOffsetOffset,
2211	/*IsReturnAdjustment=*/false);
2212	}
2213
2214	llvm::Value *
2215	ItaniumCXXABI::performReturnAdjustment(CodeGenFunction &CGF, Address Ret,
2216	const ReturnAdjustment &RA) {
2217	return performTypeAdjustment(CGF, InitialPtr: Ret, NonVirtualAdjustment: RA.NonVirtual,
2218	VirtualAdjustment: RA.Virtual.Itanium.VBaseOffsetOffset,
2219	/*IsReturnAdjustment=*/true);
2220	}
2221
2222	void ARMCXXABI::EmitReturnFromThunk(CodeGenFunction &CGF,
2223	RValue RV, QualType ResultType) {
2224	if (!isa<CXXDestructorDecl>(Val: CGF.CurGD.getDecl()))
2225	return ItaniumCXXABI::EmitReturnFromThunk(CGF, RV, ResultType);
2226
2227	// Destructor thunks in the ARM ABI have indeterminate results.
2228	llvm::Type *T = CGF.ReturnValue.getElementType();
2229	RValue Undef = RValue::get(V: llvm::UndefValue::get(T));
2230	return ItaniumCXXABI::EmitReturnFromThunk(CGF, RV: Undef, ResultType);
2231	}
2232
2233	/************************** Array allocation cookies **************************/
2234
2235	CharUnits ItaniumCXXABI::getArrayCookieSizeImpl(QualType elementType) {
2236	// The array cookie is a size_t; pad that up to the element alignment.
2237	// The cookie is actually right-justified in that space.
2238	return std::max(a: CharUnits::fromQuantity(Quantity: CGM.SizeSizeInBytes),
2239	b: CGM.getContext().getPreferredTypeAlignInChars(T: elementType));
2240	}
2241
2242	Address ItaniumCXXABI::InitializeArrayCookie(CodeGenFunction &CGF,
2243	Address NewPtr,
2244	llvm::Value *NumElements,
2245	const CXXNewExpr *expr,
2246	QualType ElementType) {
2247	assert(requiresArrayCookie(expr));
2248
2249	unsigned AS = NewPtr.getAddressSpace();
2250
2251	ASTContext &Ctx = getContext();
2252	CharUnits SizeSize = CGF.getSizeSize();
2253
2254	// The size of the cookie.
2255	CharUnits CookieSize =
2256	std::max(a: SizeSize, b: Ctx.getPreferredTypeAlignInChars(T: ElementType));
2257	assert(CookieSize == getArrayCookieSizeImpl(ElementType));
2258
2259	// Compute an offset to the cookie.
2260	Address CookiePtr = NewPtr;
2261	CharUnits CookieOffset = CookieSize - SizeSize;
2262	if (!CookieOffset.isZero())
2263	CookiePtr = CGF.Builder.CreateConstInBoundsByteGEP(Addr: CookiePtr, Offset: CookieOffset);
2264
2265	// Write the number of elements into the appropriate slot.
2266	Address NumElementsPtr = CookiePtr.withElementType(ElemTy: CGF.SizeTy);
2267	llvm::Instruction *SI = CGF.Builder.CreateStore(Val: NumElements, Addr: NumElementsPtr);
2268
2269	// Handle the array cookie specially in ASan.
2270	if (CGM.getLangOpts().Sanitize.has(K: SanitizerKind::Address) && AS == 0 &&
2271	(expr->getOperatorNew()->isReplaceableGlobalAllocationFunction() ||
2272	CGM.getCodeGenOpts().SanitizeAddressPoisonCustomArrayCookie)) {
2273	// The store to the CookiePtr does not need to be instrumented.
2274	SI->setNoSanitizeMetadata();
2275	llvm::FunctionType *FTy =
2276	llvm::FunctionType::get(Result: CGM.VoidTy, Params: NumElementsPtr.getType(), isVarArg: false);
2277	llvm::FunctionCallee F =
2278	CGM.CreateRuntimeFunction(Ty: FTy, Name: "__asan_poison_cxx_array_cookie");
2279	CGF.Builder.CreateCall(Callee: F, Args: NumElementsPtr.emitRawPointer(CGF));
2280	}
2281
2282	// Finally, compute a pointer to the actual data buffer by skipping
2283	// over the cookie completely.
2284	return CGF.Builder.CreateConstInBoundsByteGEP(Addr: NewPtr, Offset: CookieSize);
2285	}
2286
2287	llvm::Value *ItaniumCXXABI::readArrayCookieImpl(CodeGenFunction &CGF,
2288	Address allocPtr,
2289	CharUnits cookieSize) {
2290	// The element size is right-justified in the cookie.
2291	Address numElementsPtr = allocPtr;
2292	CharUnits numElementsOffset = cookieSize - CGF.getSizeSize();
2293	if (!numElementsOffset.isZero())
2294	numElementsPtr =
2295	CGF.Builder.CreateConstInBoundsByteGEP(Addr: numElementsPtr, Offset: numElementsOffset);
2296
2297	unsigned AS = allocPtr.getAddressSpace();
2298	numElementsPtr = numElementsPtr.withElementType(ElemTy: CGF.SizeTy);
2299	if (!CGM.getLangOpts().Sanitize.has(K: SanitizerKind::Address) || AS != 0)
2300	return CGF.Builder.CreateLoad(Addr: numElementsPtr);
2301	// In asan mode emit a function call instead of a regular load and let the
2302	// run-time deal with it: if the shadow is properly poisoned return the
2303	// cookie, otherwise return 0 to avoid an infinite loop calling DTORs.
2304	// We can't simply ignore this load using nosanitize metadata because
2305	// the metadata may be lost.
2306	llvm::FunctionType *FTy =
2307	llvm::FunctionType::get(Result: CGF.SizeTy, Params: CGF.UnqualPtrTy, isVarArg: false);
2308	llvm::FunctionCallee F =
2309	CGM.CreateRuntimeFunction(Ty: FTy, Name: "__asan_load_cxx_array_cookie");
2310	return CGF.Builder.CreateCall(Callee: F, Args: numElementsPtr.emitRawPointer(CGF));
2311	}
2312
2313	CharUnits ARMCXXABI::getArrayCookieSizeImpl(QualType elementType) {
2314	// ARM says that the cookie is always:
2315	// struct array_cookie {
2316	// std::size_t element_size; // element_size != 0
2317	// std::size_t element_count;
2318	// };
2319	// But the base ABI doesn't give anything an alignment greater than
2320	// 8, so we can dismiss this as typical ABI-author blindness to
2321	// actual language complexity and round up to the element alignment.
2322	return std::max(a: CharUnits::fromQuantity(Quantity: 2 * CGM.SizeSizeInBytes),
2323	b: CGM.getContext().getTypeAlignInChars(T: elementType));
2324	}
2325
2326	Address ARMCXXABI::InitializeArrayCookie(CodeGenFunction &CGF,
2327	Address newPtr,
2328	llvm::Value *numElements,
2329	const CXXNewExpr *expr,
2330	QualType elementType) {
2331	assert(requiresArrayCookie(expr));
2332
2333	// The cookie is always at the start of the buffer.
2334	Address cookie = newPtr;
2335
2336	// The first element is the element size.
2337	cookie = cookie.withElementType(ElemTy: CGF.SizeTy);
2338	llvm::Value *elementSize = llvm::ConstantInt::get(Ty: CGF.SizeTy,
2339	V: getContext().getTypeSizeInChars(T: elementType).getQuantity());
2340	CGF.Builder.CreateStore(Val: elementSize, Addr: cookie);
2341
2342	// The second element is the element count.
2343	cookie = CGF.Builder.CreateConstInBoundsGEP(Addr: cookie, Index: 1);
2344	CGF.Builder.CreateStore(Val: numElements, Addr: cookie);
2345
2346	// Finally, compute a pointer to the actual data buffer by skipping
2347	// over the cookie completely.
2348	CharUnits cookieSize = ARMCXXABI::getArrayCookieSizeImpl(elementType);
2349	return CGF.Builder.CreateConstInBoundsByteGEP(Addr: newPtr, Offset: cookieSize);
2350	}
2351
2352	llvm::Value *ARMCXXABI::readArrayCookieImpl(CodeGenFunction &CGF,
2353	Address allocPtr,
2354	CharUnits cookieSize) {
2355	// The number of elements is at offset sizeof(size_t) relative to
2356	// the allocated pointer.
2357	Address numElementsPtr
2358	= CGF.Builder.CreateConstInBoundsByteGEP(Addr: allocPtr, Offset: CGF.getSizeSize());
2359
2360	numElementsPtr = numElementsPtr.withElementType(ElemTy: CGF.SizeTy);
2361	return CGF.Builder.CreateLoad(Addr: numElementsPtr);
2362	}
2363
2364	/*********************** Static local initialization **************************/
2365
2366	static llvm::FunctionCallee getGuardAcquireFn(CodeGenModule &CGM,
2367	llvm::PointerType *GuardPtrTy) {
2368	// int __cxa_guard_acquire(__guard *guard_object);
2369	llvm::FunctionType *FTy =
2370	llvm::FunctionType::get(CGM.getTypes().ConvertType(T: CGM.getContext().IntTy),
2371	GuardPtrTy, /*isVarArg=*/false);
2372	return CGM.CreateRuntimeFunction(
2373	FTy, "__cxa_guard_acquire",
2374	llvm::AttributeList::get(CGM.getLLVMContext(),
2375	llvm::AttributeList::FunctionIndex,
2376	llvm::Attribute::NoUnwind));
2377	}
2378
2379	static llvm::FunctionCallee getGuardReleaseFn(CodeGenModule &CGM,
2380	llvm::PointerType *GuardPtrTy) {
2381	// void __cxa_guard_release(__guard *guard_object);
2382	llvm::FunctionType *FTy =
2383	llvm::FunctionType::get(Result: CGM.VoidTy, Params: GuardPtrTy, /*isVarArg=*/false);
2384	return CGM.CreateRuntimeFunction(
2385	FTy, "__cxa_guard_release",
2386	llvm::AttributeList::get(CGM.getLLVMContext(),
2387	llvm::AttributeList::FunctionIndex,
2388	llvm::Attribute::NoUnwind));
2389	}
2390
2391	static llvm::FunctionCallee getGuardAbortFn(CodeGenModule &CGM,
2392	llvm::PointerType *GuardPtrTy) {
2393	// void __cxa_guard_abort(__guard *guard_object);
2394	llvm::FunctionType *FTy =
2395	llvm::FunctionType::get(Result: CGM.VoidTy, Params: GuardPtrTy, /*isVarArg=*/false);
2396	return CGM.CreateRuntimeFunction(
2397	FTy, "__cxa_guard_abort",
2398	llvm::AttributeList::get(CGM.getLLVMContext(),
2399	llvm::AttributeList::FunctionIndex,
2400	llvm::Attribute::NoUnwind));
2401	}
2402
2403	namespace {
2404	struct CallGuardAbort final : EHScopeStack::Cleanup {
2405	llvm::GlobalVariable *Guard;
2406	CallGuardAbort(llvm::GlobalVariable *Guard) : Guard(Guard) {}
2407
2408	void Emit(CodeGenFunction &CGF, Flags flags) override {
2409	CGF.EmitNounwindRuntimeCall(callee: getGuardAbortFn(CGM&: CGF.CGM, GuardPtrTy: Guard->getType()),
2410	args: Guard);
2411	}
2412	};
2413	}
2414
2415	/// The ARM code here follows the Itanium code closely enough that we
2416	/// just special-case it at particular places.
2417	void ItaniumCXXABI::EmitGuardedInit(CodeGenFunction &CGF,
2418	const VarDecl &D,
2419	llvm::GlobalVariable *var,
2420	bool shouldPerformInit) {
2421	CGBuilderTy &Builder = CGF.Builder;
2422
2423	// Inline variables that weren't instantiated from variable templates have
2424	// partially-ordered initialization within their translation unit.
2425	bool NonTemplateInline =
2426	D.isInline() &&
2427	!isTemplateInstantiation(Kind: D.getTemplateSpecializationKind());
2428
2429	// We only need to use thread-safe statics for local non-TLS variables and
2430	// inline variables; other global initialization is always single-threaded
2431	// or (through lazy dynamic loading in multiple threads) unsequenced.
2432	bool threadsafe = getContext().getLangOpts().ThreadsafeStatics &&
2433	(D.isLocalVarDecl() || NonTemplateInline) &&
2434	!D.getTLSKind();
2435
2436	// If we have a global variable with internal linkage and thread-safe statics
2437	// are disabled, we can just let the guard variable be of type i8.
2438	bool useInt8GuardVariable = !threadsafe && var->hasInternalLinkage();
2439
2440	llvm::IntegerType *guardTy;
2441	CharUnits guardAlignment;
2442	if (useInt8GuardVariable) {
2443	guardTy = CGF.Int8Ty;
2444	guardAlignment = CharUnits::One();
2445	} else {
2446	// Guard variables are 64 bits in the generic ABI and size width on ARM
2447	// (i.e. 32-bit on AArch32, 64-bit on AArch64).
2448	if (UseARMGuardVarABI) {
2449	guardTy = CGF.SizeTy;
2450	guardAlignment = CGF.getSizeAlign();
2451	} else {
2452	guardTy = CGF.Int64Ty;
2453	guardAlignment =
2454	CharUnits::fromQuantity(Quantity: CGM.getDataLayout().getABITypeAlign(Ty: guardTy));
2455	}
2456	}
2457	llvm::PointerType *guardPtrTy = llvm::PointerType::get(
2458	C&: CGF.CGM.getLLVMContext(),
2459	AddressSpace: CGF.CGM.getDataLayout().getDefaultGlobalsAddressSpace());
2460
2461	// Create the guard variable if we don't already have it (as we
2462	// might if we're double-emitting this function body).
2463	llvm::GlobalVariable *guard = CGM.getStaticLocalDeclGuardAddress(D: &D);
2464	if (!guard) {
2465	// Mangle the name for the guard.
2466	SmallString<256> guardName;
2467	{
2468	llvm::raw_svector_ostream out(guardName);
2469	getMangleContext().mangleStaticGuardVariable(D: &D, out);
2470	}
2471
2472	// Create the guard variable with a zero-initializer.
2473	// Just absorb linkage, visibility and dll storage class from the guarded
2474	// variable.
2475	guard = new llvm::GlobalVariable(CGM.getModule(), guardTy,
2476	false, var->getLinkage(),
2477	llvm::ConstantInt::get(Ty: guardTy, V: 0),
2478	guardName.str());
2479	guard->setDSOLocal(var->isDSOLocal());
2480	guard->setVisibility(var->getVisibility());
2481	guard->setDLLStorageClass(var->getDLLStorageClass());
2482	// If the variable is thread-local, so is its guard variable.
2483	guard->setThreadLocalMode(var->getThreadLocalMode());
2484	guard->setAlignment(guardAlignment.getAsAlign());
2485
2486	// The ABI says: "It is suggested that it be emitted in the same COMDAT
2487	// group as the associated data object." In practice, this doesn't work for
2488	// non-ELF and non-Wasm object formats, so only do it for ELF and Wasm.
2489	llvm::Comdat *C = var->getComdat();
2490	if (!D.isLocalVarDecl() && C &&
2491	(CGM.getTarget().getTriple().isOSBinFormatELF() ||
2492	CGM.getTarget().getTriple().isOSBinFormatWasm())) {
2493	guard->setComdat(C);
2494	} else if (CGM.supportsCOMDAT() && guard->isWeakForLinker()) {
2495	guard->setComdat(CGM.getModule().getOrInsertComdat(Name: guard->getName()));
2496	}
2497
2498	CGM.setStaticLocalDeclGuardAddress(D: &D, C: guard);
2499	}
2500
2501	Address guardAddr = Address(guard, guard->getValueType(), guardAlignment);
2502
2503	// Test whether the variable has completed initialization.
2504	//
2505	// Itanium C++ ABI 3.3.2:
2506	// The following is pseudo-code showing how these functions can be used:
2507	// if (obj_guard.first_byte == 0) {
2508	// if ( __cxa_guard_acquire (&obj_guard) ) {
2509	// try {
2510	// ... initialize the object ...;
2511	// } catch (...) {
2512	// __cxa_guard_abort (&obj_guard);
2513	// throw;
2514	// }
2515	// ... queue object destructor with __cxa_atexit() ...;
2516	// __cxa_guard_release (&obj_guard);
2517	// }
2518	// }
2519	//
2520	// If threadsafe statics are enabled, but we don't have inline atomics, just
2521	// call __cxa_guard_acquire unconditionally. The "inline" check isn't
2522	// actually inline, and the user might not expect calls to __atomic libcalls.
2523
2524	unsigned MaxInlineWidthInBits = CGF.getTarget().getMaxAtomicInlineWidth();
2525	llvm::BasicBlock *EndBlock = CGF.createBasicBlock(name: "init.end");
2526	if (!threadsafe || MaxInlineWidthInBits) {
2527	// Load the first byte of the guard variable.
2528	llvm::LoadInst *LI =
2529	Builder.CreateLoad(Addr: guardAddr.withElementType(ElemTy: CGM.Int8Ty));
2530
2531	// Itanium ABI:
2532	// An implementation supporting thread-safety on multiprocessor
2533	// systems must also guarantee that references to the initialized
2534	// object do not occur before the load of the initialization flag.
2535	//
2536	// In LLVM, we do this by marking the load Acquire.
2537	if (threadsafe)
2538	LI->setAtomic(Ordering: llvm::AtomicOrdering::Acquire);
2539
2540	// For ARM, we should only check the first bit, rather than the entire byte:
2541	//
2542	// ARM C++ ABI 3.2.3.1:
2543	// To support the potential use of initialization guard variables
2544	// as semaphores that are the target of ARM SWP and LDREX/STREX
2545	// synchronizing instructions we define a static initialization
2546	// guard variable to be a 4-byte aligned, 4-byte word with the
2547	// following inline access protocol.
2548	// #define INITIALIZED 1
2549	// if ((obj_guard & INITIALIZED) != INITIALIZED) {
2550	// if (__cxa_guard_acquire(&obj_guard))
2551	// ...
2552	// }
2553	//
2554	// and similarly for ARM64:
2555	//
2556	// ARM64 C++ ABI 3.2.2:
2557	// This ABI instead only specifies the value bit 0 of the static guard
2558	// variable; all other bits are platform defined. Bit 0 shall be 0 when the
2559	// variable is not initialized and 1 when it is.
2560	llvm::Value *V =
2561	(UseARMGuardVarABI && !useInt8GuardVariable)
2562	? Builder.CreateAnd(LHS: LI, RHS: llvm::ConstantInt::get(Ty: CGM.Int8Ty, V: 1))
2563	: LI;
2564	llvm::Value *NeedsInit = Builder.CreateIsNull(Arg: V, Name: "guard.uninitialized");
2565
2566	llvm::BasicBlock *InitCheckBlock = CGF.createBasicBlock(name: "init.check");
2567
2568	// Check if the first byte of the guard variable is zero.
2569	CGF.EmitCXXGuardedInitBranch(NeedsInit, InitBlock: InitCheckBlock, NoInitBlock: EndBlock,
2570	Kind: CodeGenFunction::GuardKind::VariableGuard, D: &D);
2571
2572	CGF.EmitBlock(BB: InitCheckBlock);
2573	}
2574
2575	// The semantics of dynamic initialization of variables with static or thread
2576	// storage duration depends on whether they are declared at block-scope. The
2577	// initialization of such variables at block-scope can be aborted with an
2578	// exception and later retried (per C++20 [stmt.dcl]p4), and recursive entry
2579	// to their initialization has undefined behavior (also per C++20
2580	// [stmt.dcl]p4). For such variables declared at non-block scope, exceptions
2581	// lead to termination (per C++20 [except.terminate]p1), and recursive
2582	// references to the variables are governed only by the lifetime rules (per
2583	// C++20 [class.cdtor]p2), which means such references are perfectly fine as
2584	// long as they avoid touching memory. As a result, block-scope variables must
2585	// not be marked as initialized until after initialization completes (unless
2586	// the mark is reverted following an exception), but non-block-scope variables
2587	// must be marked prior to initialization so that recursive accesses during
2588	// initialization do not restart initialization.
2589
2590	// Variables used when coping with thread-safe statics and exceptions.
2591	if (threadsafe) {
2592	// Call __cxa_guard_acquire.
2593	llvm::Value *V
2594	= CGF.EmitNounwindRuntimeCall(callee: getGuardAcquireFn(CGM, GuardPtrTy: guardPtrTy), args: guard);
2595
2596	llvm::BasicBlock *InitBlock = CGF.createBasicBlock(name: "init");
2597
2598	Builder.CreateCondBr(Cond: Builder.CreateIsNotNull(Arg: V, Name: "tobool"),
2599	True: InitBlock, False: EndBlock);
2600
2601	// Call __cxa_guard_abort along the exceptional edge.
2602	CGF.EHStack.pushCleanup<CallGuardAbort>(Kind: EHCleanup, A: guard);
2603
2604	CGF.EmitBlock(BB: InitBlock);
2605	} else if (!D.isLocalVarDecl()) {
2606	// For non-local variables, store 1 into the first byte of the guard
2607	// variable before the object initialization begins so that references
2608	// to the variable during initialization don't restart initialization.
2609	Builder.CreateStore(Val: llvm::ConstantInt::get(Ty: CGM.Int8Ty, V: 1),
2610	Addr: guardAddr.withElementType(ElemTy: CGM.Int8Ty));
2611	}
2612
2613	// Emit the initializer and add a global destructor if appropriate.
2614	CGF.EmitCXXGlobalVarDeclInit(D, GV: var, PerformInit: shouldPerformInit);
2615
2616	if (threadsafe) {
2617	// Pop the guard-abort cleanup if we pushed one.
2618	CGF.PopCleanupBlock();
2619
2620	// Call __cxa_guard_release. This cannot throw.
2621	CGF.EmitNounwindRuntimeCall(callee: getGuardReleaseFn(CGM, GuardPtrTy: guardPtrTy),
2622	args: guardAddr.emitRawPointer(CGF));
2623	} else if (D.isLocalVarDecl()) {
2624	// For local variables, store 1 into the first byte of the guard variable
2625	// after the object initialization completes so that initialization is
2626	// retried if initialization is interrupted by an exception.
2627	Builder.CreateStore(Val: llvm::ConstantInt::get(Ty: CGM.Int8Ty, V: 1),
2628	Addr: guardAddr.withElementType(ElemTy: CGM.Int8Ty));
2629	}
2630
2631	CGF.EmitBlock(BB: EndBlock);
2632	}
2633
2634	/// Register a global destructor using __cxa_atexit.
2635	static void emitGlobalDtorWithCXAAtExit(CodeGenFunction &CGF,
2636	llvm::FunctionCallee dtor,
2637	llvm::Constant *addr, bool TLS) {
2638	assert(!CGF.getTarget().getTriple().isOSAIX() &&
2639	"unexpected call to emitGlobalDtorWithCXAAtExit");
2640	assert((TLS || CGF.getTypes().getCodeGenOpts().CXAAtExit) &&
2641	"__cxa_atexit is disabled");
2642	const char *Name = "__cxa_atexit";
2643	if (TLS) {
2644	const llvm::Triple &T = CGF.getTarget().getTriple();
2645	Name = T.isOSDarwin() ? "_tlv_atexit" : "__cxa_thread_atexit";
2646	}
2647
2648	// We're assuming that the destructor function is something we can
2649	// reasonably call with the default CC.
2650	llvm::Type *dtorTy = CGF.UnqualPtrTy;
2651
2652	// Preserve address space of addr.
2653	auto AddrAS = addr ? addr->getType()->getPointerAddressSpace() : 0;
2654	auto AddrPtrTy = AddrAS ? llvm::PointerType::get(C&: CGF.getLLVMContext(), AddressSpace: AddrAS)
2655	: CGF.Int8PtrTy;
2656
2657	// Create a variable that binds the atexit to this shared object.
2658	llvm::Constant *handle =
2659	CGF.CGM.CreateRuntimeVariable(Ty: CGF.Int8Ty, Name: "__dso_handle");
2660	auto *GV = cast<llvm::GlobalValue>(Val: handle->stripPointerCasts());
2661	GV->setVisibility(llvm::GlobalValue::HiddenVisibility);
2662
2663	// extern "C" int __cxa_atexit(void (*f)(void *), void *p, void *d);
2664	llvm::Type *paramTys[] = {dtorTy, AddrPtrTy, handle->getType()};
2665	llvm::FunctionType *atexitTy =
2666	llvm::FunctionType::get(Result: CGF.IntTy, Params: paramTys, isVarArg: false);
2667
2668	// Fetch the actual function.
2669	llvm::FunctionCallee atexit = CGF.CGM.CreateRuntimeFunction(Ty: atexitTy, Name);
2670	if (llvm::Function *fn = dyn_cast<llvm::Function>(Val: atexit.getCallee()))
2671	fn->setDoesNotThrow();
2672
2673	if (!addr)
2674	// addr is null when we are trying to register a dtor annotated with
2675	// __attribute__((destructor)) in a constructor function. Using null here is
2676	// okay because this argument is just passed back to the destructor
2677	// function.
2678	addr = llvm::Constant::getNullValue(Ty: CGF.Int8PtrTy);
2679
2680	llvm::Value *args[] = {dtor.getCallee(), addr, handle};
2681	CGF.EmitNounwindRuntimeCall(callee: atexit, args);
2682	}
2683
2684	static llvm::Function *createGlobalInitOrCleanupFn(CodeGen::CodeGenModule &CGM,
2685	StringRef FnName) {
2686	// Create a function that registers/unregisters destructors that have the same
2687	// priority.
2688	llvm::FunctionType *FTy = llvm::FunctionType::get(Result: CGM.VoidTy, isVarArg: false);
2689	llvm::Function *GlobalInitOrCleanupFn = CGM.CreateGlobalInitOrCleanUpFunction(
2690	ty: FTy, name: FnName, FI: CGM.getTypes().arrangeNullaryFunction(), Loc: SourceLocation());
2691
2692	return GlobalInitOrCleanupFn;
2693	}
2694
2695	void CodeGenModule::unregisterGlobalDtorsWithUnAtExit() {
2696	for (const auto &I : DtorsUsingAtExit) {
2697	int Priority = I.first;
2698	std::string GlobalCleanupFnName =
2699	std::string("__GLOBAL_cleanup_") + llvm::to_string(Value: Priority);
2700
2701	llvm::Function *GlobalCleanupFn =
2702	createGlobalInitOrCleanupFn(CGM&: *this, FnName: GlobalCleanupFnName);
2703
2704	CodeGenFunction CGF(*this);
2705	CGF.StartFunction(GD: GlobalDecl(), RetTy: getContext().VoidTy, Fn: GlobalCleanupFn,
2706	FnInfo: getTypes().arrangeNullaryFunction(), Args: FunctionArgList(),
2707	Loc: SourceLocation(), StartLoc: SourceLocation());
2708	auto AL = ApplyDebugLocation::CreateArtificial(CGF);
2709
2710	// Get the destructor function type, void(*)(void).
2711	llvm::FunctionType *dtorFuncTy = llvm::FunctionType::get(Result: CGF.VoidTy, isVarArg: false);
2712
2713	// Destructor functions are run/unregistered in non-ascending
2714	// order of their priorities.
2715	const llvm::TinyPtrVector<llvm::Function *> &Dtors = I.second;
2716	auto itv = Dtors.rbegin();
2717	while (itv != Dtors.rend()) {
2718	llvm::Function *Dtor = *itv;
2719
2720	// We're assuming that the destructor function is something we can
2721	// reasonably call with the correct CC.
2722	llvm::Value *V = CGF.unregisterGlobalDtorWithUnAtExit(dtorStub: Dtor);
2723	llvm::Value *NeedsDestruct =
2724	CGF.Builder.CreateIsNull(Arg: V, Name: "needs_destruct");
2725
2726	llvm::BasicBlock *DestructCallBlock =
2727	CGF.createBasicBlock(name: "destruct.call");
2728	llvm::BasicBlock *EndBlock = CGF.createBasicBlock(
2729	name: (itv + 1) != Dtors.rend() ? "unatexit.call" : "destruct.end");
2730	// Check if unatexit returns a value of 0. If it does, jump to
2731	// DestructCallBlock, otherwise jump to EndBlock directly.
2732	CGF.Builder.CreateCondBr(Cond: NeedsDestruct, True: DestructCallBlock, False: EndBlock);
2733
2734	CGF.EmitBlock(BB: DestructCallBlock);
2735
2736	// Emit the call to casted Dtor.
2737	llvm::CallInst *CI = CGF.Builder.CreateCall(FTy: dtorFuncTy, Callee: Dtor);
2738	// Make sure the call and the callee agree on calling convention.
2739	CI->setCallingConv(Dtor->getCallingConv());
2740
2741	CGF.EmitBlock(BB: EndBlock);
2742
2743	itv++;
2744	}
2745
2746	CGF.FinishFunction();
2747	AddGlobalDtor(Dtor: GlobalCleanupFn, Priority);
2748	}
2749	}
2750
2751	void CodeGenModule::registerGlobalDtorsWithAtExit() {
2752	for (const auto &I : DtorsUsingAtExit) {
2753	int Priority = I.first;
2754	std::string GlobalInitFnName =
2755	std::string("__GLOBAL_init_") + llvm::to_string(Value: Priority);
2756	llvm::Function *GlobalInitFn =
2757	createGlobalInitOrCleanupFn(CGM&: *this, FnName: GlobalInitFnName);
2758
2759	CodeGenFunction CGF(*this);
2760	CGF.StartFunction(GD: GlobalDecl(), RetTy: getContext().VoidTy, Fn: GlobalInitFn,
2761	FnInfo: getTypes().arrangeNullaryFunction(), Args: FunctionArgList(),
2762	Loc: SourceLocation(), StartLoc: SourceLocation());
2763	auto AL = ApplyDebugLocation::CreateArtificial(CGF);
2764
2765	// Since constructor functions are run in non-descending order of their
2766	// priorities, destructors are registered in non-descending order of their
2767	// priorities, and since destructor functions are run in the reverse order
2768	// of their registration, destructor functions are run in non-ascending
2769	// order of their priorities.
2770	const llvm::TinyPtrVector<llvm::Function *> &Dtors = I.second;
2771	for (auto *Dtor : Dtors) {
2772	// Register the destructor function calling __cxa_atexit if it is
2773	// available. Otherwise fall back on calling atexit.
2774	if (getCodeGenOpts().CXAAtExit) {
2775	emitGlobalDtorWithCXAAtExit(CGF, dtor: Dtor, addr: nullptr, TLS: false);
2776	} else {
2777	// We're assuming that the destructor function is something we can
2778	// reasonably call with the correct CC.
2779	CGF.registerGlobalDtorWithAtExit(dtorStub: Dtor);
2780	}
2781	}
2782
2783	CGF.FinishFunction();
2784	AddGlobalCtor(Ctor: GlobalInitFn, Priority);
2785	}
2786
2787	if (getCXXABI().useSinitAndSterm())
2788	unregisterGlobalDtorsWithUnAtExit();
2789	}
2790
2791	/// Register a global destructor as best as we know how.
2792	void ItaniumCXXABI::registerGlobalDtor(CodeGenFunction &CGF, const VarDecl &D,
2793	llvm::FunctionCallee dtor,
2794	llvm::Constant *addr) {
2795	if (D.isNoDestroy(CGM.getContext()))
2796	return;
2797
2798	// OpenMP offloading supports C++ constructors and destructors but we do not
2799	// always have 'atexit' available. Instead lower these to use the LLVM global
2800	// destructors which we can handle directly in the runtime. Note that this is
2801	// not strictly 1-to-1 with using `atexit` because we no longer tear down
2802	// globals in reverse order of when they were constructed.
2803	if (!CGM.getLangOpts().hasAtExit() && !D.isStaticLocal())
2804	return CGF.registerGlobalDtorWithLLVM(D, fn: dtor, addr);
2805
2806	// emitGlobalDtorWithCXAAtExit will emit a call to either __cxa_thread_atexit
2807	// or __cxa_atexit depending on whether this VarDecl is a thread-local storage
2808	// or not. CXAAtExit controls only __cxa_atexit, so use it if it is enabled.
2809	// We can always use __cxa_thread_atexit.
2810	if (CGM.getCodeGenOpts().CXAAtExit || D.getTLSKind())
2811	return emitGlobalDtorWithCXAAtExit(CGF, dtor, addr, TLS: D.getTLSKind());
2812
2813	// In Apple kexts, we want to add a global destructor entry.
2814	// FIXME: shouldn't this be guarded by some variable?
2815	if (CGM.getLangOpts().AppleKext) {
2816	// Generate a global destructor entry.
2817	return CGM.AddCXXDtorEntry(DtorFn: dtor, Object: addr);
2818	}
2819
2820	CGF.registerGlobalDtorWithAtExit(D, fn: dtor, addr);
2821	}
2822
2823	static bool isThreadWrapperReplaceable(const VarDecl *VD,
2824	CodeGen::CodeGenModule &CGM) {
2825	assert(!VD->isStaticLocal() && "static local VarDecls don't need wrappers!");
2826	// Darwin prefers to have references to thread local variables to go through
2827	// the thread wrapper instead of directly referencing the backing variable.
2828	return VD->getTLSKind() == VarDecl::TLS_Dynamic &&
2829	CGM.getTarget().getTriple().isOSDarwin();
2830	}
2831
2832	/// Get the appropriate linkage for the wrapper function. This is essentially
2833	/// the weak form of the variable's linkage; every translation unit which needs
2834	/// the wrapper emits a copy, and we want the linker to merge them.
2835	static llvm::GlobalValue::LinkageTypes
2836	getThreadLocalWrapperLinkage(const VarDecl *VD, CodeGen::CodeGenModule &CGM) {
2837	llvm::GlobalValue::LinkageTypes VarLinkage =
2838	CGM.getLLVMLinkageVarDefinition(VD);
2839
2840	// For internal linkage variables, we don't need an external or weak wrapper.
2841	if (llvm::GlobalValue::isLocalLinkage(Linkage: VarLinkage))
2842	return VarLinkage;
2843
2844	// If the thread wrapper is replaceable, give it appropriate linkage.
2845	if (isThreadWrapperReplaceable(VD, CGM))
2846	if (!llvm::GlobalVariable::isLinkOnceLinkage(Linkage: VarLinkage) &&
2847	!llvm::GlobalVariable::isWeakODRLinkage(Linkage: VarLinkage))
2848	return VarLinkage;
2849	return llvm::GlobalValue::WeakODRLinkage;
2850	}
2851
2852	llvm::Function *
2853	ItaniumCXXABI::getOrCreateThreadLocalWrapper(const VarDecl *VD,
2854	llvm::Value *Val) {
2855	// Mangle the name for the thread_local wrapper function.
2856	SmallString<256> WrapperName;
2857	{
2858	llvm::raw_svector_ostream Out(WrapperName);
2859	getMangleContext().mangleItaniumThreadLocalWrapper(D: VD, Out);
2860	}
2861
2862	// FIXME: If VD is a definition, we should regenerate the function attributes
2863	// before returning.
2864	if (llvm::Value *V = CGM.getModule().getNamedValue(Name: WrapperName))
2865	return cast<llvm::Function>(Val: V);
2866
2867	QualType RetQT = VD->getType();
2868	if (RetQT->isReferenceType())
2869	RetQT = RetQT.getNonReferenceType();
2870
2871	const CGFunctionInfo &FI = CGM.getTypes().arrangeBuiltinFunctionDeclaration(
2872	resultType: getContext().getPointerType(T: RetQT), args: FunctionArgList());
2873
2874	llvm::FunctionType *FnTy = CGM.getTypes().GetFunctionType(Info: FI);
2875	llvm::Function *Wrapper =
2876	llvm::Function::Create(Ty: FnTy, Linkage: getThreadLocalWrapperLinkage(VD, CGM),
2877	N: WrapperName.str(), M: &CGM.getModule());
2878
2879	if (CGM.supportsCOMDAT() && Wrapper->isWeakForLinker())
2880	Wrapper->setComdat(CGM.getModule().getOrInsertComdat(Name: Wrapper->getName()));
2881
2882	CGM.SetLLVMFunctionAttributes(GD: GlobalDecl(), Info: FI, F: Wrapper, /*IsThunk=*/false);
2883
2884	// Always resolve references to the wrapper at link time.
2885	if (!Wrapper->hasLocalLinkage())
2886	if (!isThreadWrapperReplaceable(VD, CGM) ||
2887	llvm::GlobalVariable::isLinkOnceLinkage(Linkage: Wrapper->getLinkage()) ||
2888	llvm::GlobalVariable::isWeakODRLinkage(Linkage: Wrapper->getLinkage()) ||
2889	VD->getVisibility() == HiddenVisibility)
2890	Wrapper->setVisibility(llvm::GlobalValue::HiddenVisibility);
2891
2892	if (isThreadWrapperReplaceable(VD, CGM)) {
2893	Wrapper->setCallingConv(llvm::CallingConv::CXX_FAST_TLS);
2894	Wrapper->addFnAttr(llvm::Attribute::NoUnwind);
2895	}
2896
2897	ThreadWrappers.push_back(Elt: {VD, Wrapper});
2898	return Wrapper;
2899	}
2900
2901	void ItaniumCXXABI::EmitThreadLocalInitFuncs(
2902	CodeGenModule &CGM, ArrayRef<const VarDecl *> CXXThreadLocals,
2903	ArrayRef<llvm::Function *> CXXThreadLocalInits,
2904	ArrayRef<const VarDecl *> CXXThreadLocalInitVars) {
2905	llvm::Function *InitFunc = nullptr;
2906
2907	// Separate initializers into those with ordered (or partially-ordered)
2908	// initialization and those with unordered initialization.
2909	llvm::SmallVector<llvm::Function *, 8> OrderedInits;
2910	llvm::SmallDenseMap<const VarDecl *, llvm::Function *> UnorderedInits;
2911	for (unsigned I = 0; I != CXXThreadLocalInits.size(); ++I) {
2912	if (isTemplateInstantiation(
2913	Kind: CXXThreadLocalInitVars[I]->getTemplateSpecializationKind()))
2914	UnorderedInits[CXXThreadLocalInitVars[I]->getCanonicalDecl()] =
2915	CXXThreadLocalInits[I];
2916	else
2917	OrderedInits.push_back(Elt: CXXThreadLocalInits[I]);
2918	}
2919
2920	if (!OrderedInits.empty()) {
2921	// Generate a guarded initialization function.
2922	llvm::FunctionType *FTy =
2923	llvm::FunctionType::get(Result: CGM.VoidTy, /*isVarArg=*/false);
2924	const CGFunctionInfo &FI = CGM.getTypes().arrangeNullaryFunction();
2925	InitFunc = CGM.CreateGlobalInitOrCleanUpFunction(ty: FTy, name: "__tls_init", FI,
2926	Loc: SourceLocation(),
2927	/*TLS=*/true);
2928	llvm::GlobalVariable *Guard = new llvm::GlobalVariable(
2929	CGM.getModule(), CGM.Int8Ty, /*isConstant=*/false,
2930	llvm::GlobalVariable::InternalLinkage,
2931	llvm::ConstantInt::get(Ty: CGM.Int8Ty, V: 0), "__tls_guard");
2932	Guard->setThreadLocal(true);
2933	Guard->setThreadLocalMode(CGM.GetDefaultLLVMTLSModel());
2934
2935	CharUnits GuardAlign = CharUnits::One();
2936	Guard->setAlignment(GuardAlign.getAsAlign());
2937
2938	CodeGenFunction(CGM).GenerateCXXGlobalInitFunc(
2939	Fn: InitFunc, CXXThreadLocals: OrderedInits, Guard: ConstantAddress(Guard, CGM.Int8Ty, GuardAlign));
2940	// On Darwin platforms, use CXX_FAST_TLS calling convention.
2941	if (CGM.getTarget().getTriple().isOSDarwin()) {
2942	InitFunc->setCallingConv(llvm::CallingConv::CXX_FAST_TLS);
2943	InitFunc->addFnAttr(llvm::Attribute::NoUnwind);
2944	}
2945	}
2946
2947	// Create declarations for thread wrappers for all thread-local variables
2948	// with non-discardable definitions in this translation unit.
2949	for (const VarDecl *VD : CXXThreadLocals) {
2950	if (VD->hasDefinition() &&
2951	!isDiscardableGVALinkage(L: getContext().GetGVALinkageForVariable(VD))) {
2952	llvm::GlobalValue *GV = CGM.GetGlobalValue(Ref: CGM.getMangledName(GD: VD));
2953	getOrCreateThreadLocalWrapper(VD, Val: GV);
2954	}
2955	}
2956
2957	// Emit all referenced thread wrappers.
2958	for (auto VDAndWrapper : ThreadWrappers) {
2959	const VarDecl *VD = VDAndWrapper.first;
2960	llvm::GlobalVariable *Var =
2961	cast<llvm::GlobalVariable>(Val: CGM.GetGlobalValue(Ref: CGM.getMangledName(GD: VD)));
2962	llvm::Function *Wrapper = VDAndWrapper.second;
2963
2964	// Some targets require that all access to thread local variables go through
2965	// the thread wrapper. This means that we cannot attempt to create a thread
2966	// wrapper or a thread helper.
2967	if (!VD->hasDefinition()) {
2968	if (isThreadWrapperReplaceable(VD, CGM)) {
2969	Wrapper->setLinkage(llvm::Function::ExternalLinkage);
2970	continue;
2971	}
2972
2973	// If this isn't a TU in which this variable is defined, the thread
2974	// wrapper is discardable.
2975	if (Wrapper->getLinkage() == llvm::Function::WeakODRLinkage)
2976	Wrapper->setLinkage(llvm::Function::LinkOnceODRLinkage);
2977	}
2978
2979	CGM.SetLLVMFunctionAttributesForDefinition(D: nullptr, F: Wrapper);
2980
2981	// Mangle the name for the thread_local initialization function.
2982	SmallString<256> InitFnName;
2983	{
2984	llvm::raw_svector_ostream Out(InitFnName);
2985	getMangleContext().mangleItaniumThreadLocalInit(D: VD, Out);
2986	}
2987
2988	llvm::FunctionType *InitFnTy = llvm::FunctionType::get(Result: CGM.VoidTy, isVarArg: false);
2989
2990	// If we have a definition for the variable, emit the initialization
2991	// function as an alias to the global Init function (if any). Otherwise,
2992	// produce a declaration of the initialization function.
2993	llvm::GlobalValue *Init = nullptr;
2994	bool InitIsInitFunc = false;
2995	bool HasConstantInitialization = false;
2996	if (!usesThreadWrapperFunction(VD)) {
2997	HasConstantInitialization = true;
2998	} else if (VD->hasDefinition()) {
2999	InitIsInitFunc = true;
3000	llvm::Function *InitFuncToUse = InitFunc;
3001	if (isTemplateInstantiation(Kind: VD->getTemplateSpecializationKind()))
3002	InitFuncToUse = UnorderedInits.lookup(Val: VD->getCanonicalDecl());
3003	if (InitFuncToUse)
3004	Init = llvm::GlobalAlias::create(Linkage: Var->getLinkage(), Name: InitFnName.str(),
3005	Aliasee: InitFuncToUse);
3006	} else {
3007	// Emit a weak global function referring to the initialization function.
3008	// This function will not exist if the TU defining the thread_local
3009	// variable in question does not need any dynamic initialization for
3010	// its thread_local variables.
3011	Init = llvm::Function::Create(Ty: InitFnTy,
3012	Linkage: llvm::GlobalVariable::ExternalWeakLinkage,
3013	N: InitFnName.str(), M: &CGM.getModule());
3014	const CGFunctionInfo &FI = CGM.getTypes().arrangeNullaryFunction();
3015	CGM.SetLLVMFunctionAttributes(
3016	GD: GlobalDecl(), Info: FI, F: cast<llvm::Function>(Val: Init), /*IsThunk=*/false);
3017	}
3018
3019	if (Init) {
3020	Init->setVisibility(Var->getVisibility());
3021	// Don't mark an extern_weak function DSO local on windows.
3022	if (!CGM.getTriple().isOSWindows() || !Init->hasExternalWeakLinkage())
3023	Init->setDSOLocal(Var->isDSOLocal());
3024	}
3025
3026	llvm::LLVMContext &Context = CGM.getModule().getContext();
3027
3028	// The linker on AIX is not happy with missing weak symbols. However,
3029	// other TUs will not know whether the initialization routine exists
3030	// so create an empty, init function to satisfy the linker.
3031	// This is needed whenever a thread wrapper function is not used, and
3032	// also when the symbol is weak.
3033	if (CGM.getTriple().isOSAIX() && VD->hasDefinition() &&
3034	isEmittedWithConstantInitializer(VD, InspectInitForWeakDef: true) &&
3035	!mayNeedDestruction(VD)) {
3036	// Init should be null. If it were non-null, then the logic above would
3037	// either be defining the function to be an alias or declaring the
3038	// function with the expectation that the definition of the variable
3039	// is elsewhere.
3040	assert(Init == nullptr && "Expected Init to be null.");
3041
3042	llvm::Function *Func = llvm::Function::Create(
3043	Ty: InitFnTy, Linkage: Var->getLinkage(), N: InitFnName.str(), M: &CGM.getModule());
3044	const CGFunctionInfo &FI = CGM.getTypes().arrangeNullaryFunction();
3045	CGM.SetLLVMFunctionAttributes(GD: GlobalDecl(), Info: FI,
3046	F: cast<llvm::Function>(Val: Func),
3047	/*IsThunk=*/false);
3048	// Create a function body that just returns
3049	llvm::BasicBlock *Entry = llvm::BasicBlock::Create(Context, Name: "", Parent: Func);
3050	CGBuilderTy Builder(CGM, Entry);
3051	Builder.CreateRetVoid();
3052	}
3053
3054	llvm::BasicBlock *Entry = llvm::BasicBlock::Create(Context, Name: "", Parent: Wrapper);
3055	CGBuilderTy Builder(CGM, Entry);
3056	if (HasConstantInitialization) {
3057	// No dynamic initialization to invoke.
3058	} else if (InitIsInitFunc) {
3059	if (Init) {
3060	llvm::CallInst *CallVal = Builder.CreateCall(FTy: InitFnTy, Callee: Init);
3061	if (isThreadWrapperReplaceable(VD, CGM)) {
3062	CallVal->setCallingConv(llvm::CallingConv::CXX_FAST_TLS);
3063	llvm::Function *Fn =
3064	cast<llvm::Function>(Val: cast<llvm::GlobalAlias>(Val: Init)->getAliasee());
3065	Fn->setCallingConv(llvm::CallingConv::CXX_FAST_TLS);
3066	}
3067	}
3068	} else if (CGM.getTriple().isOSAIX()) {
3069	// On AIX, except if constinit and also neither of class type or of
3070	// (possibly multi-dimensional) array of class type, thread_local vars
3071	// will have init routines regardless of whether they are
3072	// const-initialized. Since the routine is guaranteed to exist, we can
3073	// unconditionally call it without testing for its existance. This
3074	// avoids potentially unresolved weak symbols which the AIX linker
3075	// isn't happy with.
3076	Builder.CreateCall(FTy: InitFnTy, Callee: Init);
3077	} else {
3078	// Don't know whether we have an init function. Call it if it exists.
3079	llvm::Value *Have = Builder.CreateIsNotNull(Arg: Init);
3080	llvm::BasicBlock *InitBB = llvm::BasicBlock::Create(Context, Name: "", Parent: Wrapper);
3081	llvm::BasicBlock *ExitBB = llvm::BasicBlock::Create(Context, Name: "", Parent: Wrapper);
3082	Builder.CreateCondBr(Cond: Have, True: InitBB, False: ExitBB);
3083
3084	Builder.SetInsertPoint(InitBB);
3085	Builder.CreateCall(FTy: InitFnTy, Callee: Init);
3086	Builder.CreateBr(Dest: ExitBB);
3087
3088	Builder.SetInsertPoint(ExitBB);
3089	}
3090
3091	// For a reference, the result of the wrapper function is a pointer to
3092	// the referenced object.
3093	llvm::Value *Val = Builder.CreateThreadLocalAddress(Ptr: Var);
3094
3095	if (VD->getType()->isReferenceType()) {
3096	CharUnits Align = CGM.getContext().getDeclAlign(VD);
3097	Val = Builder.CreateAlignedLoad(Ty: Var->getValueType(), Addr: Val, Align);
3098	}
3099
3100	Builder.CreateRet(V: Val);
3101	}
3102	}
3103
3104	LValue ItaniumCXXABI::EmitThreadLocalVarDeclLValue(CodeGenFunction &CGF,
3105	const VarDecl *VD,
3106	QualType LValType) {
3107	llvm::Value *Val = CGF.CGM.GetAddrOfGlobalVar(D: VD);
3108	llvm::Function *Wrapper = getOrCreateThreadLocalWrapper(VD, Val);
3109
3110	llvm::CallInst *CallVal = CGF.Builder.CreateCall(Callee: Wrapper);
3111	CallVal->setCallingConv(Wrapper->getCallingConv());
3112
3113	LValue LV;
3114	if (VD->getType()->isReferenceType())
3115	LV = CGF.MakeNaturalAlignRawAddrLValue(V: CallVal, T: LValType);
3116	else
3117	LV = CGF.MakeRawAddrLValue(V: CallVal, T: LValType,
3118	Alignment: CGF.getContext().getDeclAlign(VD));
3119	// FIXME: need setObjCGCLValueClass?
3120	return LV;
3121	}
3122
3123	/// Return whether the given global decl needs a VTT parameter, which it does
3124	/// if it's a base constructor or destructor with virtual bases.
3125	bool ItaniumCXXABI::NeedsVTTParameter(GlobalDecl GD) {
3126	const CXXMethodDecl *MD = cast<CXXMethodDecl>(Val: GD.getDecl());
3127
3128	// We don't have any virtual bases, just return early.
3129	if (!MD->getParent()->getNumVBases())
3130	return false;
3131
3132	// Check if we have a base constructor.
3133	if (isa<CXXConstructorDecl>(Val: MD) && GD.getCtorType() == Ctor_Base)
3134	return true;
3135
3136	// Check if we have a base destructor.
3137	if (isa<CXXDestructorDecl>(Val: MD) && GD.getDtorType() == Dtor_Base)
3138	return true;
3139
3140	return false;
3141	}
3142
3143	namespace {
3144	class ItaniumRTTIBuilder {
3145	CodeGenModule &CGM; // Per-module state.
3146	llvm::LLVMContext &VMContext;
3147	const ItaniumCXXABI &CXXABI; // Per-module state.
3148
3149	/// Fields - The fields of the RTTI descriptor currently being built.
3150	SmallVector<llvm::Constant *, 16> Fields;
3151
3152	/// GetAddrOfTypeName - Returns the mangled type name of the given type.
3153	llvm::GlobalVariable *
3154	GetAddrOfTypeName(QualType Ty, llvm::GlobalVariable::LinkageTypes Linkage);
3155
3156	/// GetAddrOfExternalRTTIDescriptor - Returns the constant for the RTTI
3157	/// descriptor of the given type.
3158	llvm::Constant *GetAddrOfExternalRTTIDescriptor(QualType Ty);
3159
3160	/// BuildVTablePointer - Build the vtable pointer for the given type.
3161	void BuildVTablePointer(const Type *Ty);
3162
3163	/// BuildSIClassTypeInfo - Build an abi::__si_class_type_info, used for single
3164	/// inheritance, according to the Itanium C++ ABI, 2.9.5p6b.
3165	void BuildSIClassTypeInfo(const CXXRecordDecl *RD);
3166
3167	/// BuildVMIClassTypeInfo - Build an abi::__vmi_class_type_info, used for
3168	/// classes with bases that do not satisfy the abi::__si_class_type_info
3169	/// constraints, according ti the Itanium C++ ABI, 2.9.5p5c.
3170	void BuildVMIClassTypeInfo(const CXXRecordDecl *RD);
3171
3172	/// BuildPointerTypeInfo - Build an abi::__pointer_type_info struct, used
3173	/// for pointer types.
3174	void BuildPointerTypeInfo(QualType PointeeTy);
3175
3176	/// BuildObjCObjectTypeInfo - Build the appropriate kind of
3177	/// type_info for an object type.
3178	void BuildObjCObjectTypeInfo(const ObjCObjectType *Ty);
3179
3180	/// BuildPointerToMemberTypeInfo - Build an abi::__pointer_to_member_type_info
3181	/// struct, used for member pointer types.
3182	void BuildPointerToMemberTypeInfo(const MemberPointerType *Ty);
3183
3184	public:
3185	ItaniumRTTIBuilder(const ItaniumCXXABI &ABI)
3186	: CGM(ABI.CGM), VMContext(CGM.getModule().getContext()), CXXABI(ABI) {}
3187
3188	// Pointer type info flags.
3189	enum {
3190	/// PTI_Const - Type has const qualifier.
3191	PTI_Const = 0x1,
3192
3193	/// PTI_Volatile - Type has volatile qualifier.
3194	PTI_Volatile = 0x2,
3195
3196	/// PTI_Restrict - Type has restrict qualifier.
3197	PTI_Restrict = 0x4,
3198
3199	/// PTI_Incomplete - Type is incomplete.
3200	PTI_Incomplete = 0x8,
3201
3202	/// PTI_ContainingClassIncomplete - Containing class is incomplete.
3203	/// (in pointer to member).
3204	PTI_ContainingClassIncomplete = 0x10,
3205
3206	/// PTI_TransactionSafe - Pointee is transaction_safe function (C++ TM TS).
3207	//PTI_TransactionSafe = 0x20,
3208
3209	/// PTI_Noexcept - Pointee is noexcept function (C++1z).
3210	PTI_Noexcept = 0x40,
3211	};
3212
3213	// VMI type info flags.
3214	enum {
3215	/// VMI_NonDiamondRepeat - Class has non-diamond repeated inheritance.
3216	VMI_NonDiamondRepeat = 0x1,
3217
3218	/// VMI_DiamondShaped - Class is diamond shaped.
3219	VMI_DiamondShaped = 0x2
3220	};
3221
3222	// Base class type info flags.
3223	enum {
3224	/// BCTI_Virtual - Base class is virtual.
3225	BCTI_Virtual = 0x1,
3226
3227	/// BCTI_Public - Base class is public.
3228	BCTI_Public = 0x2
3229	};
3230
3231	/// BuildTypeInfo - Build the RTTI type info struct for the given type, or
3232	/// link to an existing RTTI descriptor if one already exists.
3233	llvm::Constant *BuildTypeInfo(QualType Ty);
3234
3235	/// BuildTypeInfo - Build the RTTI type info struct for the given type.
3236	llvm::Constant *BuildTypeInfo(
3237	QualType Ty,
3238	llvm::GlobalVariable::LinkageTypes Linkage,
3239	llvm::GlobalValue::VisibilityTypes Visibility,
3240	llvm::GlobalValue::DLLStorageClassTypes DLLStorageClass);
3241	};
3242	}
3243
3244	llvm::GlobalVariable *ItaniumRTTIBuilder::GetAddrOfTypeName(
3245	QualType Ty, llvm::GlobalVariable::LinkageTypes Linkage) {
3246	SmallString<256> Name;
3247	llvm::raw_svector_ostream Out(Name);
3248	CGM.getCXXABI().getMangleContext().mangleCXXRTTIName(T: Ty, Out);
3249
3250	// We know that the mangled name of the type starts at index 4 of the
3251	// mangled name of the typename, so we can just index into it in order to
3252	// get the mangled name of the type.
3253	llvm::Constant *Init = llvm::ConstantDataArray::getString(Context&: VMContext,
3254	Initializer: Name.substr(Start: 4));
3255	auto Align = CGM.getContext().getTypeAlignInChars(CGM.getContext().CharTy);
3256
3257	llvm::GlobalVariable *GV = CGM.CreateOrReplaceCXXRuntimeVariable(
3258	Name, Ty: Init->getType(), Linkage, Alignment: Align.getAsAlign());
3259
3260	GV->setInitializer(Init);
3261
3262	return GV;
3263	}
3264
3265	llvm::Constant *
3266	ItaniumRTTIBuilder::GetAddrOfExternalRTTIDescriptor(QualType Ty) {
3267	// Mangle the RTTI name.
3268	SmallString<256> Name;
3269	llvm::raw_svector_ostream Out(Name);
3270	CGM.getCXXABI().getMangleContext().mangleCXXRTTI(T: Ty, Out);
3271
3272	// Look for an existing global.
3273	llvm::GlobalVariable *GV = CGM.getModule().getNamedGlobal(Name);
3274
3275	if (!GV) {
3276	// Create a new global variable.
3277	// Note for the future: If we would ever like to do deferred emission of
3278	// RTTI, check if emitting vtables opportunistically need any adjustment.
3279
3280	GV = new llvm::GlobalVariable(
3281	CGM.getModule(), CGM.GlobalsInt8PtrTy,
3282	/*isConstant=*/true, llvm::GlobalValue::ExternalLinkage, nullptr, Name);
3283	const CXXRecordDecl *RD = Ty->getAsCXXRecordDecl();
3284	CGM.setGVProperties(GV, RD);
3285	// Import the typeinfo symbol when all non-inline virtual methods are
3286	// imported.
3287	if (CGM.getTarget().hasPS4DLLImportExport()) {
3288	if (RD && CXXRecordAllNonInlineVirtualsHaveAttr<DLLImportAttr>(RD)) {
3289	GV->setDLLStorageClass(llvm::GlobalVariable::DLLImportStorageClass);
3290	CGM.setDSOLocal(GV);
3291	}
3292	}
3293	}
3294
3295	return GV;
3296	}
3297
3298	/// TypeInfoIsInStandardLibrary - Given a builtin type, returns whether the type
3299	/// info for that type is defined in the standard library.
3300	static bool TypeInfoIsInStandardLibrary(const BuiltinType *Ty) {
3301	// Itanium C++ ABI 2.9.2:
3302	// Basic type information (e.g. for "int", "bool", etc.) will be kept in
3303	// the run-time support library. Specifically, the run-time support
3304	// library should contain type_info objects for the types X, X* and
3305	// X const*, for every X in: void, std::nullptr_t, bool, wchar_t, char,
3306	// unsigned char, signed char, short, unsigned short, int, unsigned int,
3307	// long, unsigned long, long long, unsigned long long, float, double,
3308	// long double, char16_t, char32_t, and the IEEE 754r decimal and
3309	// half-precision floating point types.
3310	//
3311	// GCC also emits RTTI for __int128.
3312	// FIXME: We do not emit RTTI information for decimal types here.
3313
3314	// Types added here must also be added to EmitFundamentalRTTIDescriptors.
3315	switch (Ty->getKind()) {
3316	case BuiltinType::Void:
3317	case BuiltinType::NullPtr:
3318	case BuiltinType::Bool:
3319	case BuiltinType::WChar_S:
3320	case BuiltinType::WChar_U:
3321	case BuiltinType::Char_U:
3322	case BuiltinType::Char_S:
3323	case BuiltinType::UChar:
3324	case BuiltinType::SChar:
3325	case BuiltinType::Short:
3326	case BuiltinType::UShort:
3327	case BuiltinType::Int:
3328	case BuiltinType::UInt:
3329	case BuiltinType::Long:
3330	case BuiltinType::ULong:
3331	case BuiltinType::LongLong:
3332	case BuiltinType::ULongLong:
3333	case BuiltinType::Half:
3334	case BuiltinType::Float:
3335	case BuiltinType::Double:
3336	case BuiltinType::LongDouble:
3337	case BuiltinType::Float16:
3338	case BuiltinType::Float128:
3339	case BuiltinType::Ibm128:
3340	case BuiltinType::Char8:
3341	case BuiltinType::Char16:
3342	case BuiltinType::Char32:
3343	case BuiltinType::Int128:
3344	case BuiltinType::UInt128:
3345	return true;
3346
3347	#define IMAGE_TYPE(ImgType, Id, SingletonId, Access, Suffix) \
3348	case BuiltinType::Id:
3349	#include "clang/Basic/OpenCLImageTypes.def"
3350	#define EXT_OPAQUE_TYPE(ExtType, Id, Ext) \
3351	case BuiltinType::Id:
3352	#include "clang/Basic/OpenCLExtensionTypes.def"
3353	case BuiltinType::OCLSampler:
3354	case BuiltinType::OCLEvent:
3355	case BuiltinType::OCLClkEvent:
3356	case BuiltinType::OCLQueue:
3357	case BuiltinType::OCLReserveID:
3358	#define SVE_TYPE(Name, Id, SingletonId) \
3359	case BuiltinType::Id:
3360	#include "clang/Basic/AArch64SVEACLETypes.def"
3361	#define PPC_VECTOR_TYPE(Name, Id, Size) \
3362	case BuiltinType::Id:
3363	#include "clang/Basic/PPCTypes.def"
3364	#define RVV_TYPE(Name, Id, SingletonId) case BuiltinType::Id:
3365	#include "clang/Basic/RISCVVTypes.def"
3366	#define WASM_TYPE(Name, Id, SingletonId) case BuiltinType::Id:
3367	#include "clang/Basic/WebAssemblyReferenceTypes.def"
3368	case BuiltinType::ShortAccum:
3369	case BuiltinType::Accum:
3370	case BuiltinType::LongAccum:
3371	case BuiltinType::UShortAccum:
3372	case BuiltinType::UAccum:
3373	case BuiltinType::ULongAccum:
3374	case BuiltinType::ShortFract:
3375	case BuiltinType::Fract:
3376	case BuiltinType::LongFract:
3377	case BuiltinType::UShortFract:
3378	case BuiltinType::UFract:
3379	case BuiltinType::ULongFract:
3380	case BuiltinType::SatShortAccum:
3381	case BuiltinType::SatAccum:
3382	case BuiltinType::SatLongAccum:
3383	case BuiltinType::SatUShortAccum:
3384	case BuiltinType::SatUAccum:
3385	case BuiltinType::SatULongAccum:
3386	case BuiltinType::SatShortFract:
3387	case BuiltinType::SatFract:
3388	case BuiltinType::SatLongFract:
3389	case BuiltinType::SatUShortFract:
3390	case BuiltinType::SatUFract:
3391	case BuiltinType::SatULongFract:
3392	case BuiltinType::BFloat16:
3393	return false;
3394
3395	case BuiltinType::Dependent:
3396	#define BUILTIN_TYPE(Id, SingletonId)
3397	#define PLACEHOLDER_TYPE(Id, SingletonId) \
3398	case BuiltinType::Id:
3399	#include "clang/AST/BuiltinTypes.def"
3400	llvm_unreachable("asking for RRTI for a placeholder type!");
3401
3402	case BuiltinType::ObjCId:
3403	case BuiltinType::ObjCClass:
3404	case BuiltinType::ObjCSel:
3405	llvm_unreachable("FIXME: Objective-C types are unsupported!");
3406	}
3407
3408	llvm_unreachable("Invalid BuiltinType Kind!");
3409	}
3410
3411	static bool TypeInfoIsInStandardLibrary(const PointerType *PointerTy) {
3412	QualType PointeeTy = PointerTy->getPointeeType();
3413	const BuiltinType *BuiltinTy = dyn_cast<BuiltinType>(Val&: PointeeTy);
3414	if (!BuiltinTy)
3415	return false;
3416
3417	// Check the qualifiers.
3418	Qualifiers Quals = PointeeTy.getQualifiers();
3419	Quals.removeConst();
3420
3421	if (!Quals.empty())
3422	return false;
3423
3424	return TypeInfoIsInStandardLibrary(Ty: BuiltinTy);
3425	}
3426
3427	/// IsStandardLibraryRTTIDescriptor - Returns whether the type
3428	/// information for the given type exists in the standard library.
3429	static bool IsStandardLibraryRTTIDescriptor(QualType Ty) {
3430	// Type info for builtin types is defined in the standard library.
3431	if (const BuiltinType *BuiltinTy = dyn_cast<BuiltinType>(Val&: Ty))
3432	return TypeInfoIsInStandardLibrary(Ty: BuiltinTy);
3433
3434	// Type info for some pointer types to builtin types is defined in the
3435	// standard library.
3436	if (const PointerType *PointerTy = dyn_cast<PointerType>(Val&: Ty))
3437	return TypeInfoIsInStandardLibrary(PointerTy);
3438
3439	return false;
3440	}
3441
3442	/// ShouldUseExternalRTTIDescriptor - Returns whether the type information for
3443	/// the given type exists somewhere else, and that we should not emit the type
3444	/// information in this translation unit. Assumes that it is not a
3445	/// standard-library type.
3446	static bool ShouldUseExternalRTTIDescriptor(CodeGenModule &CGM,
3447	QualType Ty) {
3448	ASTContext &Context = CGM.getContext();
3449
3450	// If RTTI is disabled, assume it might be disabled in the
3451	// translation unit that defines any potential key function, too.
3452	if (!Context.getLangOpts().RTTI) return false;
3453
3454	if (const RecordType *RecordTy = dyn_cast<RecordType>(Val&: Ty)) {
3455	const CXXRecordDecl *RD = cast<CXXRecordDecl>(Val: RecordTy->getDecl());
3456	if (!RD->hasDefinition())
3457	return false;
3458
3459	if (!RD->isDynamicClass())
3460	return false;
3461
3462	// FIXME: this may need to be reconsidered if the key function
3463	// changes.
3464	// N.B. We must always emit the RTTI data ourselves if there exists a key
3465	// function.
3466	bool IsDLLImport = RD->hasAttr<DLLImportAttr>();
3467
3468	// Don't import the RTTI but emit it locally.
3469	if (CGM.getTriple().isWindowsGNUEnvironment())
3470	return false;
3471
3472	if (CGM.getVTables().isVTableExternal(RD)) {
3473	if (CGM.getTarget().hasPS4DLLImportExport())
3474	return true;
3475
3476	return IsDLLImport && !CGM.getTriple().isWindowsItaniumEnvironment()
3477	? false
3478	: true;
3479	}
3480	if (IsDLLImport)
3481	return true;
3482	}
3483
3484	return false;
3485	}
3486
3487	/// IsIncompleteClassType - Returns whether the given record type is incomplete.
3488	static bool IsIncompleteClassType(const RecordType *RecordTy) {
3489	return !RecordTy->getDecl()->isCompleteDefinition();
3490	}
3491
3492	/// ContainsIncompleteClassType - Returns whether the given type contains an
3493	/// incomplete class type. This is true if
3494	///
3495	/// * The given type is an incomplete class type.
3496	/// * The given type is a pointer type whose pointee type contains an
3497	/// incomplete class type.
3498	/// * The given type is a member pointer type whose class is an incomplete
3499	/// class type.
3500	/// * The given type is a member pointer type whoise pointee type contains an
3501	/// incomplete class type.
3502	/// is an indirect or direct pointer to an incomplete class type.
3503	static bool ContainsIncompleteClassType(QualType Ty) {
3504	if (const RecordType *RecordTy = dyn_cast<RecordType>(Val&: Ty)) {
3505	if (IsIncompleteClassType(RecordTy))
3506	return true;
3507	}
3508
3509	if (const PointerType *PointerTy = dyn_cast<PointerType>(Val&: Ty))
3510	return ContainsIncompleteClassType(Ty: PointerTy->getPointeeType());
3511
3512	if (const MemberPointerType *MemberPointerTy =
3513	dyn_cast<MemberPointerType>(Val&: Ty)) {
3514	// Check if the class type is incomplete.
3515	const RecordType *ClassType = cast<RecordType>(Val: MemberPointerTy->getClass());
3516	if (IsIncompleteClassType(RecordTy: ClassType))
3517	return true;
3518
3519	return ContainsIncompleteClassType(Ty: MemberPointerTy->getPointeeType());
3520	}
3521
3522	return false;
3523	}
3524
3525	// CanUseSingleInheritance - Return whether the given record decl has a "single,
3526	// public, non-virtual base at offset zero (i.e. the derived class is dynamic
3527	// iff the base is)", according to Itanium C++ ABI, 2.95p6b.
3528	static bool CanUseSingleInheritance(const CXXRecordDecl *RD) {
3529	// Check the number of bases.
3530	if (RD->getNumBases() != 1)
3531	return false;
3532
3533	// Get the base.
3534	CXXRecordDecl::base_class_const_iterator Base = RD->bases_begin();
3535
3536	// Check that the base is not virtual.
3537	if (Base->isVirtual())
3538	return false;
3539
3540	// Check that the base is public.
3541	if (Base->getAccessSpecifier() != AS_public)
3542	return false;
3543
3544	// Check that the class is dynamic iff the base is.
3545	auto *BaseDecl =
3546	cast<CXXRecordDecl>(Val: Base->getType()->castAs<RecordType>()->getDecl());
3547	if (!BaseDecl->isEmpty() &&
3548	BaseDecl->isDynamicClass() != RD->isDynamicClass())
3549	return false;
3550
3551	return true;
3552	}
3553
3554	void ItaniumRTTIBuilder::BuildVTablePointer(const Type *Ty) {
3555	// abi::__class_type_info.
3556	static const char * const ClassTypeInfo =
3557	"_ZTVN10__cxxabiv117__class_type_infoE";
3558	// abi::__si_class_type_info.
3559	static const char * const SIClassTypeInfo =
3560	"_ZTVN10__cxxabiv120__si_class_type_infoE";
3561	// abi::__vmi_class_type_info.
3562	static const char * const VMIClassTypeInfo =
3563	"_ZTVN10__cxxabiv121__vmi_class_type_infoE";
3564
3565	const char *VTableName = nullptr;
3566
3567	switch (Ty->getTypeClass()) {
3568	#define TYPE(Class, Base)
3569	#define ABSTRACT_TYPE(Class, Base)
3570	#define NON_CANONICAL_UNLESS_DEPENDENT_TYPE(Class, Base) case Type::Class:
3571	#define NON_CANONICAL_TYPE(Class, Base) case Type::Class:
3572	#define DEPENDENT_TYPE(Class, Base) case Type::Class:
3573	#include "clang/AST/TypeNodes.inc"
3574	llvm_unreachable("Non-canonical and dependent types shouldn't get here");
3575
3576	case Type::LValueReference:
3577	case Type::RValueReference:
3578	llvm_unreachable("References shouldn't get here");
3579
3580	case Type::Auto:
3581	case Type::DeducedTemplateSpecialization:
3582	llvm_unreachable("Undeduced type shouldn't get here");
3583
3584	case Type::Pipe:
3585	llvm_unreachable("Pipe types shouldn't get here");
3586
3587	case Type::ArrayParameter:
3588	llvm_unreachable("Array Parameter types should not get here.");
3589
3590	case Type::Builtin:
3591	case Type::BitInt:
3592	// GCC treats vector and complex types as fundamental types.
3593	case Type::Vector:
3594	case Type::ExtVector:
3595	case Type::ConstantMatrix:
3596	case Type::Complex:
3597	case Type::Atomic:
3598	// FIXME: GCC treats block pointers as fundamental types?!
3599	case Type::BlockPointer:
3600	// abi::__fundamental_type_info.
3601	VTableName = "_ZTVN10__cxxabiv123__fundamental_type_infoE";
3602	break;
3603
3604	case Type::ConstantArray:
3605	case Type::IncompleteArray:
3606	case Type::VariableArray:
3607	// abi::__array_type_info.
3608	VTableName = "_ZTVN10__cxxabiv117__array_type_infoE";
3609	break;
3610
3611	case Type::FunctionNoProto:
3612	case Type::FunctionProto:
3613	// abi::__function_type_info.
3614	VTableName = "_ZTVN10__cxxabiv120__function_type_infoE";
3615	break;
3616
3617	case Type::Enum:
3618	// abi::__enum_type_info.
3619	VTableName = "_ZTVN10__cxxabiv116__enum_type_infoE";
3620	break;
3621
3622	case Type::Record: {
3623	const CXXRecordDecl *RD =
3624	cast<CXXRecordDecl>(cast<RecordType>(Ty)->getDecl());
3625
3626	if (!RD->hasDefinition() || !RD->getNumBases()) {
3627	VTableName = ClassTypeInfo;
3628	} else if (CanUseSingleInheritance(RD)) {
3629	VTableName = SIClassTypeInfo;
3630	} else {
3631	VTableName = VMIClassTypeInfo;
3632	}
3633
3634	break;
3635	}
3636
3637	case Type::ObjCObject:
3638	// Ignore protocol qualifiers.
3639	Ty = cast<ObjCObjectType>(Ty)->getBaseType().getTypePtr();
3640
3641	// Handle id and Class.
3642	if (isa<BuiltinType>(Ty)) {
3643	VTableName = ClassTypeInfo;
3644	break;
3645	}
3646
3647	assert(isa<ObjCInterfaceType>(Ty));
3648	[[fallthrough]];
3649
3650	case Type::ObjCInterface:
3651	if (cast<ObjCInterfaceType>(Ty)->getDecl()->getSuperClass()) {
3652	VTableName = SIClassTypeInfo;
3653	} else {
3654	VTableName = ClassTypeInfo;
3655	}
3656	break;
3657
3658	case Type::ObjCObjectPointer:
3659	case Type::Pointer:
3660	// abi::__pointer_type_info.
3661	VTableName = "_ZTVN10__cxxabiv119__pointer_type_infoE";
3662	break;
3663
3664	case Type::MemberPointer:
3665	// abi::__pointer_to_member_type_info.
3666	VTableName = "_ZTVN10__cxxabiv129__pointer_to_member_type_infoE";
3667	break;
3668	}
3669
3670	llvm::Constant *VTable = nullptr;
3671
3672	// Check if the alias exists. If it doesn't, then get or create the global.
3673	if (CGM.getItaniumVTableContext().isRelativeLayout())
3674	VTable = CGM.getModule().getNamedAlias(Name: VTableName);
3675	if (!VTable) {
3676	llvm::Type *Ty = llvm::ArrayType::get(ElementType: CGM.GlobalsInt8PtrTy, NumElements: 0);
3677	VTable = CGM.getModule().getOrInsertGlobal(Name: VTableName, Ty);
3678	}
3679
3680	CGM.setDSOLocal(cast<llvm::GlobalValue>(VTable->stripPointerCasts()));
3681
3682	llvm::Type *PtrDiffTy =
3683	CGM.getTypes().ConvertType(T: CGM.getContext().getPointerDiffType());
3684
3685	// The vtable address point is 2.
3686	if (CGM.getItaniumVTableContext().isRelativeLayout()) {
3687	// The vtable address point is 8 bytes after its start:
3688	// 4 for the offset to top + 4 for the relative offset to rtti.
3689	llvm::Constant *Eight = llvm::ConstantInt::get(Ty: CGM.Int32Ty, V: 8);
3690	VTable =
3691	llvm::ConstantExpr::getInBoundsGetElementPtr(Ty: CGM.Int8Ty, C: VTable, Idx: Eight);
3692	} else {
3693	llvm::Constant *Two = llvm::ConstantInt::get(Ty: PtrDiffTy, V: 2);
3694	VTable = llvm::ConstantExpr::getInBoundsGetElementPtr(Ty: CGM.GlobalsInt8PtrTy,
3695	C: VTable, Idx: Two);
3696	}
3697
3698	Fields.push_back(Elt: VTable);
3699	}
3700
3701	/// Return the linkage that the type info and type info name constants
3702	/// should have for the given type.
3703	static llvm::GlobalVariable::LinkageTypes getTypeInfoLinkage(CodeGenModule &CGM,
3704	QualType Ty) {
3705	// Itanium C++ ABI 2.9.5p7:
3706	// In addition, it and all of the intermediate abi::__pointer_type_info
3707	// structs in the chain down to the abi::__class_type_info for the
3708	// incomplete class type must be prevented from resolving to the
3709	// corresponding type_info structs for the complete class type, possibly
3710	// by making them local static objects. Finally, a dummy class RTTI is
3711	// generated for the incomplete type that will not resolve to the final
3712	// complete class RTTI (because the latter need not exist), possibly by
3713	// making it a local static object.
3714	if (ContainsIncompleteClassType(Ty))
3715	return llvm::GlobalValue::InternalLinkage;
3716
3717	switch (Ty->getLinkage()) {
3718	case Linkage::Invalid:
3719	llvm_unreachable("Linkage hasn't been computed!");
3720
3721	case Linkage::None:
3722	case Linkage::Internal:
3723	case Linkage::UniqueExternal:
3724	return llvm::GlobalValue::InternalLinkage;
3725
3726	case Linkage::VisibleNone:
3727	case Linkage::Module:
3728	case Linkage::External:
3729	// RTTI is not enabled, which means that this type info struct is going
3730	// to be used for exception handling. Give it linkonce_odr linkage.
3731	if (!CGM.getLangOpts().RTTI)
3732	return llvm::GlobalValue::LinkOnceODRLinkage;
3733
3734	if (const RecordType *Record = dyn_cast<RecordType>(Val&: Ty)) {
3735	const CXXRecordDecl *RD = cast<CXXRecordDecl>(Val: Record->getDecl());
3736	if (RD->hasAttr<WeakAttr>())
3737	return llvm::GlobalValue::WeakODRLinkage;
3738	if (CGM.getTriple().isWindowsItaniumEnvironment())
3739	if (RD->hasAttr<DLLImportAttr>() &&
3740	ShouldUseExternalRTTIDescriptor(CGM, Ty))
3741	return llvm::GlobalValue::ExternalLinkage;
3742	// MinGW always uses LinkOnceODRLinkage for type info.
3743	if (RD->isDynamicClass() &&
3744	!CGM.getContext()
3745	.getTargetInfo()
3746	.getTriple()
3747	.isWindowsGNUEnvironment())
3748	return CGM.getVTableLinkage(RD);
3749	}
3750
3751	return llvm::GlobalValue::LinkOnceODRLinkage;
3752	}
3753
3754	llvm_unreachable("Invalid linkage!");
3755	}
3756
3757	llvm::Constant *ItaniumRTTIBuilder::BuildTypeInfo(QualType Ty) {
3758	// We want to operate on the canonical type.
3759	Ty = Ty.getCanonicalType();
3760
3761	// Check if we've already emitted an RTTI descriptor for this type.
3762	SmallString<256> Name;
3763	llvm::raw_svector_ostream Out(Name);
3764	CGM.getCXXABI().getMangleContext().mangleCXXRTTI(T: Ty, Out);
3765
3766	llvm::GlobalVariable *OldGV = CGM.getModule().getNamedGlobal(Name);
3767	if (OldGV && !OldGV->isDeclaration()) {
3768	assert(!OldGV->hasAvailableExternallyLinkage() &&
3769	"available_externally typeinfos not yet implemented");
3770
3771	return OldGV;
3772	}
3773
3774	// Check if there is already an external RTTI descriptor for this type.
3775	if (IsStandardLibraryRTTIDescriptor(Ty) ||
3776	ShouldUseExternalRTTIDescriptor(CGM, Ty))
3777	return GetAddrOfExternalRTTIDescriptor(Ty);
3778
3779	// Emit the standard library with external linkage.
3780	llvm::GlobalVariable::LinkageTypes Linkage = getTypeInfoLinkage(CGM, Ty);
3781
3782	// Give the type_info object and name the formal visibility of the
3783	// type itself.
3784	llvm::GlobalValue::VisibilityTypes llvmVisibility;
3785	if (llvm::GlobalValue::isLocalLinkage(Linkage))
3786	// If the linkage is local, only default visibility makes sense.
3787	llvmVisibility = llvm::GlobalValue::DefaultVisibility;
3788	else if (CXXABI.classifyRTTIUniqueness(CanTy: Ty, Linkage) ==
3789	ItaniumCXXABI::RUK_NonUniqueHidden)
3790	llvmVisibility = llvm::GlobalValue::HiddenVisibility;
3791	else
3792	llvmVisibility = CodeGenModule::GetLLVMVisibility(V: Ty->getVisibility());
3793
3794	llvm::GlobalValue::DLLStorageClassTypes DLLStorageClass =
3795	llvm::GlobalValue::DefaultStorageClass;
3796	if (auto RD = Ty->getAsCXXRecordDecl()) {
3797	if ((CGM.getTriple().isWindowsItaniumEnvironment() &&
3798	RD->hasAttr<DLLExportAttr>()) ||
3799	(CGM.shouldMapVisibilityToDLLExport(RD) &&
3800	!llvm::GlobalValue::isLocalLinkage(Linkage) &&
3801	llvmVisibility == llvm::GlobalValue::DefaultVisibility))
3802	DLLStorageClass = llvm::GlobalValue::DLLExportStorageClass;
3803	}
3804	return BuildTypeInfo(Ty, Linkage, Visibility: llvmVisibility, DLLStorageClass);
3805	}
3806
3807	llvm::Constant *ItaniumRTTIBuilder::BuildTypeInfo(
3808	QualType Ty,
3809	llvm::GlobalVariable::LinkageTypes Linkage,
3810	llvm::GlobalValue::VisibilityTypes Visibility,
3811	llvm::GlobalValue::DLLStorageClassTypes DLLStorageClass) {
3812	// Add the vtable pointer.
3813	BuildVTablePointer(Ty: cast<Type>(Val&: Ty));
3814
3815	// And the name.
3816	llvm::GlobalVariable *TypeName = GetAddrOfTypeName(Ty, Linkage);
3817	llvm::Constant *TypeNameField;
3818
3819	// If we're supposed to demote the visibility, be sure to set a flag
3820	// to use a string comparison for type_info comparisons.
3821	ItaniumCXXABI::RTTIUniquenessKind RTTIUniqueness =
3822	CXXABI.classifyRTTIUniqueness(CanTy: Ty, Linkage);
3823	if (RTTIUniqueness != ItaniumCXXABI::RUK_Unique) {
3824	// The flag is the sign bit, which on ARM64 is defined to be clear
3825	// for global pointers. This is very ARM64-specific.
3826	TypeNameField = llvm::ConstantExpr::getPtrToInt(C: TypeName, Ty: CGM.Int64Ty);
3827	llvm::Constant *flag =
3828	llvm::ConstantInt::get(Ty: CGM.Int64Ty, V: ((uint64_t)1) << 63);
3829	TypeNameField = llvm::ConstantExpr::getAdd(C1: TypeNameField, C2: flag);
3830	TypeNameField =
3831	llvm::ConstantExpr::getIntToPtr(C: TypeNameField, Ty: CGM.GlobalsInt8PtrTy);
3832	} else {
3833	TypeNameField = TypeName;
3834	}
3835	Fields.push_back(Elt: TypeNameField);
3836
3837	switch (Ty->getTypeClass()) {
3838	#define TYPE(Class, Base)
3839	#define ABSTRACT_TYPE(Class, Base)
3840	#define NON_CANONICAL_UNLESS_DEPENDENT_TYPE(Class, Base) case Type::Class:
3841	#define NON_CANONICAL_TYPE(Class, Base) case Type::Class:
3842	#define DEPENDENT_TYPE(Class, Base) case Type::Class:
3843	#include "clang/AST/TypeNodes.inc"
3844	llvm_unreachable("Non-canonical and dependent types shouldn't get here");
3845
3846	// GCC treats vector types as fundamental types.
3847	case Type::Builtin:
3848	case Type::Vector:
3849	case Type::ExtVector:
3850	case Type::ConstantMatrix:
3851	case Type::Complex:
3852	case Type::BlockPointer:
3853	// Itanium C++ ABI 2.9.5p4:
3854	// abi::__fundamental_type_info adds no data members to std::type_info.
3855	break;
3856
3857	case Type::LValueReference:
3858	case Type::RValueReference:
3859	llvm_unreachable("References shouldn't get here");
3860
3861	case Type::Auto:
3862	case Type::DeducedTemplateSpecialization:
3863	llvm_unreachable("Undeduced type shouldn't get here");
3864
3865	case Type::Pipe:
3866	break;
3867
3868	case Type::BitInt:
3869	break;
3870
3871	case Type::ConstantArray:
3872	case Type::IncompleteArray:
3873	case Type::VariableArray:
3874	case Type::ArrayParameter:
3875	// Itanium C++ ABI 2.9.5p5:
3876	// abi::__array_type_info adds no data members to std::type_info.
3877	break;
3878
3879	case Type::FunctionNoProto:
3880	case Type::FunctionProto:
3881	// Itanium C++ ABI 2.9.5p5:
3882	// abi::__function_type_info adds no data members to std::type_info.
3883	break;
3884
3885	case Type::Enum:
3886	// Itanium C++ ABI 2.9.5p5:
3887	// abi::__enum_type_info adds no data members to std::type_info.
3888	break;
3889
3890	case Type::Record: {
3891	const CXXRecordDecl *RD =
3892	cast<CXXRecordDecl>(cast<RecordType>(Ty)->getDecl());
3893	if (!RD->hasDefinition() || !RD->getNumBases()) {
3894	// We don't need to emit any fields.
3895	break;
3896	}
3897
3898	if (CanUseSingleInheritance(RD))
3899	BuildSIClassTypeInfo(RD);
3900	else
3901	BuildVMIClassTypeInfo(RD);
3902
3903	break;
3904	}
3905
3906	case Type::ObjCObject:
3907	case Type::ObjCInterface:
3908	BuildObjCObjectTypeInfo(Ty: cast<ObjCObjectType>(Ty));
3909	break;
3910
3911	case Type::ObjCObjectPointer:
3912	BuildPointerTypeInfo(PointeeTy: cast<ObjCObjectPointerType>(Ty)->getPointeeType());
3913	break;
3914
3915	case Type::Pointer:
3916	BuildPointerTypeInfo(PointeeTy: cast<PointerType>(Ty)->getPointeeType());
3917	break;
3918
3919	case Type::MemberPointer:
3920	BuildPointerToMemberTypeInfo(Ty: cast<MemberPointerType>(Ty));
3921	break;
3922
3923	case Type::Atomic:
3924	// No fields, at least for the moment.
3925	break;
3926	}
3927
3928	llvm::Constant *Init = llvm::ConstantStruct::getAnon(Fields);
3929
3930	SmallString<256> Name;
3931	llvm::raw_svector_ostream Out(Name);
3932	CGM.getCXXABI().getMangleContext().mangleCXXRTTI(T: Ty, Out);
3933	llvm::Module &M = CGM.getModule();
3934	llvm::GlobalVariable *OldGV = M.getNamedGlobal(Name);
3935	llvm::GlobalVariable *GV =
3936	new llvm::GlobalVariable(M, Init->getType(),
3937	/*isConstant=*/true, Linkage, Init, Name);
3938
3939	// Export the typeinfo in the same circumstances as the vtable is exported.
3940	auto GVDLLStorageClass = DLLStorageClass;
3941	if (CGM.getTarget().hasPS4DLLImportExport()) {
3942	if (const RecordType *RecordTy = dyn_cast<RecordType>(Ty)) {
3943	const CXXRecordDecl *RD = cast<CXXRecordDecl>(RecordTy->getDecl());
3944	if (RD->hasAttr<DLLExportAttr>() ||
3945	CXXRecordAllNonInlineVirtualsHaveAttr<DLLExportAttr>(RD)) {
3946	GVDLLStorageClass = llvm::GlobalVariable::DLLExportStorageClass;
3947	}
3948	}
3949	}
3950
3951	// If there's already an old global variable, replace it with the new one.
3952	if (OldGV) {
3953	GV->takeName(V: OldGV);
3954	OldGV->replaceAllUsesWith(V: GV);
3955	OldGV->eraseFromParent();
3956	}
3957
3958	if (CGM.supportsCOMDAT() && GV->isWeakForLinker())
3959	GV->setComdat(M.getOrInsertComdat(Name: GV->getName()));
3960
3961	CharUnits Align = CGM.getContext().toCharUnitsFromBits(
3962	BitSize: CGM.getTarget().getPointerAlign(AddrSpace: CGM.GetGlobalVarAddressSpace(D: nullptr)));
3963	GV->setAlignment(Align.getAsAlign());
3964
3965	// The Itanium ABI specifies that type_info objects must be globally
3966	// unique, with one exception: if the type is an incomplete class
3967	// type or a (possibly indirect) pointer to one. That exception
3968	// affects the general case of comparing type_info objects produced
3969	// by the typeid operator, which is why the comparison operators on
3970	// std::type_info generally use the type_info name pointers instead
3971	// of the object addresses. However, the language's built-in uses
3972	// of RTTI generally require class types to be complete, even when
3973	// manipulating pointers to those class types. This allows the
3974	// implementation of dynamic_cast to rely on address equality tests,
3975	// which is much faster.
3976
3977	// All of this is to say that it's important that both the type_info
3978	// object and the type_info name be uniqued when weakly emitted.
3979
3980	TypeName->setVisibility(Visibility);
3981	CGM.setDSOLocal(TypeName);
3982
3983	GV->setVisibility(Visibility);
3984	CGM.setDSOLocal(GV);
3985
3986	TypeName->setDLLStorageClass(DLLStorageClass);
3987	GV->setDLLStorageClass(CGM.getTarget().hasPS4DLLImportExport()
3988	? GVDLLStorageClass
3989	: DLLStorageClass);
3990
3991	TypeName->setPartition(CGM.getCodeGenOpts().SymbolPartition);
3992	GV->setPartition(CGM.getCodeGenOpts().SymbolPartition);
3993
3994	return GV;
3995	}
3996
3997	/// BuildObjCObjectTypeInfo - Build the appropriate kind of type_info
3998	/// for the given Objective-C object type.
3999	void ItaniumRTTIBuilder::BuildObjCObjectTypeInfo(const ObjCObjectType *OT) {
4000	// Drop qualifiers.
4001	const Type *T = OT->getBaseType().getTypePtr();
4002	assert(isa<BuiltinType>(T) || isa<ObjCInterfaceType>(T));
4003
4004	// The builtin types are abi::__class_type_infos and don't require
4005	// extra fields.
4006	if (isa<BuiltinType>(Val: T)) return;
4007
4008	ObjCInterfaceDecl *Class = cast<ObjCInterfaceType>(Val: T)->getDecl();
4009	ObjCInterfaceDecl *Super = Class->getSuperClass();
4010
4011	// Root classes are also __class_type_info.
4012	if (!Super) return;
4013
4014	QualType SuperTy = CGM.getContext().getObjCInterfaceType(Decl: Super);
4015
4016	// Everything else is single inheritance.
4017	llvm::Constant *BaseTypeInfo =
4018	ItaniumRTTIBuilder(CXXABI).BuildTypeInfo(Ty: SuperTy);
4019	Fields.push_back(Elt: BaseTypeInfo);
4020	}
4021
4022	/// BuildSIClassTypeInfo - Build an abi::__si_class_type_info, used for single
4023	/// inheritance, according to the Itanium C++ ABI, 2.95p6b.
4024	void ItaniumRTTIBuilder::BuildSIClassTypeInfo(const CXXRecordDecl *RD) {
4025	// Itanium C++ ABI 2.9.5p6b:
4026	// It adds to abi::__class_type_info a single member pointing to the
4027	// type_info structure for the base type,
4028	llvm::Constant *BaseTypeInfo =
4029	ItaniumRTTIBuilder(CXXABI).BuildTypeInfo(Ty: RD->bases_begin()->getType());
4030	Fields.push_back(Elt: BaseTypeInfo);
4031	}
4032
4033	namespace {
4034	/// SeenBases - Contains virtual and non-virtual bases seen when traversing
4035	/// a class hierarchy.
4036	struct SeenBases {
4037	llvm::SmallPtrSet<const CXXRecordDecl *, 16> NonVirtualBases;
4038	llvm::SmallPtrSet<const CXXRecordDecl *, 16> VirtualBases;
4039	};
4040	}
4041
4042	/// ComputeVMIClassTypeInfoFlags - Compute the value of the flags member in
4043	/// abi::__vmi_class_type_info.
4044	///
4045	static unsigned ComputeVMIClassTypeInfoFlags(const CXXBaseSpecifier *Base,
4046	SeenBases &Bases) {
4047
4048	unsigned Flags = 0;
4049
4050	auto *BaseDecl =
4051	cast<CXXRecordDecl>(Val: Base->getType()->castAs<RecordType>()->getDecl());
4052
4053	if (Base->isVirtual()) {
4054	// Mark the virtual base as seen.
4055	if (!Bases.VirtualBases.insert(Ptr: BaseDecl).second) {
4056	// If this virtual base has been seen before, then the class is diamond
4057	// shaped.
4058	Flags |= ItaniumRTTIBuilder::VMI_DiamondShaped;
4059	} else {
4060	if (Bases.NonVirtualBases.count(Ptr: BaseDecl))
4061	Flags |= ItaniumRTTIBuilder::VMI_NonDiamondRepeat;
4062	}
4063	} else {
4064	// Mark the non-virtual base as seen.
4065	if (!Bases.NonVirtualBases.insert(Ptr: BaseDecl).second) {
4066	// If this non-virtual base has been seen before, then the class has non-
4067	// diamond shaped repeated inheritance.
4068	Flags |= ItaniumRTTIBuilder::VMI_NonDiamondRepeat;
4069	} else {
4070	if (Bases.VirtualBases.count(Ptr: BaseDecl))
4071	Flags |= ItaniumRTTIBuilder::VMI_NonDiamondRepeat;
4072	}
4073	}
4074
4075	// Walk all bases.
4076	for (const auto &I : BaseDecl->bases())
4077	Flags |= ComputeVMIClassTypeInfoFlags(Base: &I, Bases);
4078
4079	return Flags;
4080	}
4081
4082	static unsigned ComputeVMIClassTypeInfoFlags(const CXXRecordDecl *RD) {
4083	unsigned Flags = 0;
4084	SeenBases Bases;
4085
4086	// Walk all bases.
4087	for (const auto &I : RD->bases())
4088	Flags |= ComputeVMIClassTypeInfoFlags(Base: &I, Bases);
4089
4090	return Flags;
4091	}
4092
4093	/// BuildVMIClassTypeInfo - Build an abi::__vmi_class_type_info, used for
4094	/// classes with bases that do not satisfy the abi::__si_class_type_info
4095	/// constraints, according ti the Itanium C++ ABI, 2.9.5p5c.
4096	void ItaniumRTTIBuilder::BuildVMIClassTypeInfo(const CXXRecordDecl *RD) {
4097	llvm::Type *UnsignedIntLTy =
4098	CGM.getTypes().ConvertType(T: CGM.getContext().UnsignedIntTy);
4099
4100	// Itanium C++ ABI 2.9.5p6c:
4101	// __flags is a word with flags describing details about the class
4102	// structure, which may be referenced by using the __flags_masks
4103	// enumeration. These flags refer to both direct and indirect bases.
4104	unsigned Flags = ComputeVMIClassTypeInfoFlags(RD);
4105	Fields.push_back(Elt: llvm::ConstantInt::get(Ty: UnsignedIntLTy, V: Flags));
4106
4107	// Itanium C++ ABI 2.9.5p6c:
4108	// __base_count is a word with the number of direct proper base class
4109	// descriptions that follow.
4110	Fields.push_back(Elt: llvm::ConstantInt::get(Ty: UnsignedIntLTy, V: RD->getNumBases()));
4111
4112	if (!RD->getNumBases())
4113	return;
4114
4115	// Now add the base class descriptions.
4116
4117	// Itanium C++ ABI 2.9.5p6c:
4118	// __base_info[] is an array of base class descriptions -- one for every
4119	// direct proper base. Each description is of the type:
4120	//
4121	// struct abi::__base_class_type_info {
4122	// public:
4123	// const __class_type_info *__base_type;
4124	// long __offset_flags;
4125	//
4126	// enum __offset_flags_masks {
4127	// __virtual_mask = 0x1,
4128	// __public_mask = 0x2,
4129	// __offset_shift = 8
4130	// };
4131	// };
4132
4133	// If we're in mingw and 'long' isn't wide enough for a pointer, use 'long
4134	// long' instead of 'long' for __offset_flags. libstdc++abi uses long long on
4135	// LLP64 platforms.
4136	// FIXME: Consider updating libc++abi to match, and extend this logic to all
4137	// LLP64 platforms.
4138	QualType OffsetFlagsTy = CGM.getContext().LongTy;
4139	const TargetInfo &TI = CGM.getContext().getTargetInfo();
4140	if (TI.getTriple().isOSCygMing() &&
4141	TI.getPointerWidth(AddrSpace: LangAS::Default) > TI.getLongWidth())
4142	OffsetFlagsTy = CGM.getContext().LongLongTy;
4143	llvm::Type *OffsetFlagsLTy =
4144	CGM.getTypes().ConvertType(T: OffsetFlagsTy);
4145
4146	for (const auto &Base : RD->bases()) {
4147	// The __base_type member points to the RTTI for the base type.
4148	Fields.push_back(Elt: ItaniumRTTIBuilder(CXXABI).BuildTypeInfo(Ty: Base.getType()));
4149
4150	auto *BaseDecl =
4151	cast<CXXRecordDecl>(Val: Base.getType()->castAs<RecordType>()->getDecl());
4152
4153	int64_t OffsetFlags = 0;
4154
4155	// All but the lower 8 bits of __offset_flags are a signed offset.
4156	// For a non-virtual base, this is the offset in the object of the base
4157	// subobject. For a virtual base, this is the offset in the virtual table of
4158	// the virtual base offset for the virtual base referenced (negative).
4159	CharUnits Offset;
4160	if (Base.isVirtual())
4161	Offset =
4162	CGM.getItaniumVTableContext().getVirtualBaseOffsetOffset(RD, VBase: BaseDecl);
4163	else {
4164	const ASTRecordLayout &Layout = CGM.getContext().getASTRecordLayout(RD);
4165	Offset = Layout.getBaseClassOffset(Base: BaseDecl);
4166	};
4167
4168	OffsetFlags = uint64_t(Offset.getQuantity()) << 8;
4169
4170	// The low-order byte of __offset_flags contains flags, as given by the
4171	// masks from the enumeration __offset_flags_masks.
4172	if (Base.isVirtual())
4173	OffsetFlags |= BCTI_Virtual;
4174	if (Base.getAccessSpecifier() == AS_public)
4175	OffsetFlags |= BCTI_Public;
4176
4177	Fields.push_back(Elt: llvm::ConstantInt::get(Ty: OffsetFlagsLTy, V: OffsetFlags));
4178	}
4179	}
4180
4181	/// Compute the flags for a __pbase_type_info, and remove the corresponding
4182	/// pieces from \p Type.
4183	static unsigned extractPBaseFlags(ASTContext &Ctx, QualType &Type) {
4184	unsigned Flags = 0;
4185
4186	if (Type.isConstQualified())
4187	Flags |= ItaniumRTTIBuilder::PTI_Const;
4188	if (Type.isVolatileQualified())
4189	Flags |= ItaniumRTTIBuilder::PTI_Volatile;
4190	if (Type.isRestrictQualified())
4191	Flags |= ItaniumRTTIBuilder::PTI_Restrict;
4192	Type = Type.getUnqualifiedType();
4193
4194	// Itanium C++ ABI 2.9.5p7:
4195	// When the abi::__pbase_type_info is for a direct or indirect pointer to an
4196	// incomplete class type, the incomplete target type flag is set.
4197	if (ContainsIncompleteClassType(Ty: Type))
4198	Flags |= ItaniumRTTIBuilder::PTI_Incomplete;
4199
4200	if (auto *Proto = Type->getAs<FunctionProtoType>()) {
4201	if (Proto->isNothrow()) {
4202	Flags |= ItaniumRTTIBuilder::PTI_Noexcept;
4203	Type = Ctx.getFunctionTypeWithExceptionSpec(Orig: Type, ESI: EST_None);
4204	}
4205	}
4206
4207	return Flags;
4208	}
4209
4210	/// BuildPointerTypeInfo - Build an abi::__pointer_type_info struct,
4211	/// used for pointer types.
4212	void ItaniumRTTIBuilder::BuildPointerTypeInfo(QualType PointeeTy) {
4213	// Itanium C++ ABI 2.9.5p7:
4214	// __flags is a flag word describing the cv-qualification and other
4215	// attributes of the type pointed to
4216	unsigned Flags = extractPBaseFlags(Ctx&: CGM.getContext(), Type&: PointeeTy);
4217
4218	llvm::Type *UnsignedIntLTy =
4219	CGM.getTypes().ConvertType(T: CGM.getContext().UnsignedIntTy);
4220	Fields.push_back(Elt: llvm::ConstantInt::get(Ty: UnsignedIntLTy, V: Flags));
4221
4222	// Itanium C++ ABI 2.9.5p7:
4223	// __pointee is a pointer to the std::type_info derivation for the
4224	// unqualified type being pointed to.
4225	llvm::Constant *PointeeTypeInfo =
4226	ItaniumRTTIBuilder(CXXABI).BuildTypeInfo(Ty: PointeeTy);
4227	Fields.push_back(Elt: PointeeTypeInfo);
4228	}
4229
4230	/// BuildPointerToMemberTypeInfo - Build an abi::__pointer_to_member_type_info
4231	/// struct, used for member pointer types.
4232	void
4233	ItaniumRTTIBuilder::BuildPointerToMemberTypeInfo(const MemberPointerType *Ty) {
4234	QualType PointeeTy = Ty->getPointeeType();
4235
4236	// Itanium C++ ABI 2.9.5p7:
4237	// __flags is a flag word describing the cv-qualification and other
4238	// attributes of the type pointed to.
4239	unsigned Flags = extractPBaseFlags(Ctx&: CGM.getContext(), Type&: PointeeTy);
4240
4241	const RecordType *ClassType = cast<RecordType>(Val: Ty->getClass());
4242	if (IsIncompleteClassType(RecordTy: ClassType))
4243	Flags |= PTI_ContainingClassIncomplete;
4244
4245	llvm::Type *UnsignedIntLTy =
4246	CGM.getTypes().ConvertType(T: CGM.getContext().UnsignedIntTy);
4247	Fields.push_back(Elt: llvm::ConstantInt::get(Ty: UnsignedIntLTy, V: Flags));
4248
4249	// Itanium C++ ABI 2.9.5p7:
4250	// __pointee is a pointer to the std::type_info derivation for the
4251	// unqualified type being pointed to.
4252	llvm::Constant *PointeeTypeInfo =
4253	ItaniumRTTIBuilder(CXXABI).BuildTypeInfo(Ty: PointeeTy);
4254	Fields.push_back(Elt: PointeeTypeInfo);
4255
4256	// Itanium C++ ABI 2.9.5p9:
4257	// __context is a pointer to an abi::__class_type_info corresponding to the
4258	// class type containing the member pointed to
4259	// (e.g., the "A" in "int A::*").
4260	Fields.push_back(
4261	Elt: ItaniumRTTIBuilder(CXXABI).BuildTypeInfo(Ty: QualType(ClassType, 0)));
4262	}
4263
4264	llvm::Constant *ItaniumCXXABI::getAddrOfRTTIDescriptor(QualType Ty) {
4265	return ItaniumRTTIBuilder(*this).BuildTypeInfo(Ty);
4266	}
4267
4268	void ItaniumCXXABI::EmitFundamentalRTTIDescriptors(const CXXRecordDecl *RD) {
4269	// Types added here must also be added to TypeInfoIsInStandardLibrary.
4270	QualType FundamentalTypes[] = {
4271	getContext().VoidTy, getContext().NullPtrTy,
4272	getContext().BoolTy, getContext().WCharTy,
4273	getContext().CharTy, getContext().UnsignedCharTy,
4274	getContext().SignedCharTy, getContext().ShortTy,
4275	getContext().UnsignedShortTy, getContext().IntTy,
4276	getContext().UnsignedIntTy, getContext().LongTy,
4277	getContext().UnsignedLongTy, getContext().LongLongTy,
4278	getContext().UnsignedLongLongTy, getContext().Int128Ty,
4279	getContext().UnsignedInt128Ty, getContext().HalfTy,
4280	getContext().FloatTy, getContext().DoubleTy,
4281	getContext().LongDoubleTy, getContext().Float128Ty,
4282	getContext().Char8Ty, getContext().Char16Ty,
4283	getContext().Char32Ty
4284	};
4285	llvm::GlobalValue::DLLStorageClassTypes DLLStorageClass =
4286	RD->hasAttr<DLLExportAttr>() || CGM.shouldMapVisibilityToDLLExport(RD)
4287	? llvm::GlobalValue::DLLExportStorageClass
4288	: llvm::GlobalValue::DefaultStorageClass;
4289	llvm::GlobalValue::VisibilityTypes Visibility =
4290	CodeGenModule::GetLLVMVisibility(V: RD->getVisibility());
4291	for (const QualType &FundamentalType : FundamentalTypes) {
4292	QualType PointerType = getContext().getPointerType(FundamentalType);
4293	QualType PointerTypeConst = getContext().getPointerType(
4294	FundamentalType.withConst());
4295	for (QualType Type : {FundamentalType, PointerType, PointerTypeConst})
4296	ItaniumRTTIBuilder(*this).BuildTypeInfo(
4297	Type, llvm::GlobalValue::ExternalLinkage,
4298	Visibility, DLLStorageClass);
4299	}
4300	}
4301
4302	/// What sort of uniqueness rules should we use for the RTTI for the
4303	/// given type?
4304	ItaniumCXXABI::RTTIUniquenessKind ItaniumCXXABI::classifyRTTIUniqueness(
4305	QualType CanTy, llvm::GlobalValue::LinkageTypes Linkage) const {
4306	if (shouldRTTIBeUnique())
4307	return RUK_Unique;
4308
4309	// It's only necessary for linkonce_odr or weak_odr linkage.
4310	if (Linkage != llvm::GlobalValue::LinkOnceODRLinkage &&
4311	Linkage != llvm::GlobalValue::WeakODRLinkage)
4312	return RUK_Unique;
4313
4314	// It's only necessary with default visibility.
4315	if (CanTy->getVisibility() != DefaultVisibility)
4316	return RUK_Unique;
4317
4318	// If we're not required to publish this symbol, hide it.
4319	if (Linkage == llvm::GlobalValue::LinkOnceODRLinkage)
4320	return RUK_NonUniqueHidden;
4321
4322	// If we're required to publish this symbol, as we might be under an
4323	// explicit instantiation, leave it with default visibility but
4324	// enable string-comparisons.
4325	assert(Linkage == llvm::GlobalValue::WeakODRLinkage);
4326	return RUK_NonUniqueVisible;
4327	}
4328
4329	// Find out how to codegen the complete destructor and constructor
4330	namespace {
4331	enum class StructorCodegen { Emit, RAUW, Alias, COMDAT };
4332	}
4333	static StructorCodegen getCodegenToUse(CodeGenModule &CGM,
4334	const CXXMethodDecl *MD) {
4335	if (!CGM.getCodeGenOpts().CXXCtorDtorAliases)
4336	return StructorCodegen::Emit;
4337
4338	// The complete and base structors are not equivalent if there are any virtual
4339	// bases, so emit separate functions.
4340	if (MD->getParent()->getNumVBases())
4341	return StructorCodegen::Emit;
4342
4343	GlobalDecl AliasDecl;
4344	if (const auto *DD = dyn_cast<CXXDestructorDecl>(Val: MD)) {
4345	AliasDecl = GlobalDecl(DD, Dtor_Complete);
4346	} else {
4347	const auto *CD = cast<CXXConstructorDecl>(Val: MD);
4348	AliasDecl = GlobalDecl(CD, Ctor_Complete);
4349	}
4350	llvm::GlobalValue::LinkageTypes Linkage = CGM.getFunctionLinkage(GD: AliasDecl);
4351
4352	if (llvm::GlobalValue::isDiscardableIfUnused(Linkage))
4353	return StructorCodegen::RAUW;
4354
4355	// FIXME: Should we allow available_externally aliases?
4356	if (!llvm::GlobalAlias::isValidLinkage(L: Linkage))
4357	return StructorCodegen::RAUW;
4358
4359	if (llvm::GlobalValue::isWeakForLinker(Linkage)) {
4360	// Only ELF and wasm support COMDATs with arbitrary names (C5/D5).
4361	if (CGM.getTarget().getTriple().isOSBinFormatELF() ||
4362	CGM.getTarget().getTriple().isOSBinFormatWasm())
4363	return StructorCodegen::COMDAT;
4364	return StructorCodegen::Emit;
4365	}
4366
4367	return StructorCodegen::Alias;
4368	}
4369
4370	static void emitConstructorDestructorAlias(CodeGenModule &CGM,
4371	GlobalDecl AliasDecl,
4372	GlobalDecl TargetDecl) {
4373	llvm::GlobalValue::LinkageTypes Linkage = CGM.getFunctionLinkage(GD: AliasDecl);
4374
4375	StringRef MangledName = CGM.getMangledName(GD: AliasDecl);
4376	llvm::GlobalValue *Entry = CGM.GetGlobalValue(Ref: MangledName);
4377	if (Entry && !Entry->isDeclaration())
4378	return;
4379
4380	auto *Aliasee = cast<llvm::GlobalValue>(Val: CGM.GetAddrOfGlobal(GD: TargetDecl));
4381
4382	// Create the alias with no name.
4383	auto *Alias = llvm::GlobalAlias::create(Linkage, Name: "", Aliasee);
4384
4385	// Constructors and destructors are always unnamed_addr.
4386	Alias->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
4387
4388	// Switch any previous uses to the alias.
4389	if (Entry) {
4390	assert(Entry->getType() == Aliasee->getType() &&
4391	"declaration exists with different type");
4392	Alias->takeName(V: Entry);
4393	Entry->replaceAllUsesWith(V: Alias);
4394	Entry->eraseFromParent();
4395	} else {
4396	Alias->setName(MangledName);
4397	}
4398
4399	// Finally, set up the alias with its proper name and attributes.
4400	CGM.SetCommonAttributes(GD: AliasDecl, GV: Alias);
4401	}
4402
4403	void ItaniumCXXABI::emitCXXStructor(GlobalDecl GD) {
4404	auto *MD = cast<CXXMethodDecl>(Val: GD.getDecl());
4405	auto *CD = dyn_cast<CXXConstructorDecl>(Val: MD);
4406	const CXXDestructorDecl *DD = CD ? nullptr : cast<CXXDestructorDecl>(Val: MD);
4407
4408	StructorCodegen CGType = getCodegenToUse(CGM, MD);
4409
4410	if (CD ? GD.getCtorType() == Ctor_Complete
4411	: GD.getDtorType() == Dtor_Complete) {
4412	GlobalDecl BaseDecl;
4413	if (CD)
4414	BaseDecl = GD.getWithCtorType(Type: Ctor_Base);
4415	else
4416	BaseDecl = GD.getWithDtorType(Type: Dtor_Base);
4417
4418	if (CGType == StructorCodegen::Alias || CGType == StructorCodegen::COMDAT) {
4419	emitConstructorDestructorAlias(CGM, AliasDecl: GD, TargetDecl: BaseDecl);
4420	return;
4421	}
4422
4423	if (CGType == StructorCodegen::RAUW) {
4424	StringRef MangledName = CGM.getMangledName(GD);
4425	auto *Aliasee = CGM.GetAddrOfGlobal(GD: BaseDecl);
4426	CGM.addReplacement(Name: MangledName, C: Aliasee);
4427	return;
4428	}
4429	}
4430
4431	// The base destructor is equivalent to the base destructor of its
4432	// base class if there is exactly one non-virtual base class with a
4433	// non-trivial destructor, there are no fields with a non-trivial
4434	// destructor, and the body of the destructor is trivial.
4435	if (DD && GD.getDtorType() == Dtor_Base &&
4436	CGType != StructorCodegen::COMDAT &&
4437	!CGM.TryEmitBaseDestructorAsAlias(D: DD))
4438	return;
4439
4440	// FIXME: The deleting destructor is equivalent to the selected operator
4441	// delete if:
4442	// * either the delete is a destroying operator delete or the destructor
4443	// would be trivial if it weren't virtual,
4444	// * the conversion from the 'this' parameter to the first parameter of the
4445	// destructor is equivalent to a bitcast,
4446	// * the destructor does not have an implicit "this" return, and
4447	// * the operator delete has the same calling convention and IR function type
4448	// as the destructor.
4449	// In such cases we should try to emit the deleting dtor as an alias to the
4450	// selected 'operator delete'.
4451
4452	llvm::Function *Fn = CGM.codegenCXXStructor(GD);
4453
4454	if (CGType == StructorCodegen::COMDAT) {
4455	SmallString<256> Buffer;
4456	llvm::raw_svector_ostream Out(Buffer);
4457	if (DD)
4458	getMangleContext().mangleCXXDtorComdat(D: DD, Out);
4459	else
4460	getMangleContext().mangleCXXCtorComdat(D: CD, Out);
4461	llvm::Comdat *C = CGM.getModule().getOrInsertComdat(Name: Out.str());
4462	Fn->setComdat(C);
4463	} else {
4464	CGM.maybeSetTrivialComdat(*MD, *Fn);
4465	}
4466	}
4467
4468	static llvm::FunctionCallee getBeginCatchFn(CodeGenModule &CGM) {
4469	// void *__cxa_begin_catch(void*);
4470	llvm::FunctionType *FTy = llvm::FunctionType::get(
4471	Result: CGM.Int8PtrTy, Params: CGM.Int8PtrTy, /*isVarArg=*/false);
4472
4473	return CGM.CreateRuntimeFunction(Ty: FTy, Name: "__cxa_begin_catch");
4474	}
4475
4476	static llvm::FunctionCallee getEndCatchFn(CodeGenModule &CGM) {
4477	// void __cxa_end_catch();
4478	llvm::FunctionType *FTy =
4479	llvm::FunctionType::get(Result: CGM.VoidTy, /*isVarArg=*/false);
4480
4481	return CGM.CreateRuntimeFunction(Ty: FTy, Name: "__cxa_end_catch");
4482	}
4483
4484	static llvm::FunctionCallee getGetExceptionPtrFn(CodeGenModule &CGM) {
4485	// void *__cxa_get_exception_ptr(void*);
4486	llvm::FunctionType *FTy = llvm::FunctionType::get(
4487	Result: CGM.Int8PtrTy, Params: CGM.Int8PtrTy, /*isVarArg=*/false);
4488
4489	return CGM.CreateRuntimeFunction(Ty: FTy, Name: "__cxa_get_exception_ptr");
4490	}
4491
4492	namespace {
4493	/// A cleanup to call __cxa_end_catch. In many cases, the caught
4494	/// exception type lets us state definitively that the thrown exception
4495	/// type does not have a destructor. In particular:
4496	/// - Catch-alls tell us nothing, so we have to conservatively
4497	/// assume that the thrown exception might have a destructor.
4498	/// - Catches by reference behave according to their base types.
4499	/// - Catches of non-record types will only trigger for exceptions
4500	/// of non-record types, which never have destructors.
4501	/// - Catches of record types can trigger for arbitrary subclasses
4502	/// of the caught type, so we have to assume the actual thrown
4503	/// exception type might have a throwing destructor, even if the
4504	/// caught type's destructor is trivial or nothrow.
4505	struct CallEndCatch final : EHScopeStack::Cleanup {
4506	CallEndCatch(bool MightThrow) : MightThrow(MightThrow) {}
4507	bool MightThrow;
4508
4509	void Emit(CodeGenFunction &CGF, Flags flags) override {
4510	if (!MightThrow) {
4511	CGF.EmitNounwindRuntimeCall(callee: getEndCatchFn(CGM&: CGF.CGM));
4512	return;
4513	}
4514
4515	CGF.EmitRuntimeCallOrInvoke(callee: getEndCatchFn(CGM&: CGF.CGM));
4516	}
4517	};
4518	}
4519
4520	/// Emits a call to __cxa_begin_catch and enters a cleanup to call
4521	/// __cxa_end_catch. If -fassume-nothrow-exception-dtor is specified, we assume
4522	/// that the exception object's dtor is nothrow, therefore the __cxa_end_catch
4523	/// call can be marked as nounwind even if EndMightThrow is true.
4524	///
4525	/// \param EndMightThrow - true if __cxa_end_catch might throw
4526	static llvm::Value *CallBeginCatch(CodeGenFunction &CGF,
4527	llvm::Value *Exn,
4528	bool EndMightThrow) {
4529	llvm::CallInst *call =
4530	CGF.EmitNounwindRuntimeCall(callee: getBeginCatchFn(CGM&: CGF.CGM), args: Exn);
4531
4532	CGF.EHStack.pushCleanup<CallEndCatch>(
4533	Kind: NormalAndEHCleanup,
4534	A: EndMightThrow && !CGF.CGM.getLangOpts().AssumeNothrowExceptionDtor);
4535
4536	return call;
4537	}
4538
4539	/// A "special initializer" callback for initializing a catch
4540	/// parameter during catch initialization.
4541	static void InitCatchParam(CodeGenFunction &CGF,
4542	const VarDecl &CatchParam,
4543	Address ParamAddr,
4544	SourceLocation Loc) {
4545	// Load the exception from where the landing pad saved it.
4546	llvm::Value *Exn = CGF.getExceptionFromSlot();
4547
4548	CanQualType CatchType =
4549	CGF.CGM.getContext().getCanonicalType(CatchParam.getType());
4550	llvm::Type *LLVMCatchTy = CGF.ConvertTypeForMem(T: CatchType);
4551
4552	// If we're catching by reference, we can just cast the object
4553	// pointer to the appropriate pointer.
4554	if (isa<ReferenceType>(Val: CatchType)) {
4555	QualType CaughtType = cast<ReferenceType>(Val&: CatchType)->getPointeeType();
4556	bool EndCatchMightThrow = CaughtType->isRecordType();
4557
4558	// __cxa_begin_catch returns the adjusted object pointer.
4559	llvm::Value *AdjustedExn = CallBeginCatch(CGF, Exn, EndMightThrow: EndCatchMightThrow);
4560
4561	// We have no way to tell the personality function that we're
4562	// catching by reference, so if we're catching a pointer,
4563	// __cxa_begin_catch will actually return that pointer by value.
4564	if (const PointerType *PT = dyn_cast<PointerType>(Val&: CaughtType)) {
4565	QualType PointeeType = PT->getPointeeType();
4566
4567	// When catching by reference, generally we should just ignore
4568	// this by-value pointer and use the exception object instead.
4569	if (!PointeeType->isRecordType()) {
4570
4571	// Exn points to the struct _Unwind_Exception header, which
4572	// we have to skip past in order to reach the exception data.
4573	unsigned HeaderSize =
4574	CGF.CGM.getTargetCodeGenInfo().getSizeOfUnwindException();
4575	AdjustedExn =
4576	CGF.Builder.CreateConstGEP1_32(Ty: CGF.Int8Ty, Ptr: Exn, Idx0: HeaderSize);
4577
4578	// However, if we're catching a pointer-to-record type that won't
4579	// work, because the personality function might have adjusted
4580	// the pointer. There's actually no way for us to fully satisfy
4581	// the language/ABI contract here: we can't use Exn because it
4582	// might have the wrong adjustment, but we can't use the by-value
4583	// pointer because it's off by a level of abstraction.
4584	//
4585	// The current solution is to dump the adjusted pointer into an
4586	// alloca, which breaks language semantics (because changing the
4587	// pointer doesn't change the exception) but at least works.
4588	// The better solution would be to filter out non-exact matches
4589	// and rethrow them, but this is tricky because the rethrow
4590	// really needs to be catchable by other sites at this landing
4591	// pad. The best solution is to fix the personality function.
4592	} else {
4593	// Pull the pointer for the reference type off.
4594	llvm::Type *PtrTy = CGF.ConvertTypeForMem(T: CaughtType);
4595
4596	// Create the temporary and write the adjusted pointer into it.
4597	Address ExnPtrTmp =
4598	CGF.CreateTempAlloca(PtrTy, CGF.getPointerAlign(), "exn.byref.tmp");
4599	llvm::Value *Casted = CGF.Builder.CreateBitCast(V: AdjustedExn, DestTy: PtrTy);
4600	CGF.Builder.CreateStore(Val: Casted, Addr: ExnPtrTmp);
4601
4602	// Bind the reference to the temporary.
4603	AdjustedExn = ExnPtrTmp.emitRawPointer(CGF);
4604	}
4605	}
4606
4607	llvm::Value *ExnCast =
4608	CGF.Builder.CreateBitCast(V: AdjustedExn, DestTy: LLVMCatchTy, Name: "exn.byref");
4609	CGF.Builder.CreateStore(Val: ExnCast, Addr: ParamAddr);
4610	return;
4611	}
4612
4613	// Scalars and complexes.
4614	TypeEvaluationKind TEK = CGF.getEvaluationKind(T: CatchType);
4615	if (TEK != TEK_Aggregate) {
4616	llvm::Value *AdjustedExn = CallBeginCatch(CGF, Exn, EndMightThrow: false);
4617
4618	// If the catch type is a pointer type, __cxa_begin_catch returns
4619	// the pointer by value.
4620	if (CatchType->hasPointerRepresentation()) {
4621	llvm::Value *CastExn =
4622	CGF.Builder.CreateBitCast(V: AdjustedExn, DestTy: LLVMCatchTy, Name: "exn.casted");
4623
4624	switch (CatchType.getQualifiers().getObjCLifetime()) {
4625	case Qualifiers::OCL_Strong:
4626	CastExn = CGF.EmitARCRetainNonBlock(value: CastExn);
4627	[[fallthrough]];
4628
4629	case Qualifiers::OCL_None:
4630	case Qualifiers::OCL_ExplicitNone:
4631	case Qualifiers::OCL_Autoreleasing:
4632	CGF.Builder.CreateStore(Val: CastExn, Addr: ParamAddr);
4633	return;
4634
4635	case Qualifiers::OCL_Weak:
4636	CGF.EmitARCInitWeak(addr: ParamAddr, value: CastExn);
4637	return;
4638	}
4639	llvm_unreachable("bad ownership qualifier!");
4640	}
4641
4642	// Otherwise, it returns a pointer into the exception object.
4643
4644	LValue srcLV = CGF.MakeNaturalAlignAddrLValue(V: AdjustedExn, T: CatchType);
4645	LValue destLV = CGF.MakeAddrLValue(Addr: ParamAddr, T: CatchType);
4646	switch (TEK) {
4647	case TEK_Complex:
4648	CGF.EmitStoreOfComplex(V: CGF.EmitLoadOfComplex(src: srcLV, loc: Loc), dest: destLV,
4649	/*init*/ isInit: true);
4650	return;
4651	case TEK_Scalar: {
4652	llvm::Value *ExnLoad = CGF.EmitLoadOfScalar(lvalue: srcLV, Loc);
4653	CGF.EmitStoreOfScalar(value: ExnLoad, lvalue: destLV, /*init*/ isInit: true);
4654	return;
4655	}
4656	case TEK_Aggregate:
4657	llvm_unreachable("evaluation kind filtered out!");
4658	}
4659	llvm_unreachable("bad evaluation kind");
4660	}
4661
4662	assert(isa<RecordType>(CatchType) && "unexpected catch type!");
4663	auto catchRD = CatchType->getAsCXXRecordDecl();
4664	CharUnits caughtExnAlignment = CGF.CGM.getClassPointerAlignment(CD: catchRD);
4665
4666	llvm::Type *PtrTy = CGF.UnqualPtrTy; // addrspace 0 ok
4667
4668	// Check for a copy expression. If we don't have a copy expression,
4669	// that means a trivial copy is okay.
4670	const Expr *copyExpr = CatchParam.getInit();
4671	if (!copyExpr) {
4672	llvm::Value *rawAdjustedExn = CallBeginCatch(CGF, Exn, EndMightThrow: true);
4673	Address adjustedExn(CGF.Builder.CreateBitCast(V: rawAdjustedExn, DestTy: PtrTy),
4674	LLVMCatchTy, caughtExnAlignment);
4675	LValue Dest = CGF.MakeAddrLValue(Addr: ParamAddr, T: CatchType);
4676	LValue Src = CGF.MakeAddrLValue(Addr: adjustedExn, T: CatchType);
4677	CGF.EmitAggregateCopy(Dest, Src, EltTy: CatchType, MayOverlap: AggValueSlot::DoesNotOverlap);
4678	return;
4679	}
4680
4681	// We have to call __cxa_get_exception_ptr to get the adjusted
4682	// pointer before copying.
4683	llvm::CallInst *rawAdjustedExn =
4684	CGF.EmitNounwindRuntimeCall(callee: getGetExceptionPtrFn(CGM&: CGF.CGM), args: Exn);
4685
4686	// Cast that to the appropriate type.
4687	Address adjustedExn(CGF.Builder.CreateBitCast(V: rawAdjustedExn, DestTy: PtrTy),
4688	LLVMCatchTy, caughtExnAlignment);
4689
4690	// The copy expression is defined in terms of an OpaqueValueExpr.
4691	// Find it and map it to the adjusted expression.
4692	CodeGenFunction::OpaqueValueMapping
4693	opaque(CGF, OpaqueValueExpr::findInCopyConstruct(expr: copyExpr),
4694	CGF.MakeAddrLValue(adjustedExn, CatchParam.getType()));
4695
4696	// Call the copy ctor in a terminate scope.
4697	CGF.EHStack.pushTerminate();
4698
4699	// Perform the copy construction.
4700	CGF.EmitAggExpr(E: copyExpr,
4701	AS: AggValueSlot::forAddr(addr: ParamAddr, quals: Qualifiers(),
4702	isDestructed: AggValueSlot::IsNotDestructed,
4703	needsGC: AggValueSlot::DoesNotNeedGCBarriers,
4704	isAliased: AggValueSlot::IsNotAliased,
4705	mayOverlap: AggValueSlot::DoesNotOverlap));
4706
4707	// Leave the terminate scope.
4708	CGF.EHStack.popTerminate();
4709
4710	// Undo the opaque value mapping.
4711	opaque.pop();
4712
4713	// Finally we can call __cxa_begin_catch.
4714	CallBeginCatch(CGF, Exn, EndMightThrow: true);
4715	}
4716
4717	/// Begins a catch statement by initializing the catch variable and
4718	/// calling __cxa_begin_catch.
4719	void ItaniumCXXABI::emitBeginCatch(CodeGenFunction &CGF,
4720	const CXXCatchStmt *S) {
4721	// We have to be very careful with the ordering of cleanups here:
4722	// C++ [except.throw]p4:
4723	// The destruction [of the exception temporary] occurs
4724	// immediately after the destruction of the object declared in
4725	// the exception-declaration in the handler.
4726	//
4727	// So the precise ordering is:
4728	// 1. Construct catch variable.
4729	// 2. __cxa_begin_catch
4730	// 3. Enter __cxa_end_catch cleanup
4731	// 4. Enter dtor cleanup
4732	//
4733	// We do this by using a slightly abnormal initialization process.
4734	// Delegation sequence:
4735	// - ExitCXXTryStmt opens a RunCleanupsScope
4736	// - EmitAutoVarAlloca creates the variable and debug info
4737	// - InitCatchParam initializes the variable from the exception
4738	// - CallBeginCatch calls __cxa_begin_catch
4739	// - CallBeginCatch enters the __cxa_end_catch cleanup
4740	// - EmitAutoVarCleanups enters the variable destructor cleanup
4741	// - EmitCXXTryStmt emits the code for the catch body
4742	// - EmitCXXTryStmt close the RunCleanupsScope
4743
4744	VarDecl *CatchParam = S->getExceptionDecl();
4745	if (!CatchParam) {
4746	llvm::Value *Exn = CGF.getExceptionFromSlot();
4747	CallBeginCatch(CGF, Exn, EndMightThrow: true);
4748	return;
4749	}
4750
4751	// Emit the local.
4752	CodeGenFunction::AutoVarEmission var = CGF.EmitAutoVarAlloca(var: *CatchParam);
4753	InitCatchParam(CGF, CatchParam: *CatchParam, ParamAddr: var.getObjectAddress(CGF), Loc: S->getBeginLoc());
4754	CGF.EmitAutoVarCleanups(emission: var);
4755	}
4756
4757	/// Get or define the following function:
4758	/// void @__clang_call_terminate(i8* %exn) nounwind noreturn
4759	/// This code is used only in C++.
4760	static llvm::FunctionCallee getClangCallTerminateFn(CodeGenModule &CGM) {
4761	ASTContext &C = CGM.getContext();
4762	const CGFunctionInfo &FI = CGM.getTypes().arrangeBuiltinFunctionDeclaration(
4763	C.VoidTy, {C.getPointerType(C.CharTy)});
4764	llvm::FunctionType *fnTy = CGM.getTypes().GetFunctionType(Info: FI);
4765	llvm::FunctionCallee fnRef = CGM.CreateRuntimeFunction(
4766	Ty: fnTy, Name: "__clang_call_terminate", ExtraAttrs: llvm::AttributeList(), /*Local=*/true);
4767	llvm::Function *fn =
4768	cast<llvm::Function>(Val: fnRef.getCallee()->stripPointerCasts());
4769	if (fn->empty()) {
4770	CGM.SetLLVMFunctionAttributes(GD: GlobalDecl(), Info: FI, F: fn, /*IsThunk=*/false);
4771	CGM.SetLLVMFunctionAttributesForDefinition(D: nullptr, F: fn);
4772	fn->setDoesNotThrow();
4773	fn->setDoesNotReturn();
4774
4775	// What we really want is to massively penalize inlining without
4776	// forbidding it completely. The difference between that and
4777	// 'noinline' is negligible.
4778	fn->addFnAttr(llvm::Attribute::NoInline);
4779
4780	// Allow this function to be shared across translation units, but
4781	// we don't want it to turn into an exported symbol.
4782	fn->setLinkage(llvm::Function::LinkOnceODRLinkage);
4783	fn->setVisibility(llvm::Function::HiddenVisibility);
4784	if (CGM.supportsCOMDAT())
4785	fn->setComdat(CGM.getModule().getOrInsertComdat(Name: fn->getName()));
4786
4787	// Set up the function.
4788	llvm::BasicBlock *entry =
4789	llvm::BasicBlock::Create(Context&: CGM.getLLVMContext(), Name: "", Parent: fn);
4790	CGBuilderTy builder(CGM, entry);
4791
4792	// Pull the exception pointer out of the parameter list.
4793	llvm::Value *exn = &*fn->arg_begin();
4794
4795	// Call __cxa_begin_catch(exn).
4796	llvm::CallInst *catchCall = builder.CreateCall(Callee: getBeginCatchFn(CGM), Args: exn);
4797	catchCall->setDoesNotThrow();
4798	catchCall->setCallingConv(CGM.getRuntimeCC());
4799
4800	// Call std::terminate().
4801	llvm::CallInst *termCall = builder.CreateCall(Callee: CGM.getTerminateFn());
4802	termCall->setDoesNotThrow();
4803	termCall->setDoesNotReturn();
4804	termCall->setCallingConv(CGM.getRuntimeCC());
4805
4806	// std::terminate cannot return.
4807	builder.CreateUnreachable();
4808	}
4809	return fnRef;
4810	}
4811
4812	llvm::CallInst *
4813	ItaniumCXXABI::emitTerminateForUnexpectedException(CodeGenFunction &CGF,
4814	llvm::Value *Exn) {
4815	// In C++, we want to call __cxa_begin_catch() before terminating.
4816	if (Exn) {
4817	assert(CGF.CGM.getLangOpts().CPlusPlus);
4818	return CGF.EmitNounwindRuntimeCall(callee: getClangCallTerminateFn(CGM&: CGF.CGM), args: Exn);
4819	}
4820	return CGF.EmitNounwindRuntimeCall(callee: CGF.CGM.getTerminateFn());
4821	}
4822
4823	std::pair<llvm::Value *, const CXXRecordDecl *>
4824	ItaniumCXXABI::LoadVTablePtr(CodeGenFunction &CGF, Address This,
4825	const CXXRecordDecl *RD) {
4826	return {CGF.GetVTablePtr(This, VTableTy: CGM.Int8PtrTy, VTableClass: RD), RD};
4827	}
4828
4829	void WebAssemblyCXXABI::emitBeginCatch(CodeGenFunction &CGF,
4830	const CXXCatchStmt *C) {
4831	if (CGF.getTarget().hasFeature(Feature: "exception-handling"))
4832	CGF.EHStack.pushCleanup<CatchRetScope>(
4833	Kind: NormalCleanup, A: cast<llvm::CatchPadInst>(Val: CGF.CurrentFuncletPad));
4834	ItaniumCXXABI::emitBeginCatch(CGF, S: C);
4835	}
4836
4837	llvm::CallInst *
4838	WebAssemblyCXXABI::emitTerminateForUnexpectedException(CodeGenFunction &CGF,
4839	llvm::Value *Exn) {
4840	// Itanium ABI calls __clang_call_terminate(), which __cxa_begin_catch() on
4841	// the violating exception to mark it handled, but it is currently hard to do
4842	// with wasm EH instruction structure with catch/catch_all, we just call
4843	// std::terminate and ignore the violating exception as in CGCXXABI.
4844	// TODO Consider code transformation that makes calling __clang_call_terminate
4845	// possible.
4846	return CGCXXABI::emitTerminateForUnexpectedException(CGF, Exn);
4847	}
4848
4849	/// Register a global destructor as best as we know how.
4850	void XLCXXABI::registerGlobalDtor(CodeGenFunction &CGF, const VarDecl &D,
4851	llvm::FunctionCallee Dtor,
4852	llvm::Constant *Addr) {
4853	if (D.getTLSKind() != VarDecl::TLS_None) {
4854	llvm::PointerType *PtrTy = CGF.UnqualPtrTy;
4855
4856	// extern "C" int __pt_atexit_np(int flags, int(*)(int,...), ...);
4857	llvm::FunctionType *AtExitTy =
4858	llvm::FunctionType::get(Result: CGM.IntTy, Params: {CGM.IntTy, PtrTy}, isVarArg: true);
4859
4860	// Fetch the actual function.
4861	llvm::FunctionCallee AtExit =
4862	CGM.CreateRuntimeFunction(Ty: AtExitTy, Name: "__pt_atexit_np");
4863
4864	// Create __dtor function for the var decl.
4865	llvm::Function *DtorStub = CGF.createTLSAtExitStub(VD: D, Dtor, Addr, AtExit);
4866
4867	// Register above __dtor with atexit().
4868	// First param is flags and must be 0, second param is function ptr
4869	llvm::Value *NV = llvm::Constant::getNullValue(Ty: CGM.IntTy);
4870	CGF.EmitNounwindRuntimeCall(callee: AtExit, args: {NV, DtorStub});
4871
4872	// Cannot unregister TLS __dtor so done
4873	return;
4874	}
4875
4876	// Create __dtor function for the var decl.
4877	llvm::Function *DtorStub = CGF.createAtExitStub(VD: D, Dtor, Addr);
4878
4879	// Register above __dtor with atexit().
4880	CGF.registerGlobalDtorWithAtExit(dtorStub: DtorStub);
4881
4882	// Emit __finalize function to unregister __dtor and (as appropriate) call
4883	// __dtor.
4884	emitCXXStermFinalizer(D, dtorStub: DtorStub, addr: Addr);
4885	}
4886
4887	void XLCXXABI::emitCXXStermFinalizer(const VarDecl &D, llvm::Function *dtorStub,
4888	llvm::Constant *addr) {
4889	llvm::FunctionType *FTy = llvm::FunctionType::get(Result: CGM.VoidTy, isVarArg: false);
4890	SmallString<256> FnName;
4891	{
4892	llvm::raw_svector_ostream Out(FnName);
4893	getMangleContext().mangleDynamicStermFinalizer(D: &D, Out);
4894	}
4895
4896	// Create the finalization action associated with a variable.
4897	const CGFunctionInfo &FI = CGM.getTypes().arrangeNullaryFunction();
4898	llvm::Function *StermFinalizer = CGM.CreateGlobalInitOrCleanUpFunction(
4899	ty: FTy, name: FnName.str(), FI, Loc: D.getLocation());
4900
4901	CodeGenFunction CGF(CGM);
4902
4903	CGF.StartFunction(GD: GlobalDecl(), RetTy: CGM.getContext().VoidTy, Fn: StermFinalizer, FnInfo: FI,
4904	Args: FunctionArgList(), Loc: D.getLocation(),
4905	StartLoc: D.getInit()->getExprLoc());
4906
4907	// The unatexit subroutine unregisters __dtor functions that were previously
4908	// registered by the atexit subroutine. If the referenced function is found,
4909	// the unatexit returns a value of 0, meaning that the cleanup is still
4910	// pending (and we should call the __dtor function).
4911	llvm::Value *V = CGF.unregisterGlobalDtorWithUnAtExit(dtorStub);
4912
4913	llvm::Value *NeedsDestruct = CGF.Builder.CreateIsNull(Arg: V, Name: "needs_destruct");
4914
4915	llvm::BasicBlock *DestructCallBlock = CGF.createBasicBlock(name: "destruct.call");
4916	llvm::BasicBlock *EndBlock = CGF.createBasicBlock(name: "destruct.end");
4917
4918	// Check if unatexit returns a value of 0. If it does, jump to
4919	// DestructCallBlock, otherwise jump to EndBlock directly.
4920	CGF.Builder.CreateCondBr(Cond: NeedsDestruct, True: DestructCallBlock, False: EndBlock);
4921
4922	CGF.EmitBlock(BB: DestructCallBlock);
4923
4924	// Emit the call to dtorStub.
4925	llvm::CallInst *CI = CGF.Builder.CreateCall(Callee: dtorStub);
4926
4927	// Make sure the call and the callee agree on calling convention.
4928	CI->setCallingConv(dtorStub->getCallingConv());
4929
4930	CGF.EmitBlock(BB: EndBlock);
4931
4932	CGF.FinishFunction();
4933
4934	if (auto *IPA = D.getAttr<InitPriorityAttr>()) {
4935	CGM.AddCXXPrioritizedStermFinalizerEntry(StermFinalizer,
4936	Priority: IPA->getPriority());
4937	} else if (isTemplateInstantiation(Kind: D.getTemplateSpecializationKind()) ||
4938	getContext().GetGVALinkageForVariable(VD: &D) == GVA_DiscardableODR) {
4939	// According to C++ [basic.start.init]p2, class template static data
4940	// members (i.e., implicitly or explicitly instantiated specializations)
4941	// have unordered initialization. As a consequence, we can put them into
4942	// their own llvm.global_dtors entry.
4943	CGM.AddCXXStermFinalizerToGlobalDtor(StermFinalizer, Priority: 65535);
4944	} else {
4945	CGM.AddCXXStermFinalizerEntry(DtorFn: StermFinalizer);
4946	}
4947	}
4948