1	//===--- CodeGenModule.cpp - Emit LLVM Code from ASTs for a Module --------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This coordinates the per-module state used while generating code.
10	//
11	//===----------------------------------------------------------------------===//
12
13	#include "CodeGenModule.h"
14	#include "ABIInfo.h"
15	#include "CGBlocks.h"
16	#include "CGCUDARuntime.h"
17	#include "CGCXXABI.h"
18	#include "CGCall.h"
19	#include "CGDebugInfo.h"
20	#include "CGHLSLRuntime.h"
21	#include "CGObjCRuntime.h"
22	#include "CGOpenCLRuntime.h"
23	#include "CGOpenMPRuntime.h"
24	#include "CGOpenMPRuntimeGPU.h"
25	#include "CodeGenFunction.h"
26	#include "CodeGenPGO.h"
27	#include "ConstantEmitter.h"
28	#include "CoverageMappingGen.h"
29	#include "TargetInfo.h"
30	#include "clang/AST/ASTContext.h"
31	#include "clang/AST/ASTLambda.h"
32	#include "clang/AST/CharUnits.h"
33	#include "clang/AST/Decl.h"
34	#include "clang/AST/DeclCXX.h"
35	#include "clang/AST/DeclObjC.h"
36	#include "clang/AST/DeclTemplate.h"
37	#include "clang/AST/Mangle.h"
38	#include "clang/AST/RecursiveASTVisitor.h"
39	#include "clang/AST/StmtVisitor.h"
40	#include "clang/Basic/Builtins.h"
41	#include "clang/Basic/CodeGenOptions.h"
42	#include "clang/Basic/Diagnostic.h"
43	#include "clang/Basic/Module.h"
44	#include "clang/Basic/SourceManager.h"
45	#include "clang/Basic/TargetInfo.h"
46	#include "clang/Basic/Version.h"
47	#include "clang/CodeGen/BackendUtil.h"
48	#include "clang/CodeGen/ConstantInitBuilder.h"
49	#include "clang/Frontend/FrontendDiagnostic.h"
50	#include "llvm/ADT/STLExtras.h"
51	#include "llvm/ADT/StringExtras.h"
52	#include "llvm/ADT/StringSwitch.h"
53	#include "llvm/Analysis/TargetLibraryInfo.h"
54	#include "llvm/BinaryFormat/ELF.h"
55	#include "llvm/IR/AttributeMask.h"
56	#include "llvm/IR/CallingConv.h"
57	#include "llvm/IR/DataLayout.h"
58	#include "llvm/IR/Intrinsics.h"
59	#include "llvm/IR/LLVMContext.h"
60	#include "llvm/IR/Module.h"
61	#include "llvm/IR/ProfileSummary.h"
62	#include "llvm/ProfileData/InstrProfReader.h"
63	#include "llvm/ProfileData/SampleProf.h"
64	#include "llvm/Support/CRC.h"
65	#include "llvm/Support/CodeGen.h"
66	#include "llvm/Support/CommandLine.h"
67	#include "llvm/Support/ConvertUTF.h"
68	#include "llvm/Support/ErrorHandling.h"
69	#include "llvm/Support/TimeProfiler.h"
70	#include "llvm/Support/xxhash.h"
71	#include "llvm/TargetParser/RISCVISAInfo.h"
72	#include "llvm/TargetParser/Triple.h"
73	#include "llvm/TargetParser/X86TargetParser.h"
74	#include "llvm/Transforms/Utils/BuildLibCalls.h"
75	#include <optional>
76	#include <set>
77
78	using namespace clang;
79	using namespace CodeGen;
80
81	static llvm::cl::opt<bool> LimitedCoverage(
82	"limited-coverage-experimental", llvm::cl::Hidden,
83	llvm::cl::desc("Emit limited coverage mapping information (experimental)"));
84
85	static const char AnnotationSection[] = "llvm.metadata";
86
87	static CGCXXABI *createCXXABI(CodeGenModule &CGM) {
88	switch (CGM.getContext().getCXXABIKind()) {
89	case TargetCXXABI::AppleARM64:
90	case TargetCXXABI::Fuchsia:
91	case TargetCXXABI::GenericAArch64:
92	case TargetCXXABI::GenericARM:
93	case TargetCXXABI::iOS:
94	case TargetCXXABI::WatchOS:
95	case TargetCXXABI::GenericMIPS:
96	case TargetCXXABI::GenericItanium:
97	case TargetCXXABI::WebAssembly:
98	case TargetCXXABI::XL:
99	return CreateItaniumCXXABI(CGM);
100	case TargetCXXABI::Microsoft:
101	return CreateMicrosoftCXXABI(CGM);
102	}
103
104	llvm_unreachable("invalid C++ ABI kind");
105	}
106
107	static std::unique_ptr<TargetCodeGenInfo>
108	createTargetCodeGenInfo(CodeGenModule &CGM) {
109	const TargetInfo &Target = CGM.getTarget();
110	const llvm::Triple &Triple = Target.getTriple();
111	const CodeGenOptions &CodeGenOpts = CGM.getCodeGenOpts();
112
113	switch (Triple.getArch()) {
114	default:
115	return createDefaultTargetCodeGenInfo(CGM);
116
117	case llvm::Triple::m68k:
118	return createM68kTargetCodeGenInfo(CGM);
119	case llvm::Triple::mips:
120	case llvm::Triple::mipsel:
121	if (Triple.getOS() == llvm::Triple::NaCl)
122	return createPNaClTargetCodeGenInfo(CGM);
123	else if (Triple.getOS() == llvm::Triple::Win32)
124	return createWindowsMIPSTargetCodeGenInfo(CGM, /*IsOS32=*/true);
125	return createMIPSTargetCodeGenInfo(CGM, /*IsOS32=*/true);
126
127	case llvm::Triple::mips64:
128	case llvm::Triple::mips64el:
129	return createMIPSTargetCodeGenInfo(CGM, /*IsOS32=*/false);
130
131	case llvm::Triple::avr: {
132	// For passing parameters, R8~R25 are used on avr, and R18~R25 are used
133	// on avrtiny. For passing return value, R18~R25 are used on avr, and
134	// R22~R25 are used on avrtiny.
135	unsigned NPR = Target.getABI() == "avrtiny" ? 6 : 18;
136	unsigned NRR = Target.getABI() == "avrtiny" ? 4 : 8;
137	return createAVRTargetCodeGenInfo(CGM, NPR, NRR);
138	}
139
140	case llvm::Triple::aarch64:
141	case llvm::Triple::aarch64_32:
142	case llvm::Triple::aarch64_be: {
143	AArch64ABIKind Kind = AArch64ABIKind::AAPCS;
144	if (Target.getABI() == "darwinpcs")
145	Kind = AArch64ABIKind::DarwinPCS;
146	else if (Triple.isOSWindows())
147	return createWindowsAArch64TargetCodeGenInfo(CGM, K: AArch64ABIKind::Win64);
148	else if (Target.getABI() == "aapcs-soft")
149	Kind = AArch64ABIKind::AAPCSSoft;
150	else if (Target.getABI() == "pauthtest")
151	Kind = AArch64ABIKind::PAuthTest;
152
153	return createAArch64TargetCodeGenInfo(CGM, Kind);
154	}
155
156	case llvm::Triple::wasm32:
157	case llvm::Triple::wasm64: {
158	WebAssemblyABIKind Kind = WebAssemblyABIKind::MVP;
159	if (Target.getABI() == "experimental-mv")
160	Kind = WebAssemblyABIKind::ExperimentalMV;
161	return createWebAssemblyTargetCodeGenInfo(CGM, K: Kind);
162	}
163
164	case llvm::Triple::arm:
165	case llvm::Triple::armeb:
166	case llvm::Triple::thumb:
167	case llvm::Triple::thumbeb: {
168	if (Triple.getOS() == llvm::Triple::Win32)
169	return createWindowsARMTargetCodeGenInfo(CGM, K: ARMABIKind::AAPCS_VFP);
170
171	ARMABIKind Kind = ARMABIKind::AAPCS;
172	StringRef ABIStr = Target.getABI();
173	if (ABIStr == "apcs-gnu")
174	Kind = ARMABIKind::APCS;
175	else if (ABIStr == "aapcs16")
176	Kind = ARMABIKind::AAPCS16_VFP;
177	else if (CodeGenOpts.FloatABI == "hard" ||
178	(CodeGenOpts.FloatABI != "soft" && Triple.isHardFloatABI()))
179	Kind = ARMABIKind::AAPCS_VFP;
180
181	return createARMTargetCodeGenInfo(CGM, Kind);
182	}
183
184	case llvm::Triple::ppc: {
185	if (Triple.isOSAIX())
186	return createAIXTargetCodeGenInfo(CGM, /*Is64Bit=*/false);
187
188	bool IsSoftFloat =
189	CodeGenOpts.FloatABI == "soft" || Target.hasFeature(Feature: "spe");
190	return createPPC32TargetCodeGenInfo(CGM, SoftFloatABI: IsSoftFloat);
191	}
192	case llvm::Triple::ppcle: {
193	bool IsSoftFloat = CodeGenOpts.FloatABI == "soft";
194	return createPPC32TargetCodeGenInfo(CGM, SoftFloatABI: IsSoftFloat);
195	}
196	case llvm::Triple::ppc64:
197	if (Triple.isOSAIX())
198	return createAIXTargetCodeGenInfo(CGM, /*Is64Bit=*/true);
199
200	if (Triple.isOSBinFormatELF()) {
201	PPC64_SVR4_ABIKind Kind = PPC64_SVR4_ABIKind::ELFv1;
202	if (Target.getABI() == "elfv2")
203	Kind = PPC64_SVR4_ABIKind::ELFv2;
204	bool IsSoftFloat = CodeGenOpts.FloatABI == "soft";
205
206	return createPPC64_SVR4_TargetCodeGenInfo(CGM, Kind, SoftFloatABI: IsSoftFloat);
207	}
208	return createPPC64TargetCodeGenInfo(CGM);
209	case llvm::Triple::ppc64le: {
210	assert(Triple.isOSBinFormatELF() && "PPC64 LE non-ELF not supported!");
211	PPC64_SVR4_ABIKind Kind = PPC64_SVR4_ABIKind::ELFv2;
212	if (Target.getABI() == "elfv1")
213	Kind = PPC64_SVR4_ABIKind::ELFv1;
214	bool IsSoftFloat = CodeGenOpts.FloatABI == "soft";
215
216	return createPPC64_SVR4_TargetCodeGenInfo(CGM, Kind, SoftFloatABI: IsSoftFloat);
217	}
218
219	case llvm::Triple::nvptx:
220	case llvm::Triple::nvptx64:
221	return createNVPTXTargetCodeGenInfo(CGM);
222
223	case llvm::Triple::msp430:
224	return createMSP430TargetCodeGenInfo(CGM);
225
226	case llvm::Triple::riscv32:
227	case llvm::Triple::riscv64: {
228	StringRef ABIStr = Target.getABI();
229	unsigned XLen = Target.getPointerWidth(AddrSpace: LangAS::Default);
230	unsigned ABIFLen = 0;
231	if (ABIStr.ends_with(Suffix: "f"))
232	ABIFLen = 32;
233	else if (ABIStr.ends_with(Suffix: "d"))
234	ABIFLen = 64;
235	bool EABI = ABIStr.ends_with(Suffix: "e");
236	return createRISCVTargetCodeGenInfo(CGM, XLen, FLen: ABIFLen, EABI);
237	}
238
239	case llvm::Triple::systemz: {
240	bool SoftFloat = CodeGenOpts.FloatABI == "soft";
241	bool HasVector = !SoftFloat && Target.getABI() == "vector";
242	return createSystemZTargetCodeGenInfo(CGM, HasVector, SoftFloatABI: SoftFloat);
243	}
244
245	case llvm::Triple::tce:
246	case llvm::Triple::tcele:
247	return createTCETargetCodeGenInfo(CGM);
248
249	case llvm::Triple::x86: {
250	bool IsDarwinVectorABI = Triple.isOSDarwin();
251	bool IsWin32FloatStructABI = Triple.isOSWindows() && !Triple.isOSCygMing();
252
253	if (Triple.getOS() == llvm::Triple::Win32) {
254	return createWinX86_32TargetCodeGenInfo(
255	CGM, DarwinVectorABI: IsDarwinVectorABI, Win32StructABI: IsWin32FloatStructABI,
256	NumRegisterParameters: CodeGenOpts.NumRegisterParameters);
257	}
258	return createX86_32TargetCodeGenInfo(
259	CGM, DarwinVectorABI: IsDarwinVectorABI, Win32StructABI: IsWin32FloatStructABI,
260	NumRegisterParameters: CodeGenOpts.NumRegisterParameters, SoftFloatABI: CodeGenOpts.FloatABI == "soft");
261	}
262
263	case llvm::Triple::x86_64: {
264	StringRef ABI = Target.getABI();
265	X86AVXABILevel AVXLevel = (ABI == "avx512" ? X86AVXABILevel::AVX512
266	: ABI == "avx" ? X86AVXABILevel::AVX
267	: X86AVXABILevel::None);
268
269	switch (Triple.getOS()) {
270	case llvm::Triple::UEFI:
271	case llvm::Triple::Win32:
272	return createWinX86_64TargetCodeGenInfo(CGM, AVXLevel);
273	default:
274	return createX86_64TargetCodeGenInfo(CGM, AVXLevel);
275	}
276	}
277	case llvm::Triple::hexagon:
278	return createHexagonTargetCodeGenInfo(CGM);
279	case llvm::Triple::lanai:
280	return createLanaiTargetCodeGenInfo(CGM);
281	case llvm::Triple::r600:
282	return createAMDGPUTargetCodeGenInfo(CGM);
283	case llvm::Triple::amdgcn:
284	return createAMDGPUTargetCodeGenInfo(CGM);
285	case llvm::Triple::sparc:
286	return createSparcV8TargetCodeGenInfo(CGM);
287	case llvm::Triple::sparcv9:
288	return createSparcV9TargetCodeGenInfo(CGM);
289	case llvm::Triple::xcore:
290	return createXCoreTargetCodeGenInfo(CGM);
291	case llvm::Triple::arc:
292	return createARCTargetCodeGenInfo(CGM);
293	case llvm::Triple::spir:
294	case llvm::Triple::spir64:
295	return createCommonSPIRTargetCodeGenInfo(CGM);
296	case llvm::Triple::spirv32:
297	case llvm::Triple::spirv64:
298	case llvm::Triple::spirv:
299	return createSPIRVTargetCodeGenInfo(CGM);
300	case llvm::Triple::dxil:
301	return createDirectXTargetCodeGenInfo(CGM);
302	case llvm::Triple::ve:
303	return createVETargetCodeGenInfo(CGM);
304	case llvm::Triple::csky: {
305	bool IsSoftFloat = !Target.hasFeature(Feature: "hard-float-abi");
306	bool hasFP64 =
307	Target.hasFeature(Feature: "fpuv2_df") || Target.hasFeature(Feature: "fpuv3_df");
308	return createCSKYTargetCodeGenInfo(CGM, FLen: IsSoftFloat ? 0
309	: hasFP64 ? 64
310	: 32);
311	}
312	case llvm::Triple::bpfeb:
313	case llvm::Triple::bpfel:
314	return createBPFTargetCodeGenInfo(CGM);
315	case llvm::Triple::loongarch32:
316	case llvm::Triple::loongarch64: {
317	StringRef ABIStr = Target.getABI();
318	unsigned ABIFRLen = 0;
319	if (ABIStr.ends_with(Suffix: "f"))
320	ABIFRLen = 32;
321	else if (ABIStr.ends_with(Suffix: "d"))
322	ABIFRLen = 64;
323	return createLoongArchTargetCodeGenInfo(
324	CGM, GRLen: Target.getPointerWidth(AddrSpace: LangAS::Default), FLen: ABIFRLen);
325	}
326	}
327	}
328
329	const TargetCodeGenInfo &CodeGenModule::getTargetCodeGenInfo() {
330	if (!TheTargetCodeGenInfo)
331	TheTargetCodeGenInfo = createTargetCodeGenInfo(CGM&: *this);
332	return *TheTargetCodeGenInfo;
333	}
334
335	static void checkDataLayoutConsistency(const TargetInfo &Target,
336	llvm::LLVMContext &Context,
337	const LangOptions &Opts) {
338	#ifndef NDEBUG
339	// Don't verify non-standard ABI configurations.
340	if (Opts.AlignDouble || Opts.OpenCL || Opts.HLSL)
341	return;
342
343	llvm::Triple Triple = Target.getTriple();
344	llvm::DataLayout DL(Target.getDataLayoutString());
345	auto Check = [&](const char *Name, llvm::Type *Ty, unsigned Alignment) {
346	llvm::Align DLAlign = DL.getABITypeAlign(Ty);
347	llvm::Align ClangAlign(Alignment / 8);
348	if (DLAlign != ClangAlign) {
349	llvm::errs() << "For target " << Triple.str() << " type " << Name
350	<< " mapping to " << *Ty << " has data layout alignment "
351	<< DLAlign.value() << " while clang specifies "
352	<< ClangAlign.value() << "\n";
353	abort();
354	}
355	};
356
357	Check("bool", llvm::Type::getIntNTy(Context, Target.BoolWidth),
358	Target.BoolAlign);
359	Check("short", llvm::Type::getIntNTy(Context, Target.ShortWidth),
360	Target.ShortAlign);
361	Check("int", llvm::Type::getIntNTy(Context, Target.IntWidth),
362	Target.IntAlign);
363	Check("long", llvm::Type::getIntNTy(Context, Target.LongWidth),
364	Target.LongAlign);
365	// FIXME: M68k specifies incorrect long long alignment in both LLVM and Clang.
366	if (Triple.getArch() != llvm::Triple::m68k)
367	Check("long long", llvm::Type::getIntNTy(Context, Target.LongLongWidth),
368	Target.LongLongAlign);
369	// FIXME: There are int128 alignment mismatches on multiple targets.
370	if (Target.hasInt128Type() && !Target.getTargetOpts().ForceEnableInt128 &&
371	!Triple.isAMDGPU() && !Triple.isSPIRV() &&
372	Triple.getArch() != llvm::Triple::ve)
373	Check("__int128", llvm::Type::getIntNTy(Context, 128), Target.Int128Align);
374
375	if (Target.hasFloat16Type())
376	Check("half", llvm::Type::getFloatingPointTy(Context, *Target.HalfFormat),
377	Target.HalfAlign);
378	if (Target.hasBFloat16Type())
379	Check("bfloat", llvm::Type::getBFloatTy(Context), Target.BFloat16Align);
380	Check("float", llvm::Type::getFloatingPointTy(Context, *Target.FloatFormat),
381	Target.FloatAlign);
382	// FIXME: AIX specifies wrong double alignment in DataLayout
383	if (!Triple.isOSAIX()) {
384	Check("double",
385	llvm::Type::getFloatingPointTy(Context, *Target.DoubleFormat),
386	Target.DoubleAlign);
387	Check("long double",
388	llvm::Type::getFloatingPointTy(Context, *Target.LongDoubleFormat),
389	Target.LongDoubleAlign);
390	}
391	if (Target.hasFloat128Type())
392	Check("__float128", llvm::Type::getFP128Ty(Context), Target.Float128Align);
393	if (Target.hasIbm128Type())
394	Check("__ibm128", llvm::Type::getPPC_FP128Ty(Context), Target.Ibm128Align);
395
396	Check("void*", llvm::PointerType::getUnqual(Context), Target.PointerAlign);
397	#endif
398	}
399
400	CodeGenModule::CodeGenModule(ASTContext &C,
401	IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS,
402	const HeaderSearchOptions &HSO,
403	const PreprocessorOptions &PPO,
404	const CodeGenOptions &CGO, llvm::Module &M,
405	DiagnosticsEngine &diags,
406	CoverageSourceInfo *CoverageInfo)
407	: Context(C), LangOpts(C.getLangOpts()), FS(FS), HeaderSearchOpts(HSO),
408	PreprocessorOpts(PPO), CodeGenOpts(CGO), TheModule(M), Diags(diags),
409	Target(C.getTargetInfo()), ABI(createCXXABI(CGM&: *this)),
410	VMContext(M.getContext()), VTables(*this), StackHandler(diags),
411	SanitizerMD(new SanitizerMetadata(*this)),
412	AtomicOpts(Target.getAtomicOpts()) {
413
414	// Initialize the type cache.
415	Types.reset(p: new CodeGenTypes(*this));
416	llvm::LLVMContext &LLVMContext = M.getContext();
417	VoidTy = llvm::Type::getVoidTy(C&: LLVMContext);
418	Int8Ty = llvm::Type::getInt8Ty(C&: LLVMContext);
419	Int16Ty = llvm::Type::getInt16Ty(C&: LLVMContext);
420	Int32Ty = llvm::Type::getInt32Ty(C&: LLVMContext);
421	Int64Ty = llvm::Type::getInt64Ty(C&: LLVMContext);
422	HalfTy = llvm::Type::getHalfTy(C&: LLVMContext);
423	BFloatTy = llvm::Type::getBFloatTy(C&: LLVMContext);
424	FloatTy = llvm::Type::getFloatTy(C&: LLVMContext);
425	DoubleTy = llvm::Type::getDoubleTy(C&: LLVMContext);
426	PointerWidthInBits = C.getTargetInfo().getPointerWidth(AddrSpace: LangAS::Default);
427	PointerAlignInBytes =
428	C.toCharUnitsFromBits(BitSize: C.getTargetInfo().getPointerAlign(AddrSpace: LangAS::Default))
429	.getQuantity();
430	SizeSizeInBytes =
431	C.toCharUnitsFromBits(BitSize: C.getTargetInfo().getMaxPointerWidth()).getQuantity();
432	IntAlignInBytes =
433	C.toCharUnitsFromBits(BitSize: C.getTargetInfo().getIntAlign()).getQuantity();
434	CharTy =
435	llvm::IntegerType::get(C&: LLVMContext, NumBits: C.getTargetInfo().getCharWidth());
436	IntTy = llvm::IntegerType::get(C&: LLVMContext, NumBits: C.getTargetInfo().getIntWidth());
437	IntPtrTy = llvm::IntegerType::get(C&: LLVMContext,
438	NumBits: C.getTargetInfo().getMaxPointerWidth());
439	Int8PtrTy = llvm::PointerType::get(C&: LLVMContext,
440	AddressSpace: C.getTargetAddressSpace(AS: LangAS::Default));
441	const llvm::DataLayout &DL = M.getDataLayout();
442	AllocaInt8PtrTy =
443	llvm::PointerType::get(C&: LLVMContext, AddressSpace: DL.getAllocaAddrSpace());
444	GlobalsInt8PtrTy =
445	llvm::PointerType::get(C&: LLVMContext, AddressSpace: DL.getDefaultGlobalsAddressSpace());
446	ConstGlobalsPtrTy = llvm::PointerType::get(
447	C&: LLVMContext, AddressSpace: C.getTargetAddressSpace(AS: GetGlobalConstantAddressSpace()));
448	ASTAllocaAddressSpace = getTargetCodeGenInfo().getASTAllocaAddressSpace();
449
450	// Build C++20 Module initializers.
451	// TODO: Add Microsoft here once we know the mangling required for the
452	// initializers.
453	CXX20ModuleInits =
454	LangOpts.CPlusPlusModules && getCXXABI().getMangleContext().getKind() ==
455	ItaniumMangleContext::MK_Itanium;
456
457	RuntimeCC = getTargetCodeGenInfo().getABIInfo().getRuntimeCC();
458
459	if (LangOpts.ObjC)
460	createObjCRuntime();
461	if (LangOpts.OpenCL)
462	createOpenCLRuntime();
463	if (LangOpts.OpenMP)
464	createOpenMPRuntime();
465	if (LangOpts.CUDA)
466	createCUDARuntime();
467	if (LangOpts.HLSL)
468	createHLSLRuntime();
469
470	// Enable TBAA unless it's suppressed. TSan and TySan need TBAA even at O0.
471	if (LangOpts.Sanitize.hasOneOf(K: SanitizerKind::Thread | SanitizerKind::Type) ||
472	(!CodeGenOpts.RelaxedAliasing && CodeGenOpts.OptimizationLevel > 0))
473	TBAA.reset(p: new CodeGenTBAA(Context, getTypes(), TheModule, CodeGenOpts,
474	getLangOpts()));
475
476	// If debug info or coverage generation is enabled, create the CGDebugInfo
477	// object.
478	if (CodeGenOpts.getDebugInfo() != llvm::codegenoptions::NoDebugInfo ||
479	CodeGenOpts.CoverageNotesFile.size() ||
480	CodeGenOpts.CoverageDataFile.size())
481	DebugInfo.reset(p: new CGDebugInfo(*this));
482	else if (getTriple().isOSWindows())
483	// On Windows targets, we want to emit compiler info even if debug info is
484	// otherwise disabled. Use a temporary CGDebugInfo instance to emit only
485	// basic compiler metadata.
486	CGDebugInfo(*this);
487
488	Block.GlobalUniqueCount = 0;
489
490	if (C.getLangOpts().ObjC)
491	ObjCData.reset(p: new ObjCEntrypoints());
492
493	if (CodeGenOpts.hasProfileClangUse()) {
494	auto ReaderOrErr = llvm::IndexedInstrProfReader::create(
495	Path: CodeGenOpts.ProfileInstrumentUsePath, FS&: *FS,
496	RemappingPath: CodeGenOpts.ProfileRemappingFile);
497	// We're checking for profile read errors in CompilerInvocation, so if
498	// there was an error it should've already been caught. If it hasn't been
499	// somehow, trip an assertion.
500	assert(ReaderOrErr);
501	PGOReader = std::move(ReaderOrErr.get());
502	}
503
504	// If coverage mapping generation is enabled, create the
505	// CoverageMappingModuleGen object.
506	if (CodeGenOpts.CoverageMapping)
507	CoverageMapping.reset(p: new CoverageMappingModuleGen(*this, *CoverageInfo));
508
509	// Generate the module name hash here if needed.
510	if (CodeGenOpts.UniqueInternalLinkageNames &&
511	!getModule().getSourceFileName().empty()) {
512	std::string Path = getModule().getSourceFileName();
513	// Check if a path substitution is needed from the MacroPrefixMap.
514	for (const auto &Entry : LangOpts.MacroPrefixMap)
515	if (Path.rfind(str: Entry.first, pos: 0) != std::string::npos) {
516	Path = Entry.second + Path.substr(pos: Entry.first.size());
517	break;
518	}
519	ModuleNameHash = llvm::getUniqueInternalLinkagePostfix(FName: Path);
520	}
521
522	// Record mregparm value now so it is visible through all of codegen.
523	if (Context.getTargetInfo().getTriple().getArch() == llvm::Triple::x86)
524	getModule().addModuleFlag(Behavior: llvm::Module::Error, Key: "NumRegisterParameters",
525	Val: CodeGenOpts.NumRegisterParameters);
526
527	// If there are any functions that are marked for Windows secure hot-patching,
528	// then build the list of functions now.
529	if (!CGO.MSSecureHotPatchFunctionsFile.empty() ||
530	!CGO.MSSecureHotPatchFunctionsList.empty()) {
531	if (!CGO.MSSecureHotPatchFunctionsFile.empty()) {
532	auto BufOrErr =
533	llvm::MemoryBuffer::getFile(Filename: CGO.MSSecureHotPatchFunctionsFile);
534	if (BufOrErr) {
535	const llvm::MemoryBuffer &FileBuffer = **BufOrErr;
536	for (llvm::line_iterator I(FileBuffer.getMemBufferRef(), true), E;
537	I != E; ++I)
538	this->MSHotPatchFunctions.push_back(x: std::string{*I});
539	} else {
540	auto &DE = Context.getDiagnostics();
541	unsigned DiagID =
542	DE.getCustomDiagID(L: DiagnosticsEngine::Error,
543	FormatString: "failed to open hotpatch functions file "
544	"(-fms-hotpatch-functions-file): %0 : %1");
545	DE.Report(DiagID) << CGO.MSSecureHotPatchFunctionsFile
546	<< BufOrErr.getError().message();
547	}
548	}
549
550	for (const auto &FuncName : CGO.MSSecureHotPatchFunctionsList)
551	this->MSHotPatchFunctions.push_back(x: FuncName);
552
553	llvm::sort(C&: this->MSHotPatchFunctions);
554	}
555
556	if (!Context.getAuxTargetInfo())
557	checkDataLayoutConsistency(Target: Context.getTargetInfo(), Context&: LLVMContext, Opts: LangOpts);
558	}
559
560	CodeGenModule::~CodeGenModule() {}
561
562	void CodeGenModule::createObjCRuntime() {
563	// This is just isGNUFamily(), but we want to force implementors of
564	// new ABIs to decide how best to do this.
565	switch (LangOpts.ObjCRuntime.getKind()) {
566	case ObjCRuntime::GNUstep:
567	case ObjCRuntime::GCC:
568	case ObjCRuntime::ObjFW:
569	ObjCRuntime.reset(p: CreateGNUObjCRuntime(CGM&: *this));
570	return;
571
572	case ObjCRuntime::FragileMacOSX:
573	case ObjCRuntime::MacOSX:
574	case ObjCRuntime::iOS:
575	case ObjCRuntime::WatchOS:
576	ObjCRuntime.reset(p: CreateMacObjCRuntime(CGM&: *this));
577	return;
578	}
579	llvm_unreachable("bad runtime kind");
580	}
581
582	void CodeGenModule::createOpenCLRuntime() {
583	OpenCLRuntime.reset(p: new CGOpenCLRuntime(*this));
584	}
585
586	void CodeGenModule::createOpenMPRuntime() {
587	// Select a specialized code generation class based on the target, if any.
588	// If it does not exist use the default implementation.
589	switch (getTriple().getArch()) {
590	case llvm::Triple::nvptx:
591	case llvm::Triple::nvptx64:
592	case llvm::Triple::amdgcn:
593	case llvm::Triple::spirv64:
594	assert(
595	getLangOpts().OpenMPIsTargetDevice &&
596	"OpenMP AMDGPU/NVPTX/SPIRV is only prepared to deal with device code.");
597	OpenMPRuntime.reset(p: new CGOpenMPRuntimeGPU(*this));
598	break;
599	default:
600	if (LangOpts.OpenMPSimd)
601	OpenMPRuntime.reset(p: new CGOpenMPSIMDRuntime(*this));
602	else
603	OpenMPRuntime.reset(p: new CGOpenMPRuntime(*this));
604	break;
605	}
606	}
607
608	void CodeGenModule::createCUDARuntime() {
609	CUDARuntime.reset(p: CreateNVCUDARuntime(CGM&: *this));
610	}
611
612	void CodeGenModule::createHLSLRuntime() {
613	HLSLRuntime.reset(p: new CGHLSLRuntime(*this));
614	}
615
616	void CodeGenModule::addReplacement(StringRef Name, llvm::Constant *C) {
617	Replacements[Name] = C;
618	}
619
620	void CodeGenModule::applyReplacements() {
621	for (auto &I : Replacements) {
622	StringRef MangledName = I.first;
623	llvm::Constant *Replacement = I.second;
624	llvm::GlobalValue *Entry = GetGlobalValue(Ref: MangledName);
625	if (!Entry)
626	continue;
627	auto *OldF = cast<llvm::Function>(Val: Entry);
628	auto *NewF = dyn_cast<llvm::Function>(Val: Replacement);
629	if (!NewF) {
630	if (auto *Alias = dyn_cast<llvm::GlobalAlias>(Val: Replacement)) {
631	NewF = dyn_cast<llvm::Function>(Val: Alias->getAliasee());
632	} else {
633	auto *CE = cast<llvm::ConstantExpr>(Val: Replacement);
634	assert(CE->getOpcode() == llvm::Instruction::BitCast ||
635	CE->getOpcode() == llvm::Instruction::GetElementPtr);
636	NewF = dyn_cast<llvm::Function>(Val: CE->getOperand(i_nocapture: 0));
637	}
638	}
639
640	// Replace old with new, but keep the old order.
641	OldF->replaceAllUsesWith(V: Replacement);
642	if (NewF) {
643	NewF->removeFromParent();
644	OldF->getParent()->getFunctionList().insertAfter(where: OldF->getIterator(),
645	New: NewF);
646	}
647	OldF->eraseFromParent();
648	}
649	}
650
651	void CodeGenModule::addGlobalValReplacement(llvm::GlobalValue *GV, llvm::Constant *C) {
652	GlobalValReplacements.push_back(Elt: std::make_pair(x&: GV, y&: C));
653	}
654
655	void CodeGenModule::applyGlobalValReplacements() {
656	for (auto &I : GlobalValReplacements) {
657	llvm::GlobalValue *GV = I.first;
658	llvm::Constant *C = I.second;
659
660	GV->replaceAllUsesWith(V: C);
661	GV->eraseFromParent();
662	}
663	}
664
665	// This is only used in aliases that we created and we know they have a
666	// linear structure.
667	static const llvm::GlobalValue *getAliasedGlobal(const llvm::GlobalValue *GV) {
668	const llvm::Constant *C;
669	if (auto *GA = dyn_cast<llvm::GlobalAlias>(Val: GV))
670	C = GA->getAliasee();
671	else if (auto *GI = dyn_cast<llvm::GlobalIFunc>(Val: GV))
672	C = GI->getResolver();
673	else
674	return GV;
675
676	const auto *AliaseeGV = dyn_cast<llvm::GlobalValue>(Val: C->stripPointerCasts());
677	if (!AliaseeGV)
678	return nullptr;
679
680	const llvm::GlobalValue *FinalGV = AliaseeGV->getAliaseeObject();
681	if (FinalGV == GV)
682	return nullptr;
683
684	return FinalGV;
685	}
686
687	static bool checkAliasedGlobal(
688	const ASTContext &Context, DiagnosticsEngine &Diags, SourceLocation Location,
689	bool IsIFunc, const llvm::GlobalValue *Alias, const llvm::GlobalValue *&GV,
690	const llvm::MapVector<GlobalDecl, StringRef> &MangledDeclNames,
691	SourceRange AliasRange) {
692	GV = getAliasedGlobal(GV: Alias);
693	if (!GV) {
694	Diags.Report(Loc: Location, DiagID: diag::err_cyclic_alias) << IsIFunc;
695	return false;
696	}
697
698	if (GV->hasCommonLinkage()) {
699	const llvm::Triple &Triple = Context.getTargetInfo().getTriple();
700	if (Triple.getObjectFormat() == llvm::Triple::XCOFF) {
701	Diags.Report(Loc: Location, DiagID: diag::err_alias_to_common);
702	return false;
703	}
704	}
705
706	if (GV->isDeclaration()) {
707	Diags.Report(Loc: Location, DiagID: diag::err_alias_to_undefined) << IsIFunc << IsIFunc;
708	Diags.Report(Loc: Location, DiagID: diag::note_alias_requires_mangled_name)
709	<< IsIFunc << IsIFunc;
710	// Provide a note if the given function is not found and exists as a
711	// mangled name.
712	for (const auto &[Decl, Name] : MangledDeclNames) {
713	if (const auto *ND = dyn_cast<NamedDecl>(Val: Decl.getDecl())) {
714	IdentifierInfo *II = ND->getIdentifier();
715	if (II && II->getName() == GV->getName()) {
716	Diags.Report(Loc: Location, DiagID: diag::note_alias_mangled_name_alternative)
717	<< Name
718	<< FixItHint::CreateReplacement(
719	RemoveRange: AliasRange,
720	Code: (Twine(IsIFunc ? "ifunc" : "alias") + "(\"" + Name + "\")")
721	.str());
722	}
723	}
724	}
725	return false;
726	}
727
728	if (IsIFunc) {
729	// Check resolver function type.
730	const auto *F = dyn_cast<llvm::Function>(Val: GV);
731	if (!F) {
732	Diags.Report(Loc: Location, DiagID: diag::err_alias_to_undefined)
733	<< IsIFunc << IsIFunc;
734	return false;
735	}
736
737	llvm::FunctionType *FTy = F->getFunctionType();
738	if (!FTy->getReturnType()->isPointerTy()) {
739	Diags.Report(Loc: Location, DiagID: diag::err_ifunc_resolver_return);
740	return false;
741	}
742	}
743
744	return true;
745	}
746
747	// Emit a warning if toc-data attribute is requested for global variables that
748	// have aliases and remove the toc-data attribute.
749	static void checkAliasForTocData(llvm::GlobalVariable *GVar,
750	const CodeGenOptions &CodeGenOpts,
751	DiagnosticsEngine &Diags,
752	SourceLocation Location) {
753	if (GVar->hasAttribute(Kind: "toc-data")) {
754	auto GVId = GVar->getName();
755	// Is this a global variable specified by the user as local?
756	if ((llvm::binary_search(Range: CodeGenOpts.TocDataVarsUserSpecified, Value&: GVId))) {
757	Diags.Report(Loc: Location, DiagID: diag::warn_toc_unsupported_type)
758	<< GVId << "the variable has an alias";
759	}
760	llvm::AttributeSet CurrAttributes = GVar->getAttributes();
761	llvm::AttributeSet NewAttributes =
762	CurrAttributes.removeAttribute(C&: GVar->getContext(), Kind: "toc-data");
763	GVar->setAttributes(NewAttributes);
764	}
765	}
766
767	void CodeGenModule::checkAliases() {
768	// Check if the constructed aliases are well formed. It is really unfortunate
769	// that we have to do this in CodeGen, but we only construct mangled names
770	// and aliases during codegen.
771	bool Error = false;
772	DiagnosticsEngine &Diags = getDiags();
773	for (const GlobalDecl &GD : Aliases) {
774	const auto *D = cast<ValueDecl>(Val: GD.getDecl());
775	SourceLocation Location;
776	SourceRange Range;
777	bool IsIFunc = D->hasAttr<IFuncAttr>();
778	if (const Attr *A = D->getDefiningAttr()) {
779	Location = A->getLocation();
780	Range = A->getRange();
781	} else
782	llvm_unreachable("Not an alias or ifunc?");
783
784	StringRef MangledName = getMangledName(GD);
785	llvm::GlobalValue *Alias = GetGlobalValue(Ref: MangledName);
786	const llvm::GlobalValue *GV = nullptr;
787	if (!checkAliasedGlobal(Context: getContext(), Diags, Location, IsIFunc, Alias, GV,
788	MangledDeclNames, AliasRange: Range)) {
789	Error = true;
790	continue;
791	}
792
793	if (getContext().getTargetInfo().getTriple().isOSAIX())
794	if (const llvm::GlobalVariable *GVar =
795	dyn_cast<const llvm::GlobalVariable>(Val: GV))
796	checkAliasForTocData(GVar: const_cast<llvm::GlobalVariable *>(GVar),
797	CodeGenOpts: getCodeGenOpts(), Diags, Location);
798
799	llvm::Constant *Aliasee =
800	IsIFunc ? cast<llvm::GlobalIFunc>(Val: Alias)->getResolver()
801	: cast<llvm::GlobalAlias>(Val: Alias)->getAliasee();
802
803	llvm::GlobalValue *AliaseeGV;
804	if (auto CE = dyn_cast<llvm::ConstantExpr>(Val: Aliasee))
805	AliaseeGV = cast<llvm::GlobalValue>(Val: CE->getOperand(i_nocapture: 0));
806	else
807	AliaseeGV = cast<llvm::GlobalValue>(Val: Aliasee);
808
809	if (const SectionAttr *SA = D->getAttr<SectionAttr>()) {
810	StringRef AliasSection = SA->getName();
811	if (AliasSection != AliaseeGV->getSection())
812	Diags.Report(Loc: SA->getLocation(), DiagID: diag::warn_alias_with_section)
813	<< AliasSection << IsIFunc << IsIFunc;
814	}
815
816	// We have to handle alias to weak aliases in here. LLVM itself disallows
817	// this since the object semantics would not match the IL one. For
818	// compatibility with gcc we implement it by just pointing the alias
819	// to its aliasee's aliasee. We also warn, since the user is probably
820	// expecting the link to be weak.
821	if (auto *GA = dyn_cast<llvm::GlobalAlias>(Val: AliaseeGV)) {
822	if (GA->isInterposable()) {
823	Diags.Report(Loc: Location, DiagID: diag::warn_alias_to_weak_alias)
824	<< GV->getName() << GA->getName() << IsIFunc;
825	Aliasee = llvm::ConstantExpr::getPointerBitCastOrAddrSpaceCast(
826	C: GA->getAliasee(), Ty: Alias->getType());
827
828	if (IsIFunc)
829	cast<llvm::GlobalIFunc>(Val: Alias)->setResolver(Aliasee);
830	else
831	cast<llvm::GlobalAlias>(Val: Alias)->setAliasee(Aliasee);
832	}
833	}
834	// ifunc resolvers are usually implemented to run before sanitizer
835	// initialization. Disable instrumentation to prevent the ordering issue.
836	if (IsIFunc)
837	cast<llvm::Function>(Val: Aliasee)->addFnAttr(
838	Kind: llvm::Attribute::DisableSanitizerInstrumentation);
839	}
840	if (!Error)
841	return;
842
843	for (const GlobalDecl &GD : Aliases) {
844	StringRef MangledName = getMangledName(GD);
845	llvm::GlobalValue *Alias = GetGlobalValue(Ref: MangledName);
846	Alias->replaceAllUsesWith(V: llvm::PoisonValue::get(T: Alias->getType()));
847	Alias->eraseFromParent();
848	}
849	}
850
851	void CodeGenModule::clear() {
852	DeferredDeclsToEmit.clear();
853	EmittedDeferredDecls.clear();
854	DeferredAnnotations.clear();
855	if (OpenMPRuntime)
856	OpenMPRuntime->clear();
857	}
858
859	void InstrProfStats::reportDiagnostics(DiagnosticsEngine &Diags,
860	StringRef MainFile) {
861	if (!hasDiagnostics())
862	return;
863	if (VisitedInMainFile > 0 && VisitedInMainFile == MissingInMainFile) {
864	if (MainFile.empty())
865	MainFile = "<stdin>";
866	Diags.Report(DiagID: diag::warn_profile_data_unprofiled) << MainFile;
867	} else {
868	if (Mismatched > 0)
869	Diags.Report(DiagID: diag::warn_profile_data_out_of_date) << Visited << Mismatched;
870
871	if (Missing > 0)
872	Diags.Report(DiagID: diag::warn_profile_data_missing) << Visited << Missing;
873	}
874	}
875
876	static std::optional<llvm::GlobalValue::VisibilityTypes>
877	getLLVMVisibility(clang::LangOptions::VisibilityFromDLLStorageClassKinds K) {
878	// Map to LLVM visibility.
879	switch (K) {
880	case clang::LangOptions::VisibilityFromDLLStorageClassKinds::Keep:
881	return std::nullopt;
882	case clang::LangOptions::VisibilityFromDLLStorageClassKinds::Default:
883	return llvm::GlobalValue::DefaultVisibility;
884	case clang::LangOptions::VisibilityFromDLLStorageClassKinds::Hidden:
885	return llvm::GlobalValue::HiddenVisibility;
886	case clang::LangOptions::VisibilityFromDLLStorageClassKinds::Protected:
887	return llvm::GlobalValue::ProtectedVisibility;
888	}
889	llvm_unreachable("unknown option value!");
890	}
891
892	static void
893	setLLVMVisibility(llvm::GlobalValue &GV,
894	std::optional<llvm::GlobalValue::VisibilityTypes> V) {
895	if (!V)
896	return;
897
898	// Reset DSO locality before setting the visibility. This removes
899	// any effects that visibility options and annotations may have
900	// had on the DSO locality. Setting the visibility will implicitly set
901	// appropriate globals to DSO Local; however, this will be pessimistic
902	// w.r.t. to the normal compiler IRGen.
903	GV.setDSOLocal(false);
904	GV.setVisibility(*V);
905	}
906
907	static void setVisibilityFromDLLStorageClass(const clang::LangOptions &LO,
908	llvm::Module &M) {
909	if (!LO.VisibilityFromDLLStorageClass)
910	return;
911
912	std::optional<llvm::GlobalValue::VisibilityTypes> DLLExportVisibility =
913	getLLVMVisibility(K: LO.getDLLExportVisibility());
914
915	std::optional<llvm::GlobalValue::VisibilityTypes>
916	NoDLLStorageClassVisibility =
917	getLLVMVisibility(K: LO.getNoDLLStorageClassVisibility());
918
919	std::optional<llvm::GlobalValue::VisibilityTypes>
920	ExternDeclDLLImportVisibility =
921	getLLVMVisibility(K: LO.getExternDeclDLLImportVisibility());
922
923	std::optional<llvm::GlobalValue::VisibilityTypes>
924	ExternDeclNoDLLStorageClassVisibility =
925	getLLVMVisibility(K: LO.getExternDeclNoDLLStorageClassVisibility());
926
927	for (llvm::GlobalValue &GV : M.global_values()) {
928	if (GV.hasAppendingLinkage() || GV.hasLocalLinkage())
929	continue;
930
931	if (GV.isDeclarationForLinker())
932	setLLVMVisibility(GV, V: GV.getDLLStorageClass() ==
933	llvm::GlobalValue::DLLImportStorageClass
934	? ExternDeclDLLImportVisibility
935	: ExternDeclNoDLLStorageClassVisibility);
936	else
937	setLLVMVisibility(GV, V: GV.getDLLStorageClass() ==
938	llvm::GlobalValue::DLLExportStorageClass
939	? DLLExportVisibility
940	: NoDLLStorageClassVisibility);
941
942	GV.setDLLStorageClass(llvm::GlobalValue::DefaultStorageClass);
943	}
944	}
945
946	static bool isStackProtectorOn(const LangOptions &LangOpts,
947	const llvm::Triple &Triple,
948	clang::LangOptions::StackProtectorMode Mode) {
949	if (Triple.isGPU())
950	return false;
951	return LangOpts.getStackProtector() == Mode;
952	}
953
954	void CodeGenModule::Release() {
955	Module *Primary = getContext().getCurrentNamedModule();
956	if (CXX20ModuleInits && Primary && !Primary->isHeaderLikeModule())
957	EmitModuleInitializers(Primary);
958	EmitDeferred();
959	DeferredDecls.insert_range(R&: EmittedDeferredDecls);
960	EmittedDeferredDecls.clear();
961	EmitVTablesOpportunistically();
962	applyGlobalValReplacements();
963	applyReplacements();
964	emitMultiVersionFunctions();
965
966	if (Context.getLangOpts().IncrementalExtensions &&
967	GlobalTopLevelStmtBlockInFlight.first) {
968	const TopLevelStmtDecl *TLSD = GlobalTopLevelStmtBlockInFlight.second;
969	GlobalTopLevelStmtBlockInFlight.first->FinishFunction(EndLoc: TLSD->getEndLoc());
970	GlobalTopLevelStmtBlockInFlight = {nullptr, nullptr};
971	}
972
973	// Module implementations are initialized the same way as a regular TU that
974	// imports one or more modules.
975	if (CXX20ModuleInits && Primary && Primary->isInterfaceOrPartition())
976	EmitCXXModuleInitFunc(Primary);
977	else
978	EmitCXXGlobalInitFunc();
979	EmitCXXGlobalCleanUpFunc();
980	registerGlobalDtorsWithAtExit();
981	EmitCXXThreadLocalInitFunc();
982	if (ObjCRuntime)
983	if (llvm::Function *ObjCInitFunction = ObjCRuntime->ModuleInitFunction())
984	AddGlobalCtor(Ctor: ObjCInitFunction);
985	if (Context.getLangOpts().CUDA && CUDARuntime) {
986	if (llvm::Function *CudaCtorFunction = CUDARuntime->finalizeModule())
987	AddGlobalCtor(Ctor: CudaCtorFunction);
988	}
989	if (OpenMPRuntime) {
990	OpenMPRuntime->createOffloadEntriesAndInfoMetadata();
991	OpenMPRuntime->clear();
992	}
993	if (PGOReader) {
994	getModule().setProfileSummary(
995	M: PGOReader->getSummary(/* UseCS */ false).getMD(Context&: VMContext),
996	Kind: llvm::ProfileSummary::PSK_Instr);
997	if (PGOStats.hasDiagnostics())
998	PGOStats.reportDiagnostics(Diags&: getDiags(), MainFile: getCodeGenOpts().MainFileName);
999	}
1000	llvm::stable_sort(Range&: GlobalCtors, C: [](const Structor &L, const Structor &R) {
1001	return L.LexOrder < R.LexOrder;
1002	});
1003	EmitCtorList(Fns&: GlobalCtors, GlobalName: "llvm.global_ctors");
1004	EmitCtorList(Fns&: GlobalDtors, GlobalName: "llvm.global_dtors");
1005	EmitGlobalAnnotations();
1006	EmitStaticExternCAliases();
1007	checkAliases();
1008	EmitDeferredUnusedCoverageMappings();
1009	CodeGenPGO(*this).setValueProfilingFlag(getModule());
1010	CodeGenPGO(*this).setProfileVersion(getModule());
1011	if (CoverageMapping)
1012	CoverageMapping->emit();
1013	if (CodeGenOpts.SanitizeCfiCrossDso) {
1014	CodeGenFunction(*this).EmitCfiCheckFail();
1015	CodeGenFunction(*this).EmitCfiCheckStub();
1016	}
1017	if (LangOpts.Sanitize.has(K: SanitizerKind::KCFI))
1018	finalizeKCFITypes();
1019	emitAtAvailableLinkGuard();
1020	if (Context.getTargetInfo().getTriple().isWasm())
1021	EmitMainVoidAlias();
1022
1023	if (getTriple().isAMDGPU() ||
1024	(getTriple().isSPIRV() && getTriple().getVendor() == llvm::Triple::AMD)) {
1025	// Emit amdhsa_code_object_version module flag, which is code object version
1026	// times 100.
1027	if (getTarget().getTargetOpts().CodeObjectVersion !=
1028	llvm::CodeObjectVersionKind::COV_None) {
1029	getModule().addModuleFlag(Behavior: llvm::Module::Error,
1030	Key: "amdhsa_code_object_version",
1031	Val: getTarget().getTargetOpts().CodeObjectVersion);
1032	}
1033
1034	// Currently, "-mprintf-kind" option is only supported for HIP
1035	if (LangOpts.HIP) {
1036	auto *MDStr = llvm::MDString::get(
1037	Context&: getLLVMContext(), Str: (getTarget().getTargetOpts().AMDGPUPrintfKindVal ==
1038	TargetOptions::AMDGPUPrintfKind::Hostcall)
1039	? "hostcall"
1040	: "buffered");
1041	getModule().addModuleFlag(Behavior: llvm::Module::Error, Key: "amdgpu_printf_kind",
1042	Val: MDStr);
1043	}
1044	}
1045
1046	// Emit a global array containing all external kernels or device variables
1047	// used by host functions and mark it as used for CUDA/HIP. This is necessary
1048	// to get kernels or device variables in archives linked in even if these
1049	// kernels or device variables are only used in host functions.
1050	if (!Context.CUDAExternalDeviceDeclODRUsedByHost.empty()) {
1051	SmallVector<llvm::Constant *, 8> UsedArray;
1052	for (auto D : Context.CUDAExternalDeviceDeclODRUsedByHost) {
1053	GlobalDecl GD;
1054	if (auto *FD = dyn_cast<FunctionDecl>(Val: D))
1055	GD = GlobalDecl(FD, KernelReferenceKind::Kernel);
1056	else
1057	GD = GlobalDecl(D);
1058	UsedArray.push_back(Elt: llvm::ConstantExpr::getPointerBitCastOrAddrSpaceCast(
1059	C: GetAddrOfGlobal(GD), Ty: Int8PtrTy));
1060	}
1061
1062	llvm::ArrayType *ATy = llvm::ArrayType::get(ElementType: Int8PtrTy, NumElements: UsedArray.size());
1063
1064	auto *GV = new llvm::GlobalVariable(
1065	getModule(), ATy, false, llvm::GlobalValue::InternalLinkage,
1066	llvm::ConstantArray::get(T: ATy, V: UsedArray), "__clang_gpu_used_external");
1067	addCompilerUsedGlobal(GV);
1068	}
1069	if (LangOpts.HIP) {
1070	// Emit a unique ID so that host and device binaries from the same
1071	// compilation unit can be associated.
1072	auto *GV = new llvm::GlobalVariable(
1073	getModule(), Int8Ty, false, llvm::GlobalValue::ExternalLinkage,
1074	llvm::Constant::getNullValue(Ty: Int8Ty),
1075	"__hip_cuid_" + getContext().getCUIDHash());
1076	getSanitizerMetadata()->disableSanitizerForGlobal(GV);
1077	addCompilerUsedGlobal(GV);
1078	}
1079	emitLLVMUsed();
1080	if (SanStats)
1081	SanStats->finish();
1082
1083	if (CodeGenOpts.Autolink &&
1084	(Context.getLangOpts().Modules || !LinkerOptionsMetadata.empty())) {
1085	EmitModuleLinkOptions();
1086	}
1087
1088	// On ELF we pass the dependent library specifiers directly to the linker
1089	// without manipulating them. This is in contrast to other platforms where
1090	// they are mapped to a specific linker option by the compiler. This
1091	// difference is a result of the greater variety of ELF linkers and the fact
1092	// that ELF linkers tend to handle libraries in a more complicated fashion
1093	// than on other platforms. This forces us to defer handling the dependent
1094	// libs to the linker.
1095	//
1096	// CUDA/HIP device and host libraries are different. Currently there is no
1097	// way to differentiate dependent libraries for host or device. Existing
1098	// usage of #pragma comment(lib, *) is intended for host libraries on
1099	// Windows. Therefore emit llvm.dependent-libraries only for host.
1100	if (!ELFDependentLibraries.empty() && !Context.getLangOpts().CUDAIsDevice) {
1101	auto *NMD = getModule().getOrInsertNamedMetadata(Name: "llvm.dependent-libraries");
1102	for (auto *MD : ELFDependentLibraries)
1103	NMD->addOperand(M: MD);
1104	}
1105
1106	if (CodeGenOpts.DwarfVersion) {
1107	getModule().addModuleFlag(Behavior: llvm::Module::Max, Key: "Dwarf Version",
1108	Val: CodeGenOpts.DwarfVersion);
1109	}
1110
1111	if (CodeGenOpts.Dwarf64)
1112	getModule().addModuleFlag(Behavior: llvm::Module::Max, Key: "DWARF64", Val: 1);
1113
1114	if (Context.getLangOpts().SemanticInterposition)
1115	// Require various optimization to respect semantic interposition.
1116	getModule().setSemanticInterposition(true);
1117
1118	if (CodeGenOpts.EmitCodeView) {
1119	// Indicate that we want CodeView in the metadata.
1120	getModule().addModuleFlag(Behavior: llvm::Module::Warning, Key: "CodeView", Val: 1);
1121	}
1122	if (CodeGenOpts.CodeViewGHash) {
1123	getModule().addModuleFlag(Behavior: llvm::Module::Warning, Key: "CodeViewGHash", Val: 1);
1124	}
1125	if (CodeGenOpts.ControlFlowGuard) {
1126	// Function ID tables and checks for Control Flow Guard (cfguard=2).
1127	getModule().addModuleFlag(Behavior: llvm::Module::Warning, Key: "cfguard", Val: 2);
1128	} else if (CodeGenOpts.ControlFlowGuardNoChecks) {
1129	// Function ID tables for Control Flow Guard (cfguard=1).
1130	getModule().addModuleFlag(Behavior: llvm::Module::Warning, Key: "cfguard", Val: 1);
1131	}
1132	if (CodeGenOpts.EHContGuard) {
1133	// Function ID tables for EH Continuation Guard.
1134	getModule().addModuleFlag(Behavior: llvm::Module::Warning, Key: "ehcontguard", Val: 1);
1135	}
1136	if (Context.getLangOpts().Kernel) {
1137	// Note if we are compiling with /kernel.
1138	getModule().addModuleFlag(Behavior: llvm::Module::Warning, Key: "ms-kernel", Val: 1);
1139	}
1140	if (CodeGenOpts.OptimizationLevel > 0 && CodeGenOpts.StrictVTablePointers) {
1141	// We don't support LTO with 2 with different StrictVTablePointers
1142	// FIXME: we could support it by stripping all the information introduced
1143	// by StrictVTablePointers.
1144
1145	getModule().addModuleFlag(Behavior: llvm::Module::Error, Key: "StrictVTablePointers",Val: 1);
1146
1147	llvm::Metadata *Ops[2] = {
1148	llvm::MDString::get(Context&: VMContext, Str: "StrictVTablePointers"),
1149	llvm::ConstantAsMetadata::get(C: llvm::ConstantInt::get(
1150	Ty: llvm::Type::getInt32Ty(C&: VMContext), V: 1))};
1151
1152	getModule().addModuleFlag(Behavior: llvm::Module::Require,
1153	Key: "StrictVTablePointersRequirement",
1154	Val: llvm::MDNode::get(Context&: VMContext, MDs: Ops));
1155	}
1156	if (getModuleDebugInfo() || getTriple().isOSWindows())
1157	// We support a single version in the linked module. The LLVM
1158	// parser will drop debug info with a different version number
1159	// (and warn about it, too).
1160	getModule().addModuleFlag(Behavior: llvm::Module::Warning, Key: "Debug Info Version",
1161	Val: llvm::DEBUG_METADATA_VERSION);
1162
1163	// We need to record the widths of enums and wchar_t, so that we can generate
1164	// the correct build attributes in the ARM backend. wchar_size is also used by
1165	// TargetLibraryInfo.
1166	uint64_t WCharWidth =
1167	Context.getTypeSizeInChars(T: Context.getWideCharType()).getQuantity();
1168	getModule().addModuleFlag(Behavior: llvm::Module::Error, Key: "wchar_size", Val: WCharWidth);
1169
1170	if (getTriple().isOSzOS()) {
1171	getModule().addModuleFlag(Behavior: llvm::Module::Warning,
1172	Key: "zos_product_major_version",
1173	Val: uint32_t(CLANG_VERSION_MAJOR));
1174	getModule().addModuleFlag(Behavior: llvm::Module::Warning,
1175	Key: "zos_product_minor_version",
1176	Val: uint32_t(CLANG_VERSION_MINOR));
1177	getModule().addModuleFlag(Behavior: llvm::Module::Warning, Key: "zos_product_patchlevel",
1178	Val: uint32_t(CLANG_VERSION_PATCHLEVEL));
1179	std::string ProductId = getClangVendor() + "clang";
1180	getModule().addModuleFlag(Behavior: llvm::Module::Error, Key: "zos_product_id",
1181	Val: llvm::MDString::get(Context&: VMContext, Str: ProductId));
1182
1183	// Record the language because we need it for the PPA2.
1184	StringRef lang_str = languageToString(
1185	L: LangStandard::getLangStandardForKind(K: LangOpts.LangStd).Language);
1186	getModule().addModuleFlag(Behavior: llvm::Module::Error, Key: "zos_cu_language",
1187	Val: llvm::MDString::get(Context&: VMContext, Str: lang_str));
1188
1189	time_t TT = PreprocessorOpts.SourceDateEpoch
1190	? *PreprocessorOpts.SourceDateEpoch
1191	: std::time(timer: nullptr);
1192	getModule().addModuleFlag(Behavior: llvm::Module::Max, Key: "zos_translation_time",
1193	Val: static_cast<uint64_t>(TT));
1194
1195	// Multiple modes will be supported here.
1196	getModule().addModuleFlag(Behavior: llvm::Module::Error, Key: "zos_le_char_mode",
1197	Val: llvm::MDString::get(Context&: VMContext, Str: "ascii"));
1198	}
1199
1200	llvm::Triple T = Context.getTargetInfo().getTriple();
1201	if (T.isARM() || T.isThumb()) {
1202	// The minimum width of an enum in bytes
1203	uint64_t EnumWidth = Context.getLangOpts().ShortEnums ? 1 : 4;
1204	getModule().addModuleFlag(Behavior: llvm::Module::Error, Key: "min_enum_size", Val: EnumWidth);
1205	}
1206
1207	if (T.isRISCV()) {
1208	StringRef ABIStr = Target.getABI();
1209	llvm::LLVMContext &Ctx = TheModule.getContext();
1210	getModule().addModuleFlag(Behavior: llvm::Module::Error, Key: "target-abi",
1211	Val: llvm::MDString::get(Context&: Ctx, Str: ABIStr));
1212
1213	// Add the canonical ISA string as metadata so the backend can set the ELF
1214	// attributes correctly. We use AppendUnique so LTO will keep all of the
1215	// unique ISA strings that were linked together.
1216	const std::vector<std::string> &Features =
1217	getTarget().getTargetOpts().Features;
1218	auto ParseResult =
1219	llvm::RISCVISAInfo::parseFeatures(XLen: T.isRISCV64() ? 64 : 32, Features);
1220	if (!errorToBool(Err: ParseResult.takeError()))
1221	getModule().addModuleFlag(
1222	Behavior: llvm::Module::AppendUnique, Key: "riscv-isa",
1223	Val: llvm::MDNode::get(
1224	Context&: Ctx, MDs: llvm::MDString::get(Context&: Ctx, Str: (*ParseResult)->toString())));
1225	}
1226
1227	if (CodeGenOpts.SanitizeCfiCrossDso) {
1228	// Indicate that we want cross-DSO control flow integrity checks.
1229	getModule().addModuleFlag(Behavior: llvm::Module::Override, Key: "Cross-DSO CFI", Val: 1);
1230	}
1231
1232	if (CodeGenOpts.WholeProgramVTables) {
1233	// Indicate whether VFE was enabled for this module, so that the
1234	// vcall_visibility metadata added under whole program vtables is handled
1235	// appropriately in the optimizer.
1236	getModule().addModuleFlag(Behavior: llvm::Module::Error, Key: "Virtual Function Elim",
1237	Val: CodeGenOpts.VirtualFunctionElimination);
1238	}
1239
1240	if (LangOpts.Sanitize.has(K: SanitizerKind::CFIICall)) {
1241	getModule().addModuleFlag(Behavior: llvm::Module::Override,
1242	Key: "CFI Canonical Jump Tables",
1243	Val: CodeGenOpts.SanitizeCfiCanonicalJumpTables);
1244	}
1245
1246	if (CodeGenOpts.SanitizeCfiICallNormalizeIntegers) {
1247	getModule().addModuleFlag(Behavior: llvm::Module::Override, Key: "cfi-normalize-integers",
1248	Val: 1);
1249	}
1250
1251	if (!CodeGenOpts.UniqueSourceFileIdentifier.empty()) {
1252	getModule().addModuleFlag(
1253	Behavior: llvm::Module::Append, Key: "Unique Source File Identifier",
1254	Val: llvm::MDTuple::get(
1255	Context&: TheModule.getContext(),
1256	MDs: llvm::MDString::get(Context&: TheModule.getContext(),
1257	Str: CodeGenOpts.UniqueSourceFileIdentifier)));
1258	}
1259
1260	if (LangOpts.Sanitize.has(K: SanitizerKind::KCFI)) {
1261	getModule().addModuleFlag(Behavior: llvm::Module::Override, Key: "kcfi", Val: 1);
1262	// KCFI assumes patchable-function-prefix is the same for all indirectly
1263	// called functions. Store the expected offset for code generation.
1264	if (CodeGenOpts.PatchableFunctionEntryOffset)
1265	getModule().addModuleFlag(Behavior: llvm::Module::Override, Key: "kcfi-offset",
1266	Val: CodeGenOpts.PatchableFunctionEntryOffset);
1267	if (CodeGenOpts.SanitizeKcfiArity)
1268	getModule().addModuleFlag(Behavior: llvm::Module::Override, Key: "kcfi-arity", Val: 1);
1269	}
1270
1271	if (CodeGenOpts.CFProtectionReturn &&
1272	Target.checkCFProtectionReturnSupported(Diags&: getDiags())) {
1273	// Indicate that we want to instrument return control flow protection.
1274	getModule().addModuleFlag(Behavior: llvm::Module::Min, Key: "cf-protection-return",
1275	Val: 1);
1276	}
1277
1278	if (CodeGenOpts.CFProtectionBranch &&
1279	Target.checkCFProtectionBranchSupported(Diags&: getDiags())) {
1280	// Indicate that we want to instrument branch control flow protection.
1281	getModule().addModuleFlag(Behavior: llvm::Module::Min, Key: "cf-protection-branch",
1282	Val: 1);
1283
1284	auto Scheme = CodeGenOpts.getCFBranchLabelScheme();
1285	if (Target.checkCFBranchLabelSchemeSupported(Scheme, Diags&: getDiags())) {
1286	if (Scheme == CFBranchLabelSchemeKind::Default)
1287	Scheme = Target.getDefaultCFBranchLabelScheme();
1288	getModule().addModuleFlag(
1289	Behavior: llvm::Module::Error, Key: "cf-branch-label-scheme",
1290	Val: llvm::MDString::get(Context&: getLLVMContext(),
1291	Str: getCFBranchLabelSchemeFlagVal(Scheme)));
1292	}
1293	}
1294
1295	if (CodeGenOpts.FunctionReturnThunks)
1296	getModule().addModuleFlag(Behavior: llvm::Module::Override, Key: "function_return_thunk_extern", Val: 1);
1297
1298	if (CodeGenOpts.IndirectBranchCSPrefix)
1299	getModule().addModuleFlag(Behavior: llvm::Module::Override, Key: "indirect_branch_cs_prefix", Val: 1);
1300
1301	// Add module metadata for return address signing (ignoring
1302	// non-leaf/all) and stack tagging. These are actually turned on by function
1303	// attributes, but we use module metadata to emit build attributes. This is
1304	// needed for LTO, where the function attributes are inside bitcode
1305	// serialised into a global variable by the time build attributes are
1306	// emitted, so we can't access them. LTO objects could be compiled with
1307	// different flags therefore module flags are set to "Min" behavior to achieve
1308	// the same end result of the normal build where e.g BTI is off if any object
1309	// doesn't support it.
1310	if (Context.getTargetInfo().hasFeature(Feature: "ptrauth") &&
1311	LangOpts.getSignReturnAddressScope() !=
1312	LangOptions::SignReturnAddressScopeKind::None)
1313	getModule().addModuleFlag(Behavior: llvm::Module::Override,
1314	Key: "sign-return-address-buildattr", Val: 1);
1315	if (LangOpts.Sanitize.has(K: SanitizerKind::MemtagStack))
1316	getModule().addModuleFlag(Behavior: llvm::Module::Override,
1317	Key: "tag-stack-memory-buildattr", Val: 1);
1318
1319	if (T.isARM() || T.isThumb() || T.isAArch64()) {
1320	if (LangOpts.BranchTargetEnforcement)
1321	getModule().addModuleFlag(Behavior: llvm::Module::Min, Key: "branch-target-enforcement",
1322	Val: 1);
1323	if (LangOpts.BranchProtectionPAuthLR)
1324	getModule().addModuleFlag(Behavior: llvm::Module::Min, Key: "branch-protection-pauth-lr",
1325	Val: 1);
1326	if (LangOpts.GuardedControlStack)
1327	getModule().addModuleFlag(Behavior: llvm::Module::Min, Key: "guarded-control-stack", Val: 1);
1328	if (LangOpts.hasSignReturnAddress())
1329	getModule().addModuleFlag(Behavior: llvm::Module::Min, Key: "sign-return-address", Val: 1);
1330	if (LangOpts.isSignReturnAddressScopeAll())
1331	getModule().addModuleFlag(Behavior: llvm::Module::Min, Key: "sign-return-address-all",
1332	Val: 1);
1333	if (!LangOpts.isSignReturnAddressWithAKey())
1334	getModule().addModuleFlag(Behavior: llvm::Module::Min,
1335	Key: "sign-return-address-with-bkey", Val: 1);
1336
1337	if (LangOpts.PointerAuthELFGOT)
1338	getModule().addModuleFlag(Behavior: llvm::Module::Min, Key: "ptrauth-elf-got", Val: 1);
1339
1340	if (getTriple().isOSLinux()) {
1341	if (LangOpts.PointerAuthCalls)
1342	getModule().addModuleFlag(Behavior: llvm::Module::Min, Key: "ptrauth-sign-personality",
1343	Val: 1);
1344	assert(getTriple().isOSBinFormatELF());
1345	using namespace llvm::ELF;
1346	uint64_t PAuthABIVersion =
1347	(LangOpts.PointerAuthIntrinsics
1348	<< AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_INTRINSICS) |
1349	(LangOpts.PointerAuthCalls
1350	<< AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_CALLS) |
1351	(LangOpts.PointerAuthReturns
1352	<< AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_RETURNS) |
1353	(LangOpts.PointerAuthAuthTraps
1354	<< AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_AUTHTRAPS) |
1355	(LangOpts.PointerAuthVTPtrAddressDiscrimination
1356	<< AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_VPTRADDRDISCR) |
1357	(LangOpts.PointerAuthVTPtrTypeDiscrimination
1358	<< AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_VPTRTYPEDISCR) |
1359	(LangOpts.PointerAuthInitFini
1360	<< AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_INITFINI) |
1361	(LangOpts.PointerAuthInitFiniAddressDiscrimination
1362	<< AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_INITFINIADDRDISC) |
1363	(LangOpts.PointerAuthELFGOT
1364	<< AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_GOT) |
1365	(LangOpts.PointerAuthIndirectGotos
1366	<< AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_GOTOS) |
1367	(LangOpts.PointerAuthTypeInfoVTPtrDiscrimination
1368	<< AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_TYPEINFOVPTRDISCR) |
1369	(LangOpts.PointerAuthFunctionTypeDiscrimination
1370	<< AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_FPTRTYPEDISCR);
1371	static_assert(AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_FPTRTYPEDISCR ==
1372	AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_LAST,
1373	"Update when new enum items are defined");
1374	if (PAuthABIVersion != 0) {
1375	getModule().addModuleFlag(Behavior: llvm::Module::Error,
1376	Key: "aarch64-elf-pauthabi-platform",
1377	Val: AARCH64_PAUTH_PLATFORM_LLVM_LINUX);
1378	getModule().addModuleFlag(Behavior: llvm::Module::Error,
1379	Key: "aarch64-elf-pauthabi-version",
1380	Val: PAuthABIVersion);
1381	}
1382	}
1383	}
1384
1385	if (CodeGenOpts.StackClashProtector)
1386	getModule().addModuleFlag(
1387	Behavior: llvm::Module::Override, Key: "probe-stack",
1388	Val: llvm::MDString::get(Context&: TheModule.getContext(), Str: "inline-asm"));
1389
1390	if (CodeGenOpts.StackProbeSize && CodeGenOpts.StackProbeSize != 4096)
1391	getModule().addModuleFlag(Behavior: llvm::Module::Min, Key: "stack-probe-size",
1392	Val: CodeGenOpts.StackProbeSize);
1393
1394	if (!CodeGenOpts.MemoryProfileOutput.empty()) {
1395	llvm::LLVMContext &Ctx = TheModule.getContext();
1396	getModule().addModuleFlag(
1397	Behavior: llvm::Module::Error, Key: "MemProfProfileFilename",
1398	Val: llvm::MDString::get(Context&: Ctx, Str: CodeGenOpts.MemoryProfileOutput));
1399	}
1400
1401	if (LangOpts.CUDAIsDevice && getTriple().isNVPTX()) {
1402	// Indicate whether __nvvm_reflect should be configured to flush denormal
1403	// floating point values to 0. (This corresponds to its "__CUDA_FTZ"
1404	// property.)
1405	getModule().addModuleFlag(Behavior: llvm::Module::Override, Key: "nvvm-reflect-ftz",
1406	Val: CodeGenOpts.FP32DenormalMode.Output !=
1407	llvm::DenormalMode::IEEE);
1408	}
1409
1410	if (LangOpts.EHAsynch)
1411	getModule().addModuleFlag(Behavior: llvm::Module::Warning, Key: "eh-asynch", Val: 1);
1412
1413	// Emit Import Call section.
1414	if (CodeGenOpts.ImportCallOptimization)
1415	getModule().addModuleFlag(Behavior: llvm::Module::Warning, Key: "import-call-optimization",
1416	Val: 1);
1417
1418	// Enable unwind v2 (epilog).
1419	if (CodeGenOpts.getWinX64EHUnwindV2() != llvm::WinX64EHUnwindV2Mode::Disabled)
1420	getModule().addModuleFlag(
1421	Behavior: llvm::Module::Warning, Key: "winx64-eh-unwindv2",
1422	Val: static_cast<unsigned>(CodeGenOpts.getWinX64EHUnwindV2()));
1423
1424	// Indicate whether this Module was compiled with -fopenmp
1425	if (getLangOpts().OpenMP && !getLangOpts().OpenMPSimd)
1426	getModule().addModuleFlag(Behavior: llvm::Module::Max, Key: "openmp", Val: LangOpts.OpenMP);
1427	if (getLangOpts().OpenMPIsTargetDevice)
1428	getModule().addModuleFlag(Behavior: llvm::Module::Max, Key: "openmp-device",
1429	Val: LangOpts.OpenMP);
1430
1431	// Emit OpenCL specific module metadata: OpenCL/SPIR version.
1432	if (LangOpts.OpenCL || (LangOpts.CUDAIsDevice && getTriple().isSPIRV())) {
1433	EmitOpenCLMetadata();
1434	// Emit SPIR version.
1435	if (getTriple().isSPIR()) {
1436	// SPIR v2.0 s2.12 - The SPIR version used by the module is stored in the
1437	// opencl.spir.version named metadata.
1438	// C++ for OpenCL has a distinct mapping for version compatibility with
1439	// OpenCL.
1440	auto Version = LangOpts.getOpenCLCompatibleVersion();
1441	llvm::Metadata *SPIRVerElts[] = {
1442	llvm::ConstantAsMetadata::get(C: llvm::ConstantInt::get(
1443	Ty: Int32Ty, V: Version / 100)),
1444	llvm::ConstantAsMetadata::get(C: llvm::ConstantInt::get(
1445	Ty: Int32Ty, V: (Version / 100 > 1) ? 0 : 2))};
1446	llvm::NamedMDNode *SPIRVerMD =
1447	TheModule.getOrInsertNamedMetadata(Name: "opencl.spir.version");
1448	llvm::LLVMContext &Ctx = TheModule.getContext();
1449	SPIRVerMD->addOperand(M: llvm::MDNode::get(Context&: Ctx, MDs: SPIRVerElts));
1450	}
1451	}
1452
1453	// HLSL related end of code gen work items.
1454	if (LangOpts.HLSL)
1455	getHLSLRuntime().finishCodeGen();
1456
1457	if (uint32_t PLevel = Context.getLangOpts().PICLevel) {
1458	assert(PLevel < 3 && "Invalid PIC Level");
1459	getModule().setPICLevel(static_cast<llvm::PICLevel::Level>(PLevel));
1460	if (Context.getLangOpts().PIE)
1461	getModule().setPIELevel(static_cast<llvm::PIELevel::Level>(PLevel));
1462	}
1463
1464	if (getCodeGenOpts().CodeModel.size() > 0) {
1465	unsigned CM = llvm::StringSwitch<unsigned>(getCodeGenOpts().CodeModel)
1466	.Case(S: "tiny", Value: llvm::CodeModel::Tiny)
1467	.Case(S: "small", Value: llvm::CodeModel::Small)
1468	.Case(S: "kernel", Value: llvm::CodeModel::Kernel)
1469	.Case(S: "medium", Value: llvm::CodeModel::Medium)
1470	.Case(S: "large", Value: llvm::CodeModel::Large)
1471	.Default(Value: ~0u);
1472	if (CM != ~0u) {
1473	llvm::CodeModel::Model codeModel = static_cast<llvm::CodeModel::Model>(CM);
1474	getModule().setCodeModel(codeModel);
1475
1476	if ((CM == llvm::CodeModel::Medium || CM == llvm::CodeModel::Large) &&
1477	Context.getTargetInfo().getTriple().getArch() ==
1478	llvm::Triple::x86_64) {
1479	getModule().setLargeDataThreshold(getCodeGenOpts().LargeDataThreshold);
1480	}
1481	}
1482	}
1483
1484	if (CodeGenOpts.NoPLT)
1485	getModule().setRtLibUseGOT();
1486	if (getTriple().isOSBinFormatELF() &&
1487	CodeGenOpts.DirectAccessExternalData !=
1488	getModule().getDirectAccessExternalData()) {
1489	getModule().setDirectAccessExternalData(
1490	CodeGenOpts.DirectAccessExternalData);
1491	}
1492	if (CodeGenOpts.UnwindTables)
1493	getModule().setUwtable(llvm::UWTableKind(CodeGenOpts.UnwindTables));
1494
1495	switch (CodeGenOpts.getFramePointer()) {
1496	case CodeGenOptions::FramePointerKind::None:
1497	// 0 ("none") is the default.
1498	break;
1499	case CodeGenOptions::FramePointerKind::Reserved:
1500	getModule().setFramePointer(llvm::FramePointerKind::Reserved);
1501	break;
1502	case CodeGenOptions::FramePointerKind::NonLeaf:
1503	getModule().setFramePointer(llvm::FramePointerKind::NonLeaf);
1504	break;
1505	case CodeGenOptions::FramePointerKind::All:
1506	getModule().setFramePointer(llvm::FramePointerKind::All);
1507	break;
1508	}
1509
1510	SimplifyPersonality();
1511
1512	if (getCodeGenOpts().EmitDeclMetadata)
1513	EmitDeclMetadata();
1514
1515	if (getCodeGenOpts().CoverageNotesFile.size() ||
1516	getCodeGenOpts().CoverageDataFile.size())
1517	EmitCoverageFile();
1518
1519	if (CGDebugInfo *DI = getModuleDebugInfo())
1520	DI->finalize();
1521
1522	if (getCodeGenOpts().EmitVersionIdentMetadata)
1523	EmitVersionIdentMetadata();
1524
1525	if (!getCodeGenOpts().RecordCommandLine.empty())
1526	EmitCommandLineMetadata();
1527
1528	if (!getCodeGenOpts().StackProtectorGuard.empty())
1529	getModule().setStackProtectorGuard(getCodeGenOpts().StackProtectorGuard);
1530	if (!getCodeGenOpts().StackProtectorGuardReg.empty())
1531	getModule().setStackProtectorGuardReg(
1532	getCodeGenOpts().StackProtectorGuardReg);
1533	if (!getCodeGenOpts().StackProtectorGuardSymbol.empty())
1534	getModule().setStackProtectorGuardSymbol(
1535	getCodeGenOpts().StackProtectorGuardSymbol);
1536	if (getCodeGenOpts().StackProtectorGuardOffset != INT_MAX)
1537	getModule().setStackProtectorGuardOffset(
1538	getCodeGenOpts().StackProtectorGuardOffset);
1539	if (getCodeGenOpts().StackAlignment)
1540	getModule().setOverrideStackAlignment(getCodeGenOpts().StackAlignment);
1541	if (getCodeGenOpts().SkipRaxSetup)
1542	getModule().addModuleFlag(Behavior: llvm::Module::Override, Key: "SkipRaxSetup", Val: 1);
1543	if (getLangOpts().RegCall4)
1544	getModule().addModuleFlag(Behavior: llvm::Module::Override, Key: "RegCallv4", Val: 1);
1545
1546	if (getContext().getTargetInfo().getMaxTLSAlign())
1547	getModule().addModuleFlag(Behavior: llvm::Module::Error, Key: "MaxTLSAlign",
1548	Val: getContext().getTargetInfo().getMaxTLSAlign());
1549
1550	getTargetCodeGenInfo().emitTargetGlobals(CGM&: *this);
1551
1552	getTargetCodeGenInfo().emitTargetMetadata(CGM&: *this, MangledDeclNames);
1553
1554	EmitBackendOptionsMetadata(CodeGenOpts: getCodeGenOpts());
1555
1556	// If there is device offloading code embed it in the host now.
1557	EmbedObject(M: &getModule(), CGOpts: CodeGenOpts, Diags&: getDiags());
1558
1559	// Set visibility from DLL storage class
1560	// We do this at the end of LLVM IR generation; after any operation
1561	// that might affect the DLL storage class or the visibility, and
1562	// before anything that might act on these.
1563	setVisibilityFromDLLStorageClass(LO: LangOpts, M&: getModule());
1564
1565	// Check the tail call symbols are truly undefined.
1566	if (getTriple().isPPC() && !MustTailCallUndefinedGlobals.empty()) {
1567	for (auto &I : MustTailCallUndefinedGlobals) {
1568	if (!I.first->isDefined())
1569	getDiags().Report(Loc: I.second, DiagID: diag::err_ppc_impossible_musttail) << 2;
1570	else {
1571	StringRef MangledName = getMangledName(GD: GlobalDecl(I.first));
1572	llvm::GlobalValue *Entry = GetGlobalValue(Ref: MangledName);
1573	if (!Entry || Entry->isWeakForLinker() ||
1574	Entry->isDeclarationForLinker())
1575	getDiags().Report(Loc: I.second, DiagID: diag::err_ppc_impossible_musttail) << 2;
1576	}
1577	}
1578	}
1579	}
1580
1581	void CodeGenModule::EmitOpenCLMetadata() {
1582	// SPIR v2.0 s2.13 - The OpenCL version used by the module is stored in the
1583	// opencl.ocl.version named metadata node.
1584	// C++ for OpenCL has a distinct mapping for versions compatible with OpenCL.
1585	auto CLVersion = LangOpts.getOpenCLCompatibleVersion();
1586
1587	auto EmitVersion = [this](StringRef MDName, int Version) {
1588	llvm::Metadata *OCLVerElts[] = {
1589	llvm::ConstantAsMetadata::get(
1590	C: llvm::ConstantInt::get(Ty: Int32Ty, V: Version / 100)),
1591	llvm::ConstantAsMetadata::get(
1592	C: llvm::ConstantInt::get(Ty: Int32Ty, V: (Version % 100) / 10))};
1593	llvm::NamedMDNode *OCLVerMD = TheModule.getOrInsertNamedMetadata(Name: MDName);
1594	llvm::LLVMContext &Ctx = TheModule.getContext();
1595	OCLVerMD->addOperand(M: llvm::MDNode::get(Context&: Ctx, MDs: OCLVerElts));
1596	};
1597
1598	EmitVersion("opencl.ocl.version", CLVersion);
1599	if (LangOpts.OpenCLCPlusPlus) {
1600	// In addition to the OpenCL compatible version, emit the C++ version.
1601	EmitVersion("opencl.cxx.version", LangOpts.OpenCLCPlusPlusVersion);
1602	}
1603	}
1604
1605	void CodeGenModule::EmitBackendOptionsMetadata(
1606	const CodeGenOptions &CodeGenOpts) {
1607	if (getTriple().isRISCV()) {
1608	getModule().addModuleFlag(Behavior: llvm::Module::Min, Key: "SmallDataLimit",
1609	Val: CodeGenOpts.SmallDataLimit);
1610	}
1611	}
1612
1613	void CodeGenModule::UpdateCompletedType(const TagDecl *TD) {
1614	// Make sure that this type is translated.
1615	getTypes().UpdateCompletedType(TD);
1616	}
1617
1618	void CodeGenModule::RefreshTypeCacheForClass(const CXXRecordDecl *RD) {
1619	// Make sure that this type is translated.
1620	getTypes().RefreshTypeCacheForClass(RD);
1621	}
1622
1623	llvm::MDNode *CodeGenModule::getTBAATypeInfo(QualType QTy) {
1624	if (!TBAA)
1625	return nullptr;
1626	return TBAA->getTypeInfo(QTy);
1627	}
1628
1629	TBAAAccessInfo CodeGenModule::getTBAAAccessInfo(QualType AccessType) {
1630	if (!TBAA)
1631	return TBAAAccessInfo();
1632	if (getLangOpts().CUDAIsDevice) {
1633	// As CUDA builtin surface/texture types are replaced, skip generating TBAA
1634	// access info.
1635	if (AccessType->isCUDADeviceBuiltinSurfaceType()) {
1636	if (getTargetCodeGenInfo().getCUDADeviceBuiltinSurfaceDeviceType() !=
1637	nullptr)
1638	return TBAAAccessInfo();
1639	} else if (AccessType->isCUDADeviceBuiltinTextureType()) {
1640	if (getTargetCodeGenInfo().getCUDADeviceBuiltinTextureDeviceType() !=
1641	nullptr)
1642	return TBAAAccessInfo();
1643	}
1644	}
1645	return TBAA->getAccessInfo(AccessType);
1646	}
1647
1648	TBAAAccessInfo
1649	CodeGenModule::getTBAAVTablePtrAccessInfo(llvm::Type *VTablePtrType) {
1650	if (!TBAA)
1651	return TBAAAccessInfo();
1652	return TBAA->getVTablePtrAccessInfo(VTablePtrType);
1653	}
1654
1655	llvm::MDNode *CodeGenModule::getTBAAStructInfo(QualType QTy) {
1656	if (!TBAA)
1657	return nullptr;
1658	return TBAA->getTBAAStructInfo(QTy);
1659	}
1660
1661	llvm::MDNode *CodeGenModule::getTBAABaseTypeInfo(QualType QTy) {
1662	if (!TBAA)
1663	return nullptr;
1664	return TBAA->getBaseTypeInfo(QTy);
1665	}
1666
1667	llvm::MDNode *CodeGenModule::getTBAAAccessTagInfo(TBAAAccessInfo Info) {
1668	if (!TBAA)
1669	return nullptr;
1670	return TBAA->getAccessTagInfo(Info);
1671	}
1672
1673	TBAAAccessInfo CodeGenModule::mergeTBAAInfoForCast(TBAAAccessInfo SourceInfo,
1674	TBAAAccessInfo TargetInfo) {
1675	if (!TBAA)
1676	return TBAAAccessInfo();
1677	return TBAA->mergeTBAAInfoForCast(SourceInfo, TargetInfo);
1678	}
1679
1680	TBAAAccessInfo
1681	CodeGenModule::mergeTBAAInfoForConditionalOperator(TBAAAccessInfo InfoA,
1682	TBAAAccessInfo InfoB) {
1683	if (!TBAA)
1684	return TBAAAccessInfo();
1685	return TBAA->mergeTBAAInfoForConditionalOperator(InfoA, InfoB);
1686	}
1687
1688	TBAAAccessInfo
1689	CodeGenModule::mergeTBAAInfoForMemoryTransfer(TBAAAccessInfo DestInfo,
1690	TBAAAccessInfo SrcInfo) {
1691	if (!TBAA)
1692	return TBAAAccessInfo();
1693	return TBAA->mergeTBAAInfoForConditionalOperator(InfoA: DestInfo, InfoB: SrcInfo);
1694	}
1695
1696	void CodeGenModule::DecorateInstructionWithTBAA(llvm::Instruction *Inst,
1697	TBAAAccessInfo TBAAInfo) {
1698	if (llvm::MDNode *Tag = getTBAAAccessTagInfo(Info: TBAAInfo))
1699	Inst->setMetadata(KindID: llvm::LLVMContext::MD_tbaa, Node: Tag);
1700	}
1701
1702	void CodeGenModule::DecorateInstructionWithInvariantGroup(
1703	llvm::Instruction *I, const CXXRecordDecl *RD) {
1704	I->setMetadata(KindID: llvm::LLVMContext::MD_invariant_group,
1705	Node: llvm::MDNode::get(Context&: getLLVMContext(), MDs: {}));
1706	}
1707
1708	void CodeGenModule::Error(SourceLocation loc, StringRef message) {
1709	unsigned diagID = getDiags().getCustomDiagID(L: DiagnosticsEngine::Error, FormatString: "%0");
1710	getDiags().Report(Loc: Context.getFullLoc(Loc: loc), DiagID: diagID) << message;
1711	}
1712
1713	/// ErrorUnsupported - Print out an error that codegen doesn't support the
1714	/// specified stmt yet.
1715	void CodeGenModule::ErrorUnsupported(const Stmt *S, const char *Type) {
1716	unsigned DiagID = getDiags().getCustomDiagID(L: DiagnosticsEngine::Error,
1717	FormatString: "cannot compile this %0 yet");
1718	std::string Msg = Type;
1719	getDiags().Report(Loc: Context.getFullLoc(Loc: S->getBeginLoc()), DiagID)
1720	<< Msg << S->getSourceRange();
1721	}
1722
1723	/// ErrorUnsupported - Print out an error that codegen doesn't support the
1724	/// specified decl yet.
1725	void CodeGenModule::ErrorUnsupported(const Decl *D, const char *Type) {
1726	unsigned DiagID = getDiags().getCustomDiagID(L: DiagnosticsEngine::Error,
1727	FormatString: "cannot compile this %0 yet");
1728	std::string Msg = Type;
1729	getDiags().Report(Loc: Context.getFullLoc(Loc: D->getLocation()), DiagID) << Msg;
1730	}
1731
1732	void CodeGenModule::runWithSufficientStackSpace(SourceLocation Loc,
1733	llvm::function_ref<void()> Fn) {
1734	StackHandler.runWithSufficientStackSpace(Loc, Fn);
1735	}
1736
1737	llvm::ConstantInt *CodeGenModule::getSize(CharUnits size) {
1738	return llvm::ConstantInt::get(Ty: SizeTy, V: size.getQuantity());
1739	}
1740
1741	void CodeGenModule::setGlobalVisibility(llvm::GlobalValue *GV,
1742	const NamedDecl *D) const {
1743	// Internal definitions always have default visibility.
1744	if (GV->hasLocalLinkage()) {
1745	GV->setVisibility(llvm::GlobalValue::DefaultVisibility);
1746	return;
1747	}
1748	if (!D)
1749	return;
1750
1751	// Set visibility for definitions, and for declarations if requested globally
1752	// or set explicitly.
1753	LinkageInfo LV = D->getLinkageAndVisibility();
1754
1755	// OpenMP declare target variables must be visible to the host so they can
1756	// be registered. We require protected visibility unless the variable has
1757	// the DT_nohost modifier and does not need to be registered.
1758	if (Context.getLangOpts().OpenMP &&
1759	Context.getLangOpts().OpenMPIsTargetDevice && isa<VarDecl>(Val: D) &&
1760	D->hasAttr<OMPDeclareTargetDeclAttr>() &&
1761	D->getAttr<OMPDeclareTargetDeclAttr>()->getDevType() !=
1762	OMPDeclareTargetDeclAttr::DT_NoHost &&
1763	LV.getVisibility() == HiddenVisibility) {
1764	GV->setVisibility(llvm::GlobalValue::ProtectedVisibility);
1765	return;
1766	}
1767
1768	if (Context.getLangOpts().HLSL && !D->isInExportDeclContext()) {
1769	GV->setVisibility(llvm::GlobalValue::HiddenVisibility);
1770	return;
1771	}
1772
1773	if (GV->hasDLLExportStorageClass() || GV->hasDLLImportStorageClass()) {
1774	// Reject incompatible dlllstorage and visibility annotations.
1775	if (!LV.isVisibilityExplicit())
1776	return;
1777	if (GV->hasDLLExportStorageClass()) {
1778	if (LV.getVisibility() == HiddenVisibility)
1779	getDiags().Report(Loc: D->getLocation(),
1780	DiagID: diag::err_hidden_visibility_dllexport);
1781	} else if (LV.getVisibility() != DefaultVisibility) {
1782	getDiags().Report(Loc: D->getLocation(),
1783	DiagID: diag::err_non_default_visibility_dllimport);
1784	}
1785	return;
1786	}
1787
1788	if (LV.isVisibilityExplicit() || getLangOpts().SetVisibilityForExternDecls ||
1789	!GV->isDeclarationForLinker())
1790	GV->setVisibility(GetLLVMVisibility(V: LV.getVisibility()));
1791	}
1792
1793	static bool shouldAssumeDSOLocal(const CodeGenModule &CGM,
1794	llvm::GlobalValue *GV) {
1795	if (GV->hasLocalLinkage())
1796	return true;
1797
1798	if (!GV->hasDefaultVisibility() && !GV->hasExternalWeakLinkage())
1799	return true;
1800
1801	// DLLImport explicitly marks the GV as external.
1802	if (GV->hasDLLImportStorageClass())
1803	return false;
1804
1805	const llvm::Triple &TT = CGM.getTriple();
1806	const auto &CGOpts = CGM.getCodeGenOpts();
1807	if (TT.isOSCygMing()) {
1808	// In MinGW, variables without DLLImport can still be automatically
1809	// imported from a DLL by the linker; don't mark variables that
1810	// potentially could come from another DLL as DSO local.
1811
1812	// With EmulatedTLS, TLS variables can be autoimported from other DLLs
1813	// (and this actually happens in the public interface of libstdc++), so
1814	// such variables can't be marked as DSO local. (Native TLS variables
1815	// can't be dllimported at all, though.)
1816	if (GV->isDeclarationForLinker() && isa<llvm::GlobalVariable>(Val: GV) &&
1817	(!GV->isThreadLocal() || CGM.getCodeGenOpts().EmulatedTLS) &&
1818	CGOpts.AutoImport)
1819	return false;
1820	}
1821
1822	// On COFF, don't mark 'extern_weak' symbols as DSO local. If these symbols
1823	// remain unresolved in the link, they can be resolved to zero, which is
1824	// outside the current DSO.
1825	if (TT.isOSBinFormatCOFF() && GV->hasExternalWeakLinkage())
1826	return false;
1827
1828	// Every other GV is local on COFF.
1829	// Make an exception for windows OS in the triple: Some firmware builds use
1830	// *-win32-macho triples. This (accidentally?) produced windows relocations
1831	// without GOT tables in older clang versions; Keep this behaviour.
1832	// FIXME: even thread local variables?
1833	if (TT.isOSBinFormatCOFF() || (TT.isOSWindows() && TT.isOSBinFormatMachO()))
1834	return true;
1835
1836	// Only handle COFF and ELF for now.
1837	if (!TT.isOSBinFormatELF())
1838	return false;
1839
1840	// If this is not an executable, don't assume anything is local.
1841	llvm::Reloc::Model RM = CGOpts.RelocationModel;
1842	const auto &LOpts = CGM.getLangOpts();
1843	if (RM != llvm::Reloc::Static && !LOpts.PIE) {
1844	// On ELF, if -fno-semantic-interposition is specified and the target
1845	// supports local aliases, there will be neither CC1
1846	// -fsemantic-interposition nor -fhalf-no-semantic-interposition. Set
1847	// dso_local on the function if using a local alias is preferable (can avoid
1848	// PLT indirection).
1849	if (!(isa<llvm::Function>(Val: GV) && GV->canBenefitFromLocalAlias()))
1850	return false;
1851	return !(CGM.getLangOpts().SemanticInterposition ||
1852	CGM.getLangOpts().HalfNoSemanticInterposition);
1853	}
1854
1855	// A definition cannot be preempted from an executable.
1856	if (!GV->isDeclarationForLinker())
1857	return true;
1858
1859	// Most PIC code sequences that assume that a symbol is local cannot produce a
1860	// 0 if it turns out the symbol is undefined. While this is ABI and relocation
1861	// depended, it seems worth it to handle it here.
1862	if (RM == llvm::Reloc::PIC_ && GV->hasExternalWeakLinkage())
1863	return false;
1864
1865	// PowerPC64 prefers TOC indirection to avoid copy relocations.
1866	if (TT.isPPC64())
1867	return false;
1868
1869	if (CGOpts.DirectAccessExternalData) {
1870	// If -fdirect-access-external-data (default for -fno-pic), set dso_local
1871	// for non-thread-local variables. If the symbol is not defined in the
1872	// executable, a copy relocation will be needed at link time. dso_local is
1873	// excluded for thread-local variables because they generally don't support
1874	// copy relocations.
1875	if (auto *Var = dyn_cast<llvm::GlobalVariable>(Val: GV))
1876	if (!Var->isThreadLocal())
1877	return true;
1878
1879	// -fno-pic sets dso_local on a function declaration to allow direct
1880	// accesses when taking its address (similar to a data symbol). If the
1881	// function is not defined in the executable, a canonical PLT entry will be
1882	// needed at link time. -fno-direct-access-external-data can avoid the
1883	// canonical PLT entry. We don't generalize this condition to -fpie/-fpic as
1884	// it could just cause trouble without providing perceptible benefits.
1885	if (isa<llvm::Function>(Val: GV) && !CGOpts.NoPLT && RM == llvm::Reloc::Static)
1886	return true;
1887	}
1888
1889	// If we can use copy relocations we can assume it is local.
1890
1891	// Otherwise don't assume it is local.
1892	return false;
1893	}
1894
1895	void CodeGenModule::setDSOLocal(llvm::GlobalValue *GV) const {
1896	GV->setDSOLocal(shouldAssumeDSOLocal(CGM: *this, GV));
1897	}
1898
1899	void CodeGenModule::setDLLImportDLLExport(llvm::GlobalValue *GV,
1900	GlobalDecl GD) const {
1901	const auto *D = dyn_cast<NamedDecl>(Val: GD.getDecl());
1902	// C++ destructors have a few C++ ABI specific special cases.
1903	if (const auto *Dtor = dyn_cast_or_null<CXXDestructorDecl>(Val: D)) {
1904	getCXXABI().setCXXDestructorDLLStorage(GV, Dtor, DT: GD.getDtorType());
1905	return;
1906	}
1907	setDLLImportDLLExport(GV, D);
1908	}
1909
1910	void CodeGenModule::setDLLImportDLLExport(llvm::GlobalValue *GV,
1911	const NamedDecl *D) const {
1912	if (D && D->isExternallyVisible()) {
1913	if (D->hasAttr<DLLImportAttr>())
1914	GV->setDLLStorageClass(llvm::GlobalVariable::DLLImportStorageClass);
1915	else if ((D->hasAttr<DLLExportAttr>() ||
1916	shouldMapVisibilityToDLLExport(D)) &&
1917	!GV->isDeclarationForLinker())
1918	GV->setDLLStorageClass(llvm::GlobalVariable::DLLExportStorageClass);
1919	}
1920	}
1921
1922	void CodeGenModule::setGVProperties(llvm::GlobalValue *GV,
1923	GlobalDecl GD) const {
1924	setDLLImportDLLExport(GV, GD);
1925	setGVPropertiesAux(GV, D: dyn_cast<NamedDecl>(Val: GD.getDecl()));
1926	}
1927
1928	void CodeGenModule::setGVProperties(llvm::GlobalValue *GV,
1929	const NamedDecl *D) const {
1930	setDLLImportDLLExport(GV, D);
1931	setGVPropertiesAux(GV, D);
1932	}
1933
1934	void CodeGenModule::setGVPropertiesAux(llvm::GlobalValue *GV,
1935	const NamedDecl *D) const {
1936	setGlobalVisibility(GV, D);
1937	setDSOLocal(GV);
1938	GV->setPartition(CodeGenOpts.SymbolPartition);
1939	}
1940
1941	static llvm::GlobalVariable::ThreadLocalMode GetLLVMTLSModel(StringRef S) {
1942	return llvm::StringSwitch<llvm::GlobalVariable::ThreadLocalMode>(S)
1943	.Case(S: "global-dynamic", Value: llvm::GlobalVariable::GeneralDynamicTLSModel)
1944	.Case(S: "local-dynamic", Value: llvm::GlobalVariable::LocalDynamicTLSModel)
1945	.Case(S: "initial-exec", Value: llvm::GlobalVariable::InitialExecTLSModel)
1946	.Case(S: "local-exec", Value: llvm::GlobalVariable::LocalExecTLSModel);
1947	}
1948
1949	llvm::GlobalVariable::ThreadLocalMode
1950	CodeGenModule::GetDefaultLLVMTLSModel() const {
1951	switch (CodeGenOpts.getDefaultTLSModel()) {
1952	case CodeGenOptions::GeneralDynamicTLSModel:
1953	return llvm::GlobalVariable::GeneralDynamicTLSModel;
1954	case CodeGenOptions::LocalDynamicTLSModel:
1955	return llvm::GlobalVariable::LocalDynamicTLSModel;
1956	case CodeGenOptions::InitialExecTLSModel:
1957	return llvm::GlobalVariable::InitialExecTLSModel;
1958	case CodeGenOptions::LocalExecTLSModel:
1959	return llvm::GlobalVariable::LocalExecTLSModel;
1960	}
1961	llvm_unreachable("Invalid TLS model!");
1962	}
1963
1964	void CodeGenModule::setTLSMode(llvm::GlobalValue *GV, const VarDecl &D) const {
1965	assert(D.getTLSKind() && "setting TLS mode on non-TLS var!");
1966
1967	llvm::GlobalValue::ThreadLocalMode TLM;
1968	TLM = GetDefaultLLVMTLSModel();
1969
1970	// Override the TLS model if it is explicitly specified.
1971	if (const TLSModelAttr *Attr = D.getAttr<TLSModelAttr>()) {
1972	TLM = GetLLVMTLSModel(S: Attr->getModel());
1973	}
1974
1975	GV->setThreadLocalMode(TLM);
1976	}
1977
1978	static std::string getCPUSpecificMangling(const CodeGenModule &CGM,
1979	StringRef Name) {
1980	const TargetInfo &Target = CGM.getTarget();
1981	return (Twine('.') + Twine(Target.CPUSpecificManglingCharacter(Name))).str();
1982	}
1983
1984	static void AppendCPUSpecificCPUDispatchMangling(const CodeGenModule &CGM,
1985	const CPUSpecificAttr *Attr,
1986	unsigned CPUIndex,
1987	raw_ostream &Out) {
1988	// cpu_specific gets the current name, dispatch gets the resolver if IFunc is
1989	// supported.
1990	if (Attr)
1991	Out << getCPUSpecificMangling(CGM, Name: Attr->getCPUName(Index: CPUIndex)->getName());
1992	else if (CGM.getTarget().supportsIFunc())
1993	Out << ".resolver";
1994	}
1995
1996	// Returns true if GD is a function decl with internal linkage and
1997	// needs a unique suffix after the mangled name.
1998	static bool isUniqueInternalLinkageDecl(GlobalDecl GD,
1999	CodeGenModule &CGM) {
2000	const Decl *D = GD.getDecl();
2001	return !CGM.getModuleNameHash().empty() && isa<FunctionDecl>(Val: D) &&
2002	(CGM.getFunctionLinkage(GD) == llvm::GlobalValue::InternalLinkage);
2003	}
2004
2005	static std::string getMangledNameImpl(CodeGenModule &CGM, GlobalDecl GD,
2006	const NamedDecl *ND,
2007	bool OmitMultiVersionMangling = false) {
2008	SmallString<256> Buffer;
2009	llvm::raw_svector_ostream Out(Buffer);
2010	MangleContext &MC = CGM.getCXXABI().getMangleContext();
2011	if (!CGM.getModuleNameHash().empty())
2012	MC.needsUniqueInternalLinkageNames();
2013	bool ShouldMangle = MC.shouldMangleDeclName(D: ND);
2014	if (ShouldMangle)
2015	MC.mangleName(GD: GD.getWithDecl(D: ND), Out);
2016	else {
2017	IdentifierInfo *II = ND->getIdentifier();
2018	assert(II && "Attempt to mangle unnamed decl.");
2019	const auto *FD = dyn_cast<FunctionDecl>(Val: ND);
2020
2021	if (FD &&
2022	FD->getType()->castAs<FunctionType>()->getCallConv() == CC_X86RegCall) {
2023	if (CGM.getLangOpts().RegCall4)
2024	Out << "__regcall4__" << II->getName();
2025	else
2026	Out << "__regcall3__" << II->getName();
2027	} else if (FD && FD->hasAttr<CUDAGlobalAttr>() &&
2028	GD.getKernelReferenceKind() == KernelReferenceKind::Stub) {
2029	Out << "__device_stub__" << II->getName();
2030	} else if (FD &&
2031	DeviceKernelAttr::isOpenCLSpelling(
2032	A: FD->getAttr<DeviceKernelAttr>()) &&
2033	GD.getKernelReferenceKind() == KernelReferenceKind::Stub) {
2034	Out << "__clang_ocl_kern_imp_" << II->getName();
2035	} else {
2036	Out << II->getName();
2037	}
2038	}
2039
2040	// Check if the module name hash should be appended for internal linkage
2041	// symbols. This should come before multi-version target suffixes are
2042	// appended. This is to keep the name and module hash suffix of the
2043	// internal linkage function together. The unique suffix should only be
2044	// added when name mangling is done to make sure that the final name can
2045	// be properly demangled. For example, for C functions without prototypes,
2046	// name mangling is not done and the unique suffix should not be appeneded
2047	// then.
2048	if (ShouldMangle && isUniqueInternalLinkageDecl(GD, CGM)) {
2049	assert(CGM.getCodeGenOpts().UniqueInternalLinkageNames &&
2050	"Hash computed when not explicitly requested");
2051	Out << CGM.getModuleNameHash();
2052	}
2053
2054	if (const auto *FD = dyn_cast<FunctionDecl>(Val: ND))
2055	if (FD->isMultiVersion() && !OmitMultiVersionMangling) {
2056	switch (FD->getMultiVersionKind()) {
2057	case MultiVersionKind::CPUDispatch:
2058	case MultiVersionKind::CPUSpecific:
2059	AppendCPUSpecificCPUDispatchMangling(CGM,
2060	Attr: FD->getAttr<CPUSpecificAttr>(),
2061	CPUIndex: GD.getMultiVersionIndex(), Out);
2062	break;
2063	case MultiVersionKind::Target: {
2064	auto *Attr = FD->getAttr<TargetAttr>();
2065	assert(Attr && "Expected TargetAttr to be present "
2066	"for attribute mangling");
2067	const ABIInfo &Info = CGM.getTargetCodeGenInfo().getABIInfo();
2068	Info.appendAttributeMangling(Attr, Out);
2069	break;
2070	}
2071	case MultiVersionKind::TargetVersion: {
2072	auto *Attr = FD->getAttr<TargetVersionAttr>();
2073	assert(Attr && "Expected TargetVersionAttr to be present "
2074	"for attribute mangling");
2075	const ABIInfo &Info = CGM.getTargetCodeGenInfo().getABIInfo();
2076	Info.appendAttributeMangling(Attr, Out);
2077	break;
2078	}
2079	case MultiVersionKind::TargetClones: {
2080	auto *Attr = FD->getAttr<TargetClonesAttr>();
2081	assert(Attr && "Expected TargetClonesAttr to be present "
2082	"for attribute mangling");
2083	unsigned Index = GD.getMultiVersionIndex();
2084	const ABIInfo &Info = CGM.getTargetCodeGenInfo().getABIInfo();
2085	Info.appendAttributeMangling(Attr, Index, Out);
2086	break;
2087	}
2088	case MultiVersionKind::None:
2089	llvm_unreachable("None multiversion type isn't valid here");
2090	}
2091	}
2092
2093	// Make unique name for device side static file-scope variable for HIP.
2094	if (CGM.getContext().shouldExternalize(D: ND) &&
2095	CGM.getLangOpts().GPURelocatableDeviceCode &&
2096	CGM.getLangOpts().CUDAIsDevice)
2097	CGM.printPostfixForExternalizedDecl(OS&: Out, D: ND);
2098
2099	return std::string(Out.str());
2100	}
2101
2102	void CodeGenModule::UpdateMultiVersionNames(GlobalDecl GD,
2103	const FunctionDecl *FD,
2104	StringRef &CurName) {
2105	if (!FD->isMultiVersion())
2106	return;
2107
2108	// Get the name of what this would be without the 'target' attribute. This
2109	// allows us to lookup the version that was emitted when this wasn't a
2110	// multiversion function.
2111	std::string NonTargetName =
2112	getMangledNameImpl(CGM&: *this, GD, ND: FD, /*OmitMultiVersionMangling=*/true);
2113	GlobalDecl OtherGD;
2114	if (lookupRepresentativeDecl(MangledName: NonTargetName, Result&: OtherGD)) {
2115	assert(OtherGD.getCanonicalDecl()
2116	.getDecl()
2117	->getAsFunction()
2118	->isMultiVersion() &&
2119	"Other GD should now be a multiversioned function");
2120	// OtherFD is the version of this function that was mangled BEFORE
2121	// becoming a MultiVersion function. It potentially needs to be updated.
2122	const FunctionDecl *OtherFD = OtherGD.getCanonicalDecl()
2123	.getDecl()
2124	->getAsFunction()
2125	->getMostRecentDecl();
2126	std::string OtherName = getMangledNameImpl(CGM&: *this, GD: OtherGD, ND: OtherFD);
2127	// This is so that if the initial version was already the 'default'
2128	// version, we don't try to update it.
2129	if (OtherName != NonTargetName) {
2130	// Remove instead of erase, since others may have stored the StringRef
2131	// to this.
2132	const auto ExistingRecord = Manglings.find(Key: NonTargetName);
2133	if (ExistingRecord != std::end(cont&: Manglings))
2134	Manglings.remove(KeyValue: &(*ExistingRecord));
2135	auto Result = Manglings.insert(KV: std::make_pair(x&: OtherName, y&: OtherGD));
2136	StringRef OtherNameRef = MangledDeclNames[OtherGD.getCanonicalDecl()] =
2137	Result.first->first();
2138	// If this is the current decl is being created, make sure we update the name.
2139	if (GD.getCanonicalDecl() == OtherGD.getCanonicalDecl())
2140	CurName = OtherNameRef;
2141	if (llvm::GlobalValue *Entry = GetGlobalValue(Ref: NonTargetName))
2142	Entry->setName(OtherName);
2143	}
2144	}
2145	}
2146
2147	StringRef CodeGenModule::getMangledName(GlobalDecl GD) {
2148	GlobalDecl CanonicalGD = GD.getCanonicalDecl();
2149
2150	// Some ABIs don't have constructor variants. Make sure that base and
2151	// complete constructors get mangled the same.
2152	if (const auto *CD = dyn_cast<CXXConstructorDecl>(Val: CanonicalGD.getDecl())) {
2153	if (!getTarget().getCXXABI().hasConstructorVariants()) {
2154	CXXCtorType OrigCtorType = GD.getCtorType();
2155	assert(OrigCtorType == Ctor_Base || OrigCtorType == Ctor_Complete);
2156	if (OrigCtorType == Ctor_Base)
2157	CanonicalGD = GlobalDecl(CD, Ctor_Complete);
2158	}
2159	}
2160
2161	// In CUDA/HIP device compilation with -fgpu-rdc, the mangled name of a
2162	// static device variable depends on whether the variable is referenced by
2163	// a host or device host function. Therefore the mangled name cannot be
2164	// cached.
2165	if (!LangOpts.CUDAIsDevice || !getContext().mayExternalize(D: GD.getDecl())) {
2166	auto FoundName = MangledDeclNames.find(Key: CanonicalGD);
2167	if (FoundName != MangledDeclNames.end())
2168	return FoundName->second;
2169	}
2170
2171	// Keep the first result in the case of a mangling collision.
2172	const auto *ND = cast<NamedDecl>(Val: GD.getDecl());
2173	std::string MangledName = getMangledNameImpl(CGM&: *this, GD, ND);
2174
2175	// Ensure either we have different ABIs between host and device compilations,
2176	// says host compilation following MSVC ABI but device compilation follows
2177	// Itanium C++ ABI or, if they follow the same ABI, kernel names after
2178	// mangling should be the same after name stubbing. The later checking is
2179	// very important as the device kernel name being mangled in host-compilation
2180	// is used to resolve the device binaries to be executed. Inconsistent naming
2181	// result in undefined behavior. Even though we cannot check that naming
2182	// directly between host- and device-compilations, the host- and
2183	// device-mangling in host compilation could help catching certain ones.
2184	assert(!isa<FunctionDecl>(ND) || !ND->hasAttr<CUDAGlobalAttr>() ||
2185	getContext().shouldExternalize(ND) || getLangOpts().CUDAIsDevice ||
2186	(getContext().getAuxTargetInfo() &&
2187	(getContext().getAuxTargetInfo()->getCXXABI() !=
2188	getContext().getTargetInfo().getCXXABI())) ||
2189	getCUDARuntime().getDeviceSideName(ND) ==
2190	getMangledNameImpl(
2191	*this,
2192	GD.getWithKernelReferenceKind(KernelReferenceKind::Kernel),
2193	ND));
2194
2195	// This invariant should hold true in the future.
2196	// Prior work:
2197	// https://discourse.llvm.org/t/rfc-clang-diagnostic-for-demangling-failures/82835/8
2198	// https://github.com/llvm/llvm-project/issues/111345
2199	// assert(!((StringRef(MangledName).starts_with("_Z") ||
2200	// StringRef(MangledName).starts_with("?")) &&
2201	// !GD.getDecl()->hasAttr<AsmLabelAttr>() &&
2202	// llvm::demangle(MangledName) == MangledName) &&
2203	// "LLVM demangler must demangle clang-generated names");
2204
2205	auto Result = Manglings.insert(KV: std::make_pair(x&: MangledName, y&: GD));
2206	return MangledDeclNames[CanonicalGD] = Result.first->first();
2207	}
2208
2209	StringRef CodeGenModule::getBlockMangledName(GlobalDecl GD,
2210	const BlockDecl *BD) {
2211	MangleContext &MangleCtx = getCXXABI().getMangleContext();
2212	const Decl *D = GD.getDecl();
2213
2214	SmallString<256> Buffer;
2215	llvm::raw_svector_ostream Out(Buffer);
2216	if (!D)
2217	MangleCtx.mangleGlobalBlock(BD,
2218	ID: dyn_cast_or_null<VarDecl>(Val: initializedGlobalDecl.getDecl()), Out);
2219	else if (const auto *CD = dyn_cast<CXXConstructorDecl>(Val: D))
2220	MangleCtx.mangleCtorBlock(CD, CT: GD.getCtorType(), BD, Out);
2221	else if (const auto *DD = dyn_cast<CXXDestructorDecl>(Val: D))
2222	MangleCtx.mangleDtorBlock(CD: DD, DT: GD.getDtorType(), BD, Out);
2223	else
2224	MangleCtx.mangleBlock(DC: cast<DeclContext>(Val: D), BD, Out);
2225
2226	auto Result = Manglings.insert(KV: std::make_pair(x: Out.str(), y&: BD));
2227	return Result.first->first();
2228	}
2229
2230	const GlobalDecl CodeGenModule::getMangledNameDecl(StringRef Name) {
2231	auto it = MangledDeclNames.begin();
2232	while (it != MangledDeclNames.end()) {
2233	if (it->second == Name)
2234	return it->first;
2235	it++;
2236	}
2237	return GlobalDecl();
2238	}
2239
2240	llvm::GlobalValue *CodeGenModule::GetGlobalValue(StringRef Name) {
2241	return getModule().getNamedValue(Name);
2242	}
2243
2244	/// AddGlobalCtor - Add a function to the list that will be called before
2245	/// main() runs.
2246	void CodeGenModule::AddGlobalCtor(llvm::Function *Ctor, int Priority,
2247	unsigned LexOrder,
2248	llvm::Constant *AssociatedData) {
2249	// FIXME: Type coercion of void()* types.
2250	GlobalCtors.push_back(x: Structor(Priority, LexOrder, Ctor, AssociatedData));
2251	}
2252
2253	/// AddGlobalDtor - Add a function to the list that will be called
2254	/// when the module is unloaded.
2255	void CodeGenModule::AddGlobalDtor(llvm::Function *Dtor, int Priority,
2256	bool IsDtorAttrFunc) {
2257	if (CodeGenOpts.RegisterGlobalDtorsWithAtExit &&
2258	(!getContext().getTargetInfo().getTriple().isOSAIX() || IsDtorAttrFunc)) {
2259	DtorsUsingAtExit[Priority].push_back(NewVal: Dtor);
2260	return;
2261	}
2262
2263	// FIXME: Type coercion of void()* types.
2264	GlobalDtors.push_back(x: Structor(Priority, ~0U, Dtor, nullptr));
2265	}
2266
2267	void CodeGenModule::EmitCtorList(CtorList &Fns, const char *GlobalName) {
2268	if (Fns.empty()) return;
2269
2270	const PointerAuthSchema &InitFiniAuthSchema =
2271	getCodeGenOpts().PointerAuth.InitFiniPointers;
2272
2273	// Ctor function type is ptr.
2274	llvm::PointerType *PtrTy = llvm::PointerType::get(
2275	C&: getLLVMContext(), AddressSpace: TheModule.getDataLayout().getProgramAddressSpace());
2276
2277	// Get the type of a ctor entry, { i32, ptr, ptr }.
2278	llvm::StructType *CtorStructTy = llvm::StructType::get(elt1: Int32Ty, elts: PtrTy, elts: PtrTy);
2279
2280	// Construct the constructor and destructor arrays.
2281	ConstantInitBuilder Builder(*this);
2282	auto Ctors = Builder.beginArray(eltTy: CtorStructTy);
2283	for (const auto &I : Fns) {
2284	auto Ctor = Ctors.beginStruct(ty: CtorStructTy);
2285	Ctor.addInt(intTy: Int32Ty, value: I.Priority);
2286	if (InitFiniAuthSchema) {
2287	llvm::Constant *StorageAddress =
2288	(InitFiniAuthSchema.isAddressDiscriminated()
2289	? llvm::ConstantExpr::getIntToPtr(
2290	C: llvm::ConstantInt::get(
2291	Ty: IntPtrTy,
2292	V: llvm::ConstantPtrAuth::AddrDiscriminator_CtorsDtors),
2293	Ty: PtrTy)
2294	: nullptr);
2295	llvm::Constant *SignedCtorPtr = getConstantSignedPointer(
2296	Pointer: I.Initializer, Key: InitFiniAuthSchema.getKey(), StorageAddress,
2297	OtherDiscriminator: llvm::ConstantInt::get(
2298	Ty: SizeTy, V: InitFiniAuthSchema.getConstantDiscrimination()));
2299	Ctor.add(value: SignedCtorPtr);
2300	} else {
2301	Ctor.add(value: I.Initializer);
2302	}
2303	if (I.AssociatedData)
2304	Ctor.add(value: I.AssociatedData);
2305	else
2306	Ctor.addNullPointer(ptrTy: PtrTy);
2307	Ctor.finishAndAddTo(parent&: Ctors);
2308	}
2309
2310	auto List = Ctors.finishAndCreateGlobal(args&: GlobalName, args: getPointerAlign(),
2311	/*constant*/ args: false,
2312	args: llvm::GlobalValue::AppendingLinkage);
2313
2314	// The LTO linker doesn't seem to like it when we set an alignment
2315	// on appending variables. Take it off as a workaround.
2316	List->setAlignment(std::nullopt);
2317
2318	Fns.clear();
2319	}
2320
2321	llvm::GlobalValue::LinkageTypes
2322	CodeGenModule::getFunctionLinkage(GlobalDecl GD) {
2323	const auto *D = cast<FunctionDecl>(Val: GD.getDecl());
2324
2325	GVALinkage Linkage = getContext().GetGVALinkageForFunction(FD: D);
2326
2327	if (const auto *Dtor = dyn_cast<CXXDestructorDecl>(Val: D))
2328	return getCXXABI().getCXXDestructorLinkage(Linkage, Dtor, DT: GD.getDtorType());
2329
2330	return getLLVMLinkageForDeclarator(D, Linkage);
2331	}
2332
2333	llvm::ConstantInt *CodeGenModule::CreateCrossDsoCfiTypeId(llvm::Metadata *MD) {
2334	llvm::MDString *MDS = dyn_cast<llvm::MDString>(Val: MD);
2335	if (!MDS) return nullptr;
2336
2337	return llvm::ConstantInt::get(Ty: Int64Ty, V: llvm::MD5Hash(Str: MDS->getString()));
2338	}
2339
2340	llvm::ConstantInt *CodeGenModule::CreateKCFITypeId(QualType T) {
2341	if (auto *FnType = T->getAs<FunctionProtoType>())
2342	T = getContext().getFunctionType(
2343	ResultTy: FnType->getReturnType(), Args: FnType->getParamTypes(),
2344	EPI: FnType->getExtProtoInfo().withExceptionSpec(ESI: EST_None));
2345
2346	std::string OutName;
2347	llvm::raw_string_ostream Out(OutName);
2348	getCXXABI().getMangleContext().mangleCanonicalTypeName(
2349	T, Out, NormalizeIntegers: getCodeGenOpts().SanitizeCfiICallNormalizeIntegers);
2350
2351	if (getCodeGenOpts().SanitizeCfiICallNormalizeIntegers)
2352	Out << ".normalized";
2353
2354	return llvm::ConstantInt::get(Ty: Int32Ty,
2355	V: static_cast<uint32_t>(llvm::xxHash64(Data: OutName)));
2356	}
2357
2358	void CodeGenModule::SetLLVMFunctionAttributes(GlobalDecl GD,
2359	const CGFunctionInfo &Info,
2360	llvm::Function *F, bool IsThunk) {
2361	unsigned CallingConv;
2362	llvm::AttributeList PAL;
2363	ConstructAttributeList(Name: F->getName(), Info, CalleeInfo: GD, Attrs&: PAL, CallingConv,
2364	/*AttrOnCallSite=*/false, IsThunk);
2365	if (CallingConv == llvm::CallingConv::X86_VectorCall &&
2366	getTarget().getTriple().isWindowsArm64EC()) {
2367	SourceLocation Loc;
2368	if (const Decl *D = GD.getDecl())
2369	Loc = D->getLocation();
2370
2371	Error(loc: Loc, message: "__vectorcall calling convention is not currently supported");
2372	}
2373	F->setAttributes(PAL);
2374	F->setCallingConv(static_cast<llvm::CallingConv::ID>(CallingConv));
2375	}
2376
2377	static void removeImageAccessQualifier(std::string& TyName) {
2378	std::string ReadOnlyQual("__read_only");
2379	std::string::size_type ReadOnlyPos = TyName.find(str: ReadOnlyQual);
2380	if (ReadOnlyPos != std::string::npos)
2381	// "+ 1" for the space after access qualifier.
2382	TyName.erase(pos: ReadOnlyPos, n: ReadOnlyQual.size() + 1);
2383	else {
2384	std::string WriteOnlyQual("__write_only");
2385	std::string::size_type WriteOnlyPos = TyName.find(str: WriteOnlyQual);
2386	if (WriteOnlyPos != std::string::npos)
2387	TyName.erase(pos: WriteOnlyPos, n: WriteOnlyQual.size() + 1);
2388	else {
2389	std::string ReadWriteQual("__read_write");
2390	std::string::size_type ReadWritePos = TyName.find(str: ReadWriteQual);
2391	if (ReadWritePos != std::string::npos)
2392	TyName.erase(pos: ReadWritePos, n: ReadWriteQual.size() + 1);
2393	}
2394	}
2395	}
2396
2397	// Returns the address space id that should be produced to the
2398	// kernel_arg_addr_space metadata. This is always fixed to the ids
2399	// as specified in the SPIR 2.0 specification in order to differentiate
2400	// for example in clGetKernelArgInfo() implementation between the address
2401	// spaces with targets without unique mapping to the OpenCL address spaces
2402	// (basically all single AS CPUs).
2403	static unsigned ArgInfoAddressSpace(LangAS AS) {
2404	switch (AS) {
2405	case LangAS::opencl_global:
2406	return 1;
2407	case LangAS::opencl_constant:
2408	return 2;
2409	case LangAS::opencl_local:
2410	return 3;
2411	case LangAS::opencl_generic:
2412	return 4; // Not in SPIR 2.0 specs.
2413	case LangAS::opencl_global_device:
2414	return 5;
2415	case LangAS::opencl_global_host:
2416	return 6;
2417	default:
2418	return 0; // Assume private.
2419	}
2420	}
2421
2422	void CodeGenModule::GenKernelArgMetadata(llvm::Function *Fn,
2423	const FunctionDecl *FD,
2424	CodeGenFunction *CGF) {
2425	assert(((FD && CGF) || (!FD && !CGF)) &&
2426	"Incorrect use - FD and CGF should either be both null or not!");
2427	// Create MDNodes that represent the kernel arg metadata.
2428	// Each MDNode is a list in the form of "key", N number of values which is
2429	// the same number of values as their are kernel arguments.
2430
2431	const PrintingPolicy &Policy = Context.getPrintingPolicy();
2432
2433	// MDNode for the kernel argument address space qualifiers.
2434	SmallVector<llvm::Metadata *, 8> addressQuals;
2435
2436	// MDNode for the kernel argument access qualifiers (images only).
2437	SmallVector<llvm::Metadata *, 8> accessQuals;
2438
2439	// MDNode for the kernel argument type names.
2440	SmallVector<llvm::Metadata *, 8> argTypeNames;
2441
2442	// MDNode for the kernel argument base type names.
2443	SmallVector<llvm::Metadata *, 8> argBaseTypeNames;
2444
2445	// MDNode for the kernel argument type qualifiers.
2446	SmallVector<llvm::Metadata *, 8> argTypeQuals;
2447
2448	// MDNode for the kernel argument names.
2449	SmallVector<llvm::Metadata *, 8> argNames;
2450
2451	if (FD && CGF)
2452	for (unsigned i = 0, e = FD->getNumParams(); i != e; ++i) {
2453	const ParmVarDecl *parm = FD->getParamDecl(i);
2454	// Get argument name.
2455	argNames.push_back(Elt: llvm::MDString::get(Context&: VMContext, Str: parm->getName()));
2456
2457	if (!getLangOpts().OpenCL)
2458	continue;
2459	QualType ty = parm->getType();
2460	std::string typeQuals;
2461
2462	// Get image and pipe access qualifier:
2463	if (ty->isImageType() || ty->isPipeType()) {
2464	const Decl *PDecl = parm;
2465	if (const auto *TD = ty->getAs<TypedefType>())
2466	PDecl = TD->getDecl();
2467	const OpenCLAccessAttr *A = PDecl->getAttr<OpenCLAccessAttr>();
2468	if (A && A->isWriteOnly())
2469	accessQuals.push_back(Elt: llvm::MDString::get(Context&: VMContext, Str: "write_only"));
2470	else if (A && A->isReadWrite())
2471	accessQuals.push_back(Elt: llvm::MDString::get(Context&: VMContext, Str: "read_write"));
2472	else
2473	accessQuals.push_back(Elt: llvm::MDString::get(Context&: VMContext, Str: "read_only"));
2474	} else
2475	accessQuals.push_back(Elt: llvm::MDString::get(Context&: VMContext, Str: "none"));
2476
2477	auto getTypeSpelling = [&](QualType Ty) {
2478	auto typeName = Ty.getUnqualifiedType().getAsString(Policy);
2479
2480	if (Ty.isCanonical()) {
2481	StringRef typeNameRef = typeName;
2482	// Turn "unsigned type" to "utype"
2483	if (typeNameRef.consume_front(Prefix: "unsigned "))
2484	return std::string("u") + typeNameRef.str();
2485	if (typeNameRef.consume_front(Prefix: "signed "))
2486	return typeNameRef.str();
2487	}
2488
2489	return typeName;
2490	};
2491
2492	if (ty->isPointerType()) {
2493	QualType pointeeTy = ty->getPointeeType();
2494
2495	// Get address qualifier.
2496	addressQuals.push_back(
2497	Elt: llvm::ConstantAsMetadata::get(C: CGF->Builder.getInt32(
2498	C: ArgInfoAddressSpace(AS: pointeeTy.getAddressSpace()))));
2499
2500	// Get argument type name.
2501	std::string typeName = getTypeSpelling(pointeeTy) + "*";
2502	std::string baseTypeName =
2503	getTypeSpelling(pointeeTy.getCanonicalType()) + "*";
2504	argTypeNames.push_back(Elt: llvm::MDString::get(Context&: VMContext, Str: typeName));
2505	argBaseTypeNames.push_back(
2506	Elt: llvm::MDString::get(Context&: VMContext, Str: baseTypeName));
2507
2508	// Get argument type qualifiers:
2509	if (ty.isRestrictQualified())
2510	typeQuals = "restrict";
2511	if (pointeeTy.isConstQualified() ||
2512	(pointeeTy.getAddressSpace() == LangAS::opencl_constant))
2513	typeQuals += typeQuals.empty() ? "const" : " const";
2514	if (pointeeTy.isVolatileQualified())
2515	typeQuals += typeQuals.empty() ? "volatile" : " volatile";
2516	} else {
2517	uint32_t AddrSpc = 0;
2518	bool isPipe = ty->isPipeType();
2519	if (ty->isImageType() || isPipe)
2520	AddrSpc = ArgInfoAddressSpace(AS: LangAS::opencl_global);
2521
2522	addressQuals.push_back(
2523	Elt: llvm::ConstantAsMetadata::get(C: CGF->Builder.getInt32(C: AddrSpc)));
2524
2525	// Get argument type name.
2526	ty = isPipe ? ty->castAs<PipeType>()->getElementType() : ty;
2527	std::string typeName = getTypeSpelling(ty);
2528	std::string baseTypeName = getTypeSpelling(ty.getCanonicalType());
2529
2530	// Remove access qualifiers on images
2531	// (as they are inseparable from type in clang implementation,
2532	// but OpenCL spec provides a special query to get access qualifier
2533	// via clGetKernelArgInfo with CL_KERNEL_ARG_ACCESS_QUALIFIER):
2534	if (ty->isImageType()) {
2535	removeImageAccessQualifier(TyName&: typeName);
2536	removeImageAccessQualifier(TyName&: baseTypeName);
2537	}
2538
2539	argTypeNames.push_back(Elt: llvm::MDString::get(Context&: VMContext, Str: typeName));
2540	argBaseTypeNames.push_back(
2541	Elt: llvm::MDString::get(Context&: VMContext, Str: baseTypeName));
2542
2543	if (isPipe)
2544	typeQuals = "pipe";
2545	}
2546	argTypeQuals.push_back(Elt: llvm::MDString::get(Context&: VMContext, Str: typeQuals));
2547	}
2548
2549	if (getLangOpts().OpenCL) {
2550	Fn->setMetadata(Kind: "kernel_arg_addr_space",
2551	Node: llvm::MDNode::get(Context&: VMContext, MDs: addressQuals));
2552	Fn->setMetadata(Kind: "kernel_arg_access_qual",
2553	Node: llvm::MDNode::get(Context&: VMContext, MDs: accessQuals));
2554	Fn->setMetadata(Kind: "kernel_arg_type",
2555	Node: llvm::MDNode::get(Context&: VMContext, MDs: argTypeNames));
2556	Fn->setMetadata(Kind: "kernel_arg_base_type",
2557	Node: llvm::MDNode::get(Context&: VMContext, MDs: argBaseTypeNames));
2558	Fn->setMetadata(Kind: "kernel_arg_type_qual",
2559	Node: llvm::MDNode::get(Context&: VMContext, MDs: argTypeQuals));
2560	}
2561	if (getCodeGenOpts().EmitOpenCLArgMetadata ||
2562	getCodeGenOpts().HIPSaveKernelArgName)
2563	Fn->setMetadata(Kind: "kernel_arg_name",
2564	Node: llvm::MDNode::get(Context&: VMContext, MDs: argNames));
2565	}
2566
2567	/// Determines whether the language options require us to model
2568	/// unwind exceptions. We treat -fexceptions as mandating this
2569	/// except under the fragile ObjC ABI with only ObjC exceptions
2570	/// enabled. This means, for example, that C with -fexceptions
2571	/// enables this.
2572	static bool hasUnwindExceptions(const LangOptions &LangOpts) {
2573	// If exceptions are completely disabled, obviously this is false.
2574	if (!LangOpts.Exceptions) return false;
2575
2576	// If C++ exceptions are enabled, this is true.
2577	if (LangOpts.CXXExceptions) return true;
2578
2579	// If ObjC exceptions are enabled, this depends on the ABI.
2580	if (LangOpts.ObjCExceptions) {
2581	return LangOpts.ObjCRuntime.hasUnwindExceptions();
2582	}
2583
2584	return true;
2585	}
2586
2587	static bool requiresMemberFunctionPointerTypeMetadata(CodeGenModule &CGM,
2588	const CXXMethodDecl *MD) {
2589	// Check that the type metadata can ever actually be used by a call.
2590	if (!CGM.getCodeGenOpts().LTOUnit ||
2591	!CGM.HasHiddenLTOVisibility(RD: MD->getParent()))
2592	return false;
2593
2594	// Only functions whose address can be taken with a member function pointer
2595	// need this sort of type metadata.
2596	return MD->isImplicitObjectMemberFunction() && !MD->isVirtual() &&
2597	!isa<CXXConstructorDecl, CXXDestructorDecl>(Val: MD);
2598	}
2599
2600	SmallVector<const CXXRecordDecl *, 0>
2601	CodeGenModule::getMostBaseClasses(const CXXRecordDecl *RD) {
2602	llvm::SetVector<const CXXRecordDecl *> MostBases;
2603
2604	std::function<void (const CXXRecordDecl *)> CollectMostBases;
2605	CollectMostBases = [&](const CXXRecordDecl *RD) {
2606	if (RD->getNumBases() == 0)
2607	MostBases.insert(X: RD);
2608	for (const CXXBaseSpecifier &B : RD->bases())
2609	CollectMostBases(B.getType()->getAsCXXRecordDecl());
2610	};
2611	CollectMostBases(RD);
2612	return MostBases.takeVector();
2613	}
2614
2615	void CodeGenModule::SetLLVMFunctionAttributesForDefinition(const Decl *D,
2616	llvm::Function *F) {
2617	llvm::AttrBuilder B(F->getContext());
2618
2619	if ((!D || !D->hasAttr<NoUwtableAttr>()) && CodeGenOpts.UnwindTables)
2620	B.addUWTableAttr(Kind: llvm::UWTableKind(CodeGenOpts.UnwindTables));
2621
2622	if (CodeGenOpts.StackClashProtector)
2623	B.addAttribute(A: "probe-stack", V: "inline-asm");
2624
2625	if (CodeGenOpts.StackProbeSize && CodeGenOpts.StackProbeSize != 4096)
2626	B.addAttribute(A: "stack-probe-size",
2627	V: std::to_string(val: CodeGenOpts.StackProbeSize));
2628
2629	if (!hasUnwindExceptions(LangOpts))
2630	B.addAttribute(Val: llvm::Attribute::NoUnwind);
2631
2632	if (D && D->hasAttr<NoStackProtectorAttr>())
2633	; // Do nothing.
2634	else if (D && D->hasAttr<StrictGuardStackCheckAttr>() &&
2635	isStackProtectorOn(LangOpts, Triple: getTriple(), Mode: LangOptions::SSPOn))
2636	B.addAttribute(Val: llvm::Attribute::StackProtectStrong);
2637	else if (isStackProtectorOn(LangOpts, Triple: getTriple(), Mode: LangOptions::SSPOn))
2638	B.addAttribute(Val: llvm::Attribute::StackProtect);
2639	else if (isStackProtectorOn(LangOpts, Triple: getTriple(), Mode: LangOptions::SSPStrong))
2640	B.addAttribute(Val: llvm::Attribute::StackProtectStrong);
2641	else if (isStackProtectorOn(LangOpts, Triple: getTriple(), Mode: LangOptions::SSPReq))
2642	B.addAttribute(Val: llvm::Attribute::StackProtectReq);
2643
2644	if (!D) {
2645	// Non-entry HLSL functions must always be inlined.
2646	if (getLangOpts().HLSL && !F->hasFnAttribute(Kind: llvm::Attribute::NoInline))
2647	B.addAttribute(Val: llvm::Attribute::AlwaysInline);
2648	// If we don't have a declaration to control inlining, the function isn't
2649	// explicitly marked as alwaysinline for semantic reasons, and inlining is
2650	// disabled, mark the function as noinline.
2651	else if (!F->hasFnAttribute(Kind: llvm::Attribute::AlwaysInline) &&
2652	CodeGenOpts.getInlining() == CodeGenOptions::OnlyAlwaysInlining)
2653	B.addAttribute(Val: llvm::Attribute::NoInline);
2654
2655	F->addFnAttrs(Attrs: B);
2656	return;
2657	}
2658
2659	// Handle SME attributes that apply to function definitions,
2660	// rather than to function prototypes.
2661	if (D->hasAttr<ArmLocallyStreamingAttr>())
2662	B.addAttribute(A: "aarch64_pstate_sm_body");
2663
2664	if (auto *Attr = D->getAttr<ArmNewAttr>()) {
2665	if (Attr->isNewZA())
2666	B.addAttribute(A: "aarch64_new_za");
2667	if (Attr->isNewZT0())
2668	B.addAttribute(A: "aarch64_new_zt0");
2669	}
2670
2671	// Track whether we need to add the optnone LLVM attribute,
2672	// starting with the default for this optimization level.
2673	bool ShouldAddOptNone =
2674	!CodeGenOpts.DisableO0ImplyOptNone && CodeGenOpts.OptimizationLevel == 0;
2675	// We can't add optnone in the following cases, it won't pass the verifier.
2676	ShouldAddOptNone &= !D->hasAttr<MinSizeAttr>();
2677	ShouldAddOptNone &= !D->hasAttr<AlwaysInlineAttr>();
2678
2679	// Non-entry HLSL functions must always be inlined.
2680	if (getLangOpts().HLSL && !F->hasFnAttribute(Kind: llvm::Attribute::NoInline) &&
2681	!D->hasAttr<NoInlineAttr>()) {
2682	B.addAttribute(Val: llvm::Attribute::AlwaysInline);
2683	} else if ((ShouldAddOptNone || D->hasAttr<OptimizeNoneAttr>()) &&
2684	!F->hasFnAttribute(Kind: llvm::Attribute::AlwaysInline)) {
2685	// Add optnone, but do so only if the function isn't always_inline.
2686	B.addAttribute(Val: llvm::Attribute::OptimizeNone);
2687
2688	// OptimizeNone implies noinline; we should not be inlining such functions.
2689	B.addAttribute(Val: llvm::Attribute::NoInline);
2690
2691	// We still need to handle naked functions even though optnone subsumes
2692	// much of their semantics.
2693	if (D->hasAttr<NakedAttr>())
2694	B.addAttribute(Val: llvm::Attribute::Naked);
2695
2696	// OptimizeNone wins over OptimizeForSize and MinSize.
2697	F->removeFnAttr(Kind: llvm::Attribute::OptimizeForSize);
2698	F->removeFnAttr(Kind: llvm::Attribute::MinSize);
2699	} else if (D->hasAttr<NakedAttr>()) {
2700	// Naked implies noinline: we should not be inlining such functions.
2701	B.addAttribute(Val: llvm::Attribute::Naked);
2702	B.addAttribute(Val: llvm::Attribute::NoInline);
2703	} else if (D->hasAttr<NoDuplicateAttr>()) {
2704	B.addAttribute(Val: llvm::Attribute::NoDuplicate);
2705	} else if (D->hasAttr<NoInlineAttr>() &&
2706	!F->hasFnAttribute(Kind: llvm::Attribute::AlwaysInline)) {
2707	// Add noinline if the function isn't always_inline.
2708	B.addAttribute(Val: llvm::Attribute::NoInline);
2709	} else if (D->hasAttr<AlwaysInlineAttr>() &&
2710	!F->hasFnAttribute(Kind: llvm::Attribute::NoInline)) {
2711	// (noinline wins over always_inline, and we can't specify both in IR)
2712	B.addAttribute(Val: llvm::Attribute::AlwaysInline);
2713	} else if (CodeGenOpts.getInlining() == CodeGenOptions::OnlyAlwaysInlining) {
2714	// If we're not inlining, then force everything that isn't always_inline to
2715	// carry an explicit noinline attribute.
2716	if (!F->hasFnAttribute(Kind: llvm::Attribute::AlwaysInline))
2717	B.addAttribute(Val: llvm::Attribute::NoInline);
2718	} else {
2719	// Otherwise, propagate the inline hint attribute and potentially use its
2720	// absence to mark things as noinline.
2721	if (auto *FD = dyn_cast<FunctionDecl>(Val: D)) {
2722	// Search function and template pattern redeclarations for inline.
2723	auto CheckForInline = [](const FunctionDecl *FD) {
2724	auto CheckRedeclForInline = [](const FunctionDecl *Redecl) {
2725	return Redecl->isInlineSpecified();
2726	};
2727	if (any_of(Range: FD->redecls(), P: CheckRedeclForInline))
2728	return true;
2729	const FunctionDecl *Pattern = FD->getTemplateInstantiationPattern();
2730	if (!Pattern)
2731	return false;
2732	return any_of(Range: Pattern->redecls(), P: CheckRedeclForInline);
2733	};
2734	if (CheckForInline(FD)) {
2735	B.addAttribute(Val: llvm::Attribute::InlineHint);
2736	} else if (CodeGenOpts.getInlining() ==
2737	CodeGenOptions::OnlyHintInlining &&
2738	!FD->isInlined() &&
2739	!F->hasFnAttribute(Kind: llvm::Attribute::AlwaysInline)) {
2740	B.addAttribute(Val: llvm::Attribute::NoInline);
2741	}
2742	}
2743	}
2744
2745	// Add other optimization related attributes if we are optimizing this
2746	// function.
2747	if (!D->hasAttr<OptimizeNoneAttr>()) {
2748	if (D->hasAttr<ColdAttr>()) {
2749	if (!ShouldAddOptNone)
2750	B.addAttribute(Val: llvm::Attribute::OptimizeForSize);
2751	B.addAttribute(Val: llvm::Attribute::Cold);
2752	}
2753	if (D->hasAttr<HotAttr>())
2754	B.addAttribute(Val: llvm::Attribute::Hot);
2755	if (D->hasAttr<MinSizeAttr>())
2756	B.addAttribute(Val: llvm::Attribute::MinSize);
2757	}
2758
2759	F->addFnAttrs(Attrs: B);
2760
2761	unsigned alignment = D->getMaxAlignment() / Context.getCharWidth();
2762	if (alignment)
2763	F->setAlignment(llvm::Align(alignment));
2764
2765	if (!D->hasAttr<AlignedAttr>())
2766	if (LangOpts.FunctionAlignment)
2767	F->setAlignment(llvm::Align(1ull << LangOpts.FunctionAlignment));
2768
2769	// Some C++ ABIs require 2-byte alignment for member functions, in order to
2770	// reserve a bit for differentiating between virtual and non-virtual member
2771	// functions. If the current target's C++ ABI requires this and this is a
2772	// member function, set its alignment accordingly.
2773	if (getTarget().getCXXABI().areMemberFunctionsAligned()) {
2774	if (isa<CXXMethodDecl>(Val: D) && F->getPointerAlignment(DL: getDataLayout()) < 2)
2775	F->setAlignment(std::max(a: llvm::Align(2), b: F->getAlign().valueOrOne()));
2776	}
2777
2778	// In the cross-dso CFI mode with canonical jump tables, we want !type
2779	// attributes on definitions only.
2780	if (CodeGenOpts.SanitizeCfiCrossDso &&
2781	CodeGenOpts.SanitizeCfiCanonicalJumpTables) {
2782	if (auto *FD = dyn_cast<FunctionDecl>(Val: D)) {
2783	// Skip available_externally functions. They won't be codegen'ed in the
2784	// current module anyway.
2785	if (getContext().GetGVALinkageForFunction(FD) != GVA_AvailableExternally)
2786	createFunctionTypeMetadataForIcall(FD, F);
2787	}
2788	}
2789
2790	// Emit type metadata on member functions for member function pointer checks.
2791	// These are only ever necessary on definitions; we're guaranteed that the
2792	// definition will be present in the LTO unit as a result of LTO visibility.
2793	auto *MD = dyn_cast<CXXMethodDecl>(Val: D);
2794	if (MD && requiresMemberFunctionPointerTypeMetadata(CGM&: *this, MD)) {
2795	for (const CXXRecordDecl *Base : getMostBaseClasses(RD: MD->getParent())) {
2796	llvm::Metadata *Id =
2797	CreateMetadataIdentifierForType(T: Context.getMemberPointerType(
2798	T: MD->getType(), /*Qualifier=*/nullptr, Cls: Base));
2799	F->addTypeMetadata(Offset: 0, TypeID: Id);
2800	}
2801	}
2802	}
2803
2804	void CodeGenModule::SetCommonAttributes(GlobalDecl GD, llvm::GlobalValue *GV) {
2805	const Decl *D = GD.getDecl();
2806	if (isa_and_nonnull<NamedDecl>(Val: D))
2807	setGVProperties(GV, GD);
2808	else
2809	GV->setVisibility(llvm::GlobalValue::DefaultVisibility);
2810
2811	if (D && D->hasAttr<UsedAttr>())
2812	addUsedOrCompilerUsedGlobal(GV);
2813
2814	if (const auto *VD = dyn_cast_if_present<VarDecl>(Val: D);
2815	VD &&
2816	((CodeGenOpts.KeepPersistentStorageVariables &&
2817	(VD->getStorageDuration() == SD_Static ||
2818	VD->getStorageDuration() == SD_Thread)) ||
2819	(CodeGenOpts.KeepStaticConsts && VD->getStorageDuration() == SD_Static &&
2820	VD->getType().isConstQualified())))
2821	addUsedOrCompilerUsedGlobal(GV);
2822	}
2823
2824	bool CodeGenModule::GetCPUAndFeaturesAttributes(GlobalDecl GD,
2825	llvm::AttrBuilder &Attrs,
2826	bool SetTargetFeatures) {
2827	// Add target-cpu and target-features attributes to functions. If
2828	// we have a decl for the function and it has a target attribute then
2829	// parse that and add it to the feature set.
2830	StringRef TargetCPU = getTarget().getTargetOpts().CPU;
2831	StringRef TuneCPU = getTarget().getTargetOpts().TuneCPU;
2832	std::vector<std::string> Features;
2833	const auto *FD = dyn_cast_or_null<FunctionDecl>(Val: GD.getDecl());
2834	FD = FD ? FD->getMostRecentDecl() : FD;
2835	const auto *TD = FD ? FD->getAttr<TargetAttr>() : nullptr;
2836	const auto *TV = FD ? FD->getAttr<TargetVersionAttr>() : nullptr;
2837	assert((!TD || !TV) && "both target_version and target specified");
2838	const auto *SD = FD ? FD->getAttr<CPUSpecificAttr>() : nullptr;
2839	const auto *TC = FD ? FD->getAttr<TargetClonesAttr>() : nullptr;
2840	bool AddedAttr = false;
2841	if (TD || TV || SD || TC) {
2842	llvm::StringMap<bool> FeatureMap;
2843	getContext().getFunctionFeatureMap(FeatureMap, GD);
2844
2845	// Produce the canonical string for this set of features.
2846	for (const llvm::StringMap<bool>::value_type &Entry : FeatureMap)
2847	Features.push_back(x: (Entry.getValue() ? "+" : "-") + Entry.getKey().str());
2848
2849	// Now add the target-cpu and target-features to the function.
2850	// While we populated the feature map above, we still need to
2851	// get and parse the target attribute so we can get the cpu for
2852	// the function.
2853	if (TD) {
2854	ParsedTargetAttr ParsedAttr =
2855	Target.parseTargetAttr(Str: TD->getFeaturesStr());
2856	if (!ParsedAttr.CPU.empty() &&
2857	getTarget().isValidCPUName(Name: ParsedAttr.CPU)) {
2858	TargetCPU = ParsedAttr.CPU;
2859	TuneCPU = ""; // Clear the tune CPU.
2860	}
2861	if (!ParsedAttr.Tune.empty() &&
2862	getTarget().isValidCPUName(Name: ParsedAttr.Tune))
2863	TuneCPU = ParsedAttr.Tune;
2864	}
2865
2866	if (SD) {
2867	// Apply the given CPU name as the 'tune-cpu' so that the optimizer can
2868	// favor this processor.
2869	TuneCPU = SD->getCPUName(Index: GD.getMultiVersionIndex())->getName();
2870	}
2871	} else {
2872	// Otherwise just add the existing target cpu and target features to the
2873	// function.
2874	Features = getTarget().getTargetOpts().Features;
2875	}
2876
2877	if (!TargetCPU.empty()) {
2878	Attrs.addAttribute(A: "target-cpu", V: TargetCPU);
2879	AddedAttr = true;
2880	}
2881	if (!TuneCPU.empty()) {
2882	Attrs.addAttribute(A: "tune-cpu", V: TuneCPU);
2883	AddedAttr = true;
2884	}
2885	if (!Features.empty() && SetTargetFeatures) {
2886	llvm::erase_if(C&: Features, P: [&](const std::string& F) {
2887	return getTarget().isReadOnlyFeature(Feature: F.substr(pos: 1));
2888	});
2889	llvm::sort(C&: Features);
2890	Attrs.addAttribute(A: "target-features", V: llvm::join(R&: Features, Separator: ","));
2891	AddedAttr = true;
2892	}
2893	// Add metadata for AArch64 Function Multi Versioning.
2894	if (getTarget().getTriple().isAArch64()) {
2895	llvm::SmallVector<StringRef, 8> Feats;
2896	bool IsDefault = false;
2897	if (TV) {
2898	IsDefault = TV->isDefaultVersion();
2899	TV->getFeatures(Out&: Feats);
2900	} else if (TC) {
2901	IsDefault = TC->isDefaultVersion(Index: GD.getMultiVersionIndex());
2902	TC->getFeatures(Out&: Feats, Index: GD.getMultiVersionIndex());
2903	}
2904	if (IsDefault) {
2905	Attrs.addAttribute(A: "fmv-features");
2906	AddedAttr = true;
2907	} else if (!Feats.empty()) {
2908	// Sort features and remove duplicates.
2909	std::set<StringRef> OrderedFeats(Feats.begin(), Feats.end());
2910	std::string FMVFeatures;
2911	for (StringRef F : OrderedFeats)
2912	FMVFeatures.append(str: "," + F.str());
2913	Attrs.addAttribute(A: "fmv-features", V: FMVFeatures.substr(pos: 1));
2914	AddedAttr = true;
2915	}
2916	}
2917	return AddedAttr;
2918	}
2919
2920	void CodeGenModule::setNonAliasAttributes(GlobalDecl GD,
2921	llvm::GlobalObject *GO) {
2922	const Decl *D = GD.getDecl();
2923	SetCommonAttributes(GD, GV: GO);
2924
2925	if (D) {
2926	if (auto *GV = dyn_cast<llvm::GlobalVariable>(Val: GO)) {
2927	if (D->hasAttr<RetainAttr>())
2928	addUsedGlobal(GV);
2929	if (auto *SA = D->getAttr<PragmaClangBSSSectionAttr>())
2930	GV->addAttribute(Kind: "bss-section", Val: SA->getName());
2931	if (auto *SA = D->getAttr<PragmaClangDataSectionAttr>())
2932	GV->addAttribute(Kind: "data-section", Val: SA->getName());
2933	if (auto *SA = D->getAttr<PragmaClangRodataSectionAttr>())
2934	GV->addAttribute(Kind: "rodata-section", Val: SA->getName());
2935	if (auto *SA = D->getAttr<PragmaClangRelroSectionAttr>())
2936	GV->addAttribute(Kind: "relro-section", Val: SA->getName());
2937	}
2938
2939	if (auto *F = dyn_cast<llvm::Function>(Val: GO)) {
2940	if (D->hasAttr<RetainAttr>())
2941	addUsedGlobal(GV: F);
2942	if (auto *SA = D->getAttr<PragmaClangTextSectionAttr>())
2943	if (!D->getAttr<SectionAttr>())
2944	F->setSection(SA->getName());
2945
2946	llvm::AttrBuilder Attrs(F->getContext());
2947	if (GetCPUAndFeaturesAttributes(GD, Attrs)) {
2948	// We know that GetCPUAndFeaturesAttributes will always have the
2949	// newest set, since it has the newest possible FunctionDecl, so the
2950	// new ones should replace the old.
2951	llvm::AttributeMask RemoveAttrs;
2952	RemoveAttrs.addAttribute(A: "target-cpu");
2953	RemoveAttrs.addAttribute(A: "target-features");
2954	RemoveAttrs.addAttribute(A: "fmv-features");
2955	RemoveAttrs.addAttribute(A: "tune-cpu");
2956	F->removeFnAttrs(Attrs: RemoveAttrs);
2957	F->addFnAttrs(Attrs);
2958	}
2959	}
2960
2961	if (const auto *CSA = D->getAttr<CodeSegAttr>())
2962	GO->setSection(CSA->getName());
2963	else if (const auto *SA = D->getAttr<SectionAttr>())
2964	GO->setSection(SA->getName());
2965	}
2966
2967	getTargetCodeGenInfo().setTargetAttributes(D, GV: GO, M&: *this);
2968	}
2969
2970	void CodeGenModule::SetInternalFunctionAttributes(GlobalDecl GD,
2971	llvm::Function *F,
2972	const CGFunctionInfo &FI) {
2973	const Decl *D = GD.getDecl();
2974	SetLLVMFunctionAttributes(GD, Info: FI, F, /*IsThunk=*/false);
2975	SetLLVMFunctionAttributesForDefinition(D, F);
2976
2977	F->setLinkage(llvm::Function::InternalLinkage);
2978
2979	setNonAliasAttributes(GD, GO: F);
2980	}
2981
2982	static void setLinkageForGV(llvm::GlobalValue *GV, const NamedDecl *ND) {
2983	// Set linkage and visibility in case we never see a definition.
2984	LinkageInfo LV = ND->getLinkageAndVisibility();
2985	// Don't set internal linkage on declarations.
2986	// "extern_weak" is overloaded in LLVM; we probably should have
2987	// separate linkage types for this.
2988	if (isExternallyVisible(L: LV.getLinkage()) &&
2989	(ND->hasAttr<WeakAttr>() || ND->isWeakImported()))
2990	GV->setLinkage(llvm::GlobalValue::ExternalWeakLinkage);
2991	}
2992
2993	void CodeGenModule::createFunctionTypeMetadataForIcall(const FunctionDecl *FD,
2994	llvm::Function *F) {
2995	// Only if we are checking indirect calls.
2996	if (!LangOpts.Sanitize.has(K: SanitizerKind::CFIICall))
2997	return;
2998
2999	// Non-static class methods are handled via vtable or member function pointer
3000	// checks elsewhere.
3001	if (isa<CXXMethodDecl>(Val: FD) && !cast<CXXMethodDecl>(Val: FD)->isStatic())
3002	return;
3003
3004	llvm::Metadata *MD = CreateMetadataIdentifierForType(T: FD->getType());
3005	F->addTypeMetadata(Offset: 0, TypeID: MD);
3006	F->addTypeMetadata(Offset: 0, TypeID: CreateMetadataIdentifierGeneralized(T: FD->getType()));
3007
3008	// Emit a hash-based bit set entry for cross-DSO calls.
3009	if (CodeGenOpts.SanitizeCfiCrossDso)
3010	if (auto CrossDsoTypeId = CreateCrossDsoCfiTypeId(MD))
3011	F->addTypeMetadata(Offset: 0, TypeID: llvm::ConstantAsMetadata::get(C: CrossDsoTypeId));
3012	}
3013
3014	void CodeGenModule::setKCFIType(const FunctionDecl *FD, llvm::Function *F) {
3015	llvm::LLVMContext &Ctx = F->getContext();
3016	llvm::MDBuilder MDB(Ctx);
3017	F->setMetadata(KindID: llvm::LLVMContext::MD_kcfi_type,
3018	Node: llvm::MDNode::get(
3019	Context&: Ctx, MDs: MDB.createConstant(C: CreateKCFITypeId(T: FD->getType()))));
3020	}
3021
3022	static bool allowKCFIIdentifier(StringRef Name) {
3023	// KCFI type identifier constants are only necessary for external assembly
3024	// functions, which means it's safe to skip unusual names. Subset of
3025	// MCAsmInfo::isAcceptableChar() and MCAsmInfoXCOFF::isAcceptableChar().
3026	return llvm::all_of(Range&: Name, P: [](const char &C) {
3027	return llvm::isAlnum(C) || C == '_' || C == '.';
3028	});
3029	}
3030
3031	void CodeGenModule::finalizeKCFITypes() {
3032	llvm::Module &M = getModule();
3033	for (auto &F : M.functions()) {
3034	// Remove KCFI type metadata from non-address-taken local functions.
3035	bool AddressTaken = F.hasAddressTaken();
3036	if (!AddressTaken && F.hasLocalLinkage())
3037	F.eraseMetadata(KindID: llvm::LLVMContext::MD_kcfi_type);
3038
3039	// Generate a constant with the expected KCFI type identifier for all
3040	// address-taken function declarations to support annotating indirectly
3041	// called assembly functions.
3042	if (!AddressTaken || !F.isDeclaration())
3043	continue;
3044
3045	const llvm::ConstantInt *Type;
3046	if (const llvm::MDNode *MD = F.getMetadata(KindID: llvm::LLVMContext::MD_kcfi_type))
3047	Type = llvm::mdconst::extract<llvm::ConstantInt>(MD: MD->getOperand(I: 0));
3048	else
3049	continue;
3050
3051	StringRef Name = F.getName();
3052	if (!allowKCFIIdentifier(Name))
3053	continue;
3054
3055	std::string Asm = (".weak __kcfi_typeid_" + Name + "\n.set __kcfi_typeid_" +
3056	Name + ", " + Twine(Type->getZExtValue()) + "\n")
3057	.str();
3058	M.appendModuleInlineAsm(Asm);
3059	}
3060	}
3061
3062	void CodeGenModule::SetFunctionAttributes(GlobalDecl GD, llvm::Function *F,
3063	bool IsIncompleteFunction,
3064	bool IsThunk) {
3065
3066	if (F->getIntrinsicID() != llvm::Intrinsic::not_intrinsic) {
3067	// If this is an intrinsic function, the attributes will have been set
3068	// when the function was created.
3069	return;
3070	}
3071
3072	const auto *FD = cast<FunctionDecl>(Val: GD.getDecl());
3073
3074	if (!IsIncompleteFunction)
3075	SetLLVMFunctionAttributes(GD, Info: getTypes().arrangeGlobalDeclaration(GD), F,
3076	IsThunk);
3077
3078	// Add the Returned attribute for "this", except for iOS 5 and earlier
3079	// where substantial code, including the libstdc++ dylib, was compiled with
3080	// GCC and does not actually return "this".
3081	if (!IsThunk && getCXXABI().HasThisReturn(GD) &&
3082	!(getTriple().isiOS() && getTriple().isOSVersionLT(Major: 6))) {
3083	assert(!F->arg_empty() &&
3084	F->arg_begin()->getType()
3085	->canLosslesslyBitCastTo(F->getReturnType()) &&
3086	"unexpected this return");
3087	F->addParamAttr(ArgNo: 0, Kind: llvm::Attribute::Returned);
3088	}
3089
3090	// Only a few attributes are set on declarations; these may later be
3091	// overridden by a definition.
3092
3093	setLinkageForGV(GV: F, ND: FD);
3094	setGVProperties(GV: F, D: FD);
3095
3096	// Setup target-specific attributes.
3097	if (!IsIncompleteFunction && F->isDeclaration())
3098	getTargetCodeGenInfo().setTargetAttributes(D: FD, GV: F, M&: *this);
3099
3100	if (const auto *CSA = FD->getAttr<CodeSegAttr>())
3101	F->setSection(CSA->getName());
3102	else if (const auto *SA = FD->getAttr<SectionAttr>())
3103	F->setSection(SA->getName());
3104
3105	if (const auto *EA = FD->getAttr<ErrorAttr>()) {
3106	if (EA->isError())
3107	F->addFnAttr(Kind: "dontcall-error", Val: EA->getUserDiagnostic());
3108	else if (EA->isWarning())
3109	F->addFnAttr(Kind: "dontcall-warn", Val: EA->getUserDiagnostic());
3110	}
3111
3112	// If we plan on emitting this inline builtin, we can't treat it as a builtin.
3113	if (FD->isInlineBuiltinDeclaration()) {
3114	const FunctionDecl *FDBody;
3115	bool HasBody = FD->hasBody(Definition&: FDBody);
3116	(void)HasBody;
3117	assert(HasBody && "Inline builtin declarations should always have an "
3118	"available body!");
3119	if (shouldEmitFunction(GD: FDBody))
3120	F->addFnAttr(Kind: llvm::Attribute::NoBuiltin);
3121	}
3122
3123	if (FD->isReplaceableGlobalAllocationFunction()) {
3124	// A replaceable global allocation function does not act like a builtin by
3125	// default, only if it is invoked by a new-expression or delete-expression.
3126	F->addFnAttr(Kind: llvm::Attribute::NoBuiltin);
3127	}
3128
3129	if (isa<CXXConstructorDecl>(Val: FD) || isa<CXXDestructorDecl>(Val: FD))
3130	F->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
3131	else if (const auto *MD = dyn_cast<CXXMethodDecl>(Val: FD))
3132	if (MD->isVirtual())
3133	F->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
3134
3135	// Don't emit entries for function declarations in the cross-DSO mode. This
3136	// is handled with better precision by the receiving DSO. But if jump tables
3137	// are non-canonical then we need type metadata in order to produce the local
3138	// jump table.
3139	if (!CodeGenOpts.SanitizeCfiCrossDso ||
3140	!CodeGenOpts.SanitizeCfiCanonicalJumpTables)
3141	createFunctionTypeMetadataForIcall(FD, F);
3142
3143	if (LangOpts.Sanitize.has(K: SanitizerKind::KCFI))
3144	setKCFIType(FD, F);
3145
3146	if (getLangOpts().OpenMP && FD->hasAttr<OMPDeclareSimdDeclAttr>())
3147	getOpenMPRuntime().emitDeclareSimdFunction(FD, Fn: F);
3148
3149	if (CodeGenOpts.InlineMaxStackSize != UINT_MAX)
3150	F->addFnAttr(Kind: "inline-max-stacksize", Val: llvm::utostr(X: CodeGenOpts.InlineMaxStackSize));
3151
3152	if (const auto *CB = FD->getAttr<CallbackAttr>()) {
3153	// Annotate the callback behavior as metadata:
3154	// - The callback callee (as argument number).
3155	// - The callback payloads (as argument numbers).
3156	llvm::LLVMContext &Ctx = F->getContext();
3157	llvm::MDBuilder MDB(Ctx);
3158
3159	// The payload indices are all but the first one in the encoding. The first
3160	// identifies the callback callee.
3161	int CalleeIdx = *CB->encoding_begin();
3162	ArrayRef<int> PayloadIndices(CB->encoding_begin() + 1, CB->encoding_end());
3163	F->addMetadata(KindID: llvm::LLVMContext::MD_callback,
3164	MD&: *llvm::MDNode::get(Context&: Ctx, MDs: {MDB.createCallbackEncoding(
3165	CalleeArgNo: CalleeIdx, Arguments: PayloadIndices,
3166	/* VarArgsArePassed */ false)}));
3167	}
3168	}
3169
3170	void CodeGenModule::addUsedGlobal(llvm::GlobalValue *GV) {
3171	assert((isa<llvm::Function>(GV) || !GV->isDeclaration()) &&
3172	"Only globals with definition can force usage.");
3173	LLVMUsed.emplace_back(args&: GV);
3174	}
3175
3176	void CodeGenModule::addCompilerUsedGlobal(llvm::GlobalValue *GV) {
3177	assert(!GV->isDeclaration() &&
3178	"Only globals with definition can force usage.");
3179	LLVMCompilerUsed.emplace_back(args&: GV);
3180	}
3181
3182	void CodeGenModule::addUsedOrCompilerUsedGlobal(llvm::GlobalValue *GV) {
3183	assert((isa<llvm::Function>(GV) || !GV->isDeclaration()) &&
3184	"Only globals with definition can force usage.");
3185	if (getTriple().isOSBinFormatELF())
3186	LLVMCompilerUsed.emplace_back(args&: GV);
3187	else
3188	LLVMUsed.emplace_back(args&: GV);
3189	}
3190
3191	static void emitUsed(CodeGenModule &CGM, StringRef Name,
3192	std::vector<llvm::WeakTrackingVH> &List) {
3193	// Don't create llvm.used if there is no need.
3194	if (List.empty())
3195	return;
3196
3197	// Convert List to what ConstantArray needs.
3198	SmallVector<llvm::Constant*, 8> UsedArray;
3199	UsedArray.resize(N: List.size());
3200	for (unsigned i = 0, e = List.size(); i != e; ++i) {
3201	UsedArray[i] =
3202	llvm::ConstantExpr::getPointerBitCastOrAddrSpaceCast(
3203	C: cast<llvm::Constant>(Val: &*List[i]), Ty: CGM.Int8PtrTy);
3204	}
3205
3206	if (UsedArray.empty())
3207	return;
3208	llvm::ArrayType *ATy = llvm::ArrayType::get(ElementType: CGM.Int8PtrTy, NumElements: UsedArray.size());
3209
3210	auto *GV = new llvm::GlobalVariable(
3211	CGM.getModule(), ATy, false, llvm::GlobalValue::AppendingLinkage,
3212	llvm::ConstantArray::get(T: ATy, V: UsedArray), Name);
3213
3214	GV->setSection("llvm.metadata");
3215	}
3216
3217	void CodeGenModule::emitLLVMUsed() {
3218	emitUsed(CGM&: *this, Name: "llvm.used", List&: LLVMUsed);
3219	emitUsed(CGM&: *this, Name: "llvm.compiler.used", List&: LLVMCompilerUsed);
3220	}
3221
3222	void CodeGenModule::AppendLinkerOptions(StringRef Opts) {
3223	auto *MDOpts = llvm::MDString::get(Context&: getLLVMContext(), Str: Opts);
3224	LinkerOptionsMetadata.push_back(Elt: llvm::MDNode::get(Context&: getLLVMContext(), MDs: MDOpts));
3225	}
3226
3227	void CodeGenModule::AddDetectMismatch(StringRef Name, StringRef Value) {
3228	llvm::SmallString<32> Opt;
3229	getTargetCodeGenInfo().getDetectMismatchOption(Name, Value, Opt);
3230	if (Opt.empty())
3231	return;
3232	auto *MDOpts = llvm::MDString::get(Context&: getLLVMContext(), Str: Opt);
3233	LinkerOptionsMetadata.push_back(Elt: llvm::MDNode::get(Context&: getLLVMContext(), MDs: MDOpts));
3234	}
3235
3236	void CodeGenModule::AddDependentLib(StringRef Lib) {
3237	auto &C = getLLVMContext();
3238	if (getTarget().getTriple().isOSBinFormatELF()) {
3239	ELFDependentLibraries.push_back(
3240	Elt: llvm::MDNode::get(Context&: C, MDs: llvm::MDString::get(Context&: C, Str: Lib)));
3241	return;
3242	}
3243
3244	llvm::SmallString<24> Opt;
3245	getTargetCodeGenInfo().getDependentLibraryOption(Lib, Opt);
3246	auto *MDOpts = llvm::MDString::get(Context&: getLLVMContext(), Str: Opt);
3247	LinkerOptionsMetadata.push_back(Elt: llvm::MDNode::get(Context&: C, MDs: MDOpts));
3248	}
3249
3250	/// Add link options implied by the given module, including modules
3251	/// it depends on, using a postorder walk.
3252	static void addLinkOptionsPostorder(CodeGenModule &CGM, Module *Mod,
3253	SmallVectorImpl<llvm::MDNode *> &Metadata,
3254	llvm::SmallPtrSet<Module *, 16> &Visited) {
3255	// Import this module's parent.
3256	if (Mod->Parent && Visited.insert(Ptr: Mod->Parent).second) {
3257	addLinkOptionsPostorder(CGM, Mod: Mod->Parent, Metadata, Visited);
3258	}
3259
3260	// Import this module's dependencies.
3261	for (Module *Import : llvm::reverse(C&: Mod->Imports)) {
3262	if (Visited.insert(Ptr: Import).second)
3263	addLinkOptionsPostorder(CGM, Mod: Import, Metadata, Visited);
3264	}
3265
3266	// Add linker options to link against the libraries/frameworks
3267	// described by this module.
3268	llvm::LLVMContext &Context = CGM.getLLVMContext();
3269	bool IsELF = CGM.getTarget().getTriple().isOSBinFormatELF();
3270
3271	// For modules that use export_as for linking, use that module
3272	// name instead.
3273	if (Mod->UseExportAsModuleLinkName)
3274	return;
3275
3276	for (const Module::LinkLibrary &LL : llvm::reverse(C&: Mod->LinkLibraries)) {
3277	// Link against a framework. Frameworks are currently Darwin only, so we
3278	// don't to ask TargetCodeGenInfo for the spelling of the linker option.
3279	if (LL.IsFramework) {
3280	llvm::Metadata *Args[2] = {llvm::MDString::get(Context, Str: "-framework"),
3281	llvm::MDString::get(Context, Str: LL.Library)};
3282
3283	Metadata.push_back(Elt: llvm::MDNode::get(Context, MDs: Args));
3284	continue;
3285	}
3286
3287	// Link against a library.
3288	if (IsELF) {
3289	llvm::Metadata *Args[2] = {
3290	llvm::MDString::get(Context, Str: "lib"),
3291	llvm::MDString::get(Context, Str: LL.Library),
3292	};
3293	Metadata.push_back(Elt: llvm::MDNode::get(Context, MDs: Args));
3294	} else {
3295	llvm::SmallString<24> Opt;
3296	CGM.getTargetCodeGenInfo().getDependentLibraryOption(Lib: LL.Library, Opt);
3297	auto *OptString = llvm::MDString::get(Context, Str: Opt);
3298	Metadata.push_back(Elt: llvm::MDNode::get(Context, MDs: OptString));
3299	}
3300	}
3301	}
3302
3303	void CodeGenModule::EmitModuleInitializers(clang::Module *Primary) {
3304	assert(Primary->isNamedModuleUnit() &&
3305	"We should only emit module initializers for named modules.");
3306
3307	// Emit the initializers in the order that sub-modules appear in the
3308	// source, first Global Module Fragments, if present.
3309	if (auto GMF = Primary->getGlobalModuleFragment()) {
3310	for (Decl *D : getContext().getModuleInitializers(M: GMF)) {
3311	if (isa<ImportDecl>(Val: D))
3312	continue;
3313	assert(isa<VarDecl>(D) && "GMF initializer decl is not a var?");
3314	EmitTopLevelDecl(D);
3315	}
3316	}
3317	// Second any associated with the module, itself.
3318	for (Decl *D : getContext().getModuleInitializers(M: Primary)) {
3319	// Skip import decls, the inits for those are called explicitly.
3320	if (isa<ImportDecl>(Val: D))
3321	continue;
3322	EmitTopLevelDecl(D);
3323	}
3324	// Third any associated with the Privat eMOdule Fragment, if present.
3325	if (auto PMF = Primary->getPrivateModuleFragment()) {
3326	for (Decl *D : getContext().getModuleInitializers(M: PMF)) {
3327	// Skip import decls, the inits for those are called explicitly.
3328	if (isa<ImportDecl>(Val: D))
3329	continue;
3330	assert(isa<VarDecl>(D) && "PMF initializer decl is not a var?");
3331	EmitTopLevelDecl(D);
3332	}
3333	}
3334	}
3335
3336	void CodeGenModule::EmitModuleLinkOptions() {
3337	// Collect the set of all of the modules we want to visit to emit link
3338	// options, which is essentially the imported modules and all of their
3339	// non-explicit child modules.
3340	llvm::SetVector<clang::Module *> LinkModules;
3341	llvm::SmallPtrSet<clang::Module *, 16> Visited;
3342	SmallVector<clang::Module *, 16> Stack;
3343
3344	// Seed the stack with imported modules.
3345	for (Module *M : ImportedModules) {
3346	// Do not add any link flags when an implementation TU of a module imports
3347	// a header of that same module.
3348	if (M->getTopLevelModuleName() == getLangOpts().CurrentModule &&
3349	!getLangOpts().isCompilingModule())
3350	continue;
3351	if (Visited.insert(Ptr: M).second)
3352	Stack.push_back(Elt: M);
3353	}
3354
3355	// Find all of the modules to import, making a little effort to prune
3356	// non-leaf modules.
3357	while (!Stack.empty()) {
3358	clang::Module *Mod = Stack.pop_back_val();
3359
3360	bool AnyChildren = false;
3361
3362	// Visit the submodules of this module.
3363	for (const auto &SM : Mod->submodules()) {
3364	// Skip explicit children; they need to be explicitly imported to be
3365	// linked against.
3366	if (SM->IsExplicit)
3367	continue;
3368
3369	if (Visited.insert(Ptr: SM).second) {
3370	Stack.push_back(Elt: SM);
3371	AnyChildren = true;
3372	}
3373	}
3374
3375	// We didn't find any children, so add this module to the list of
3376	// modules to link against.
3377	if (!AnyChildren) {
3378	LinkModules.insert(X: Mod);
3379	}
3380	}
3381
3382	// Add link options for all of the imported modules in reverse topological
3383	// order. We don't do anything to try to order import link flags with respect
3384	// to linker options inserted by things like #pragma comment().
3385	SmallVector<llvm::MDNode *, 16> MetadataArgs;
3386	Visited.clear();
3387	for (Module *M : LinkModules)
3388	if (Visited.insert(Ptr: M).second)
3389	addLinkOptionsPostorder(CGM&: *this, Mod: M, Metadata&: MetadataArgs, Visited);
3390	std::reverse(first: MetadataArgs.begin(), last: MetadataArgs.end());
3391	LinkerOptionsMetadata.append(in_start: MetadataArgs.begin(), in_end: MetadataArgs.end());
3392
3393	// Add the linker options metadata flag.
3394	if (!LinkerOptionsMetadata.empty()) {
3395	auto *NMD = getModule().getOrInsertNamedMetadata(Name: "llvm.linker.options");
3396	for (auto *MD : LinkerOptionsMetadata)
3397	NMD->addOperand(M: MD);
3398	}
3399	}
3400
3401	void CodeGenModule::EmitDeferred() {
3402	// Emit deferred declare target declarations.
3403	if (getLangOpts().OpenMP && !getLangOpts().OpenMPSimd)
3404	getOpenMPRuntime().emitDeferredTargetDecls();
3405
3406	// Emit code for any potentially referenced deferred decls. Since a
3407	// previously unused static decl may become used during the generation of code
3408	// for a static function, iterate until no changes are made.
3409
3410	if (!DeferredVTables.empty()) {
3411	EmitDeferredVTables();
3412
3413	// Emitting a vtable doesn't directly cause more vtables to
3414	// become deferred, although it can cause functions to be
3415	// emitted that then need those vtables.
3416	assert(DeferredVTables.empty());
3417	}
3418
3419	// Emit CUDA/HIP static device variables referenced by host code only.
3420	// Note we should not clear CUDADeviceVarODRUsedByHost since it is still
3421	// needed for further handling.
3422	if (getLangOpts().CUDA && getLangOpts().CUDAIsDevice)
3423	llvm::append_range(C&: DeferredDeclsToEmit,
3424	R&: getContext().CUDADeviceVarODRUsedByHost);
3425
3426	// Stop if we're out of both deferred vtables and deferred declarations.
3427	if (DeferredDeclsToEmit.empty())
3428	return;
3429
3430	// Grab the list of decls to emit. If EmitGlobalDefinition schedules more
3431	// work, it will not interfere with this.
3432	std::vector<GlobalDecl> CurDeclsToEmit;
3433	CurDeclsToEmit.swap(x&: DeferredDeclsToEmit);
3434
3435	for (GlobalDecl &D : CurDeclsToEmit) {
3436	// Functions declared with the sycl_kernel_entry_point attribute are
3437	// emitted normally during host compilation. During device compilation,
3438	// a SYCL kernel caller offload entry point function is generated and
3439	// emitted in place of each of these functions.
3440	if (const auto *FD = D.getDecl()->getAsFunction()) {
3441	if (LangOpts.SYCLIsDevice && FD->hasAttr<SYCLKernelEntryPointAttr>() &&
3442	FD->isDefined()) {
3443	// Functions with an invalid sycl_kernel_entry_point attribute are
3444	// ignored during device compilation.
3445	if (!FD->getAttr<SYCLKernelEntryPointAttr>()->isInvalidAttr()) {
3446	// Generate and emit the SYCL kernel caller function.
3447	EmitSYCLKernelCaller(KernelEntryPointFn: FD, Ctx&: getContext());
3448	// Recurse to emit any symbols directly or indirectly referenced
3449	// by the SYCL kernel caller function.
3450	EmitDeferred();
3451	}
3452	// Do not emit the sycl_kernel_entry_point attributed function.
3453	continue;
3454	}
3455	}
3456
3457	// We should call GetAddrOfGlobal with IsForDefinition set to true in order
3458	// to get GlobalValue with exactly the type we need, not something that
3459	// might had been created for another decl with the same mangled name but
3460	// different type.
3461	llvm::GlobalValue *GV = dyn_cast<llvm::GlobalValue>(
3462	Val: GetAddrOfGlobal(GD: D, IsForDefinition: ForDefinition));
3463
3464	// In case of different address spaces, we may still get a cast, even with
3465	// IsForDefinition equal to true. Query mangled names table to get
3466	// GlobalValue.
3467	if (!GV)
3468	GV = GetGlobalValue(Name: getMangledName(GD: D));
3469
3470	// Make sure GetGlobalValue returned non-null.
3471	assert(GV);
3472
3473	// Check to see if we've already emitted this. This is necessary
3474	// for a couple of reasons: first, decls can end up in the
3475	// deferred-decls queue multiple times, and second, decls can end
3476	// up with definitions in unusual ways (e.g. by an extern inline
3477	// function acquiring a strong function redefinition). Just
3478	// ignore these cases.
3479	if (!GV->isDeclaration())
3480	continue;
3481
3482	// If this is OpenMP, check if it is legal to emit this global normally.
3483	if (LangOpts.OpenMP && OpenMPRuntime && OpenMPRuntime->emitTargetGlobal(GD: D))
3484	continue;
3485
3486	// Otherwise, emit the definition and move on to the next one.
3487	EmitGlobalDefinition(D, GV);
3488
3489	// If we found out that we need to emit more decls, do that recursively.
3490	// This has the advantage that the decls are emitted in a DFS and related
3491	// ones are close together, which is convenient for testing.
3492	if (!DeferredVTables.empty() || !DeferredDeclsToEmit.empty()) {
3493	EmitDeferred();
3494	assert(DeferredVTables.empty() && DeferredDeclsToEmit.empty());
3495	}
3496	}
3497	}
3498
3499	void CodeGenModule::EmitVTablesOpportunistically() {
3500	// Try to emit external vtables as available_externally if they have emitted
3501	// all inlined virtual functions. It runs after EmitDeferred() and therefore
3502	// is not allowed to create new references to things that need to be emitted
3503	// lazily. Note that it also uses fact that we eagerly emitting RTTI.
3504
3505	assert((OpportunisticVTables.empty() || shouldOpportunisticallyEmitVTables())
3506	&& "Only emit opportunistic vtables with optimizations");
3507
3508	for (const CXXRecordDecl *RD : OpportunisticVTables) {
3509	assert(getVTables().isVTableExternal(RD) &&
3510	"This queue should only contain external vtables");
3511	if (getCXXABI().canSpeculativelyEmitVTable(RD))
3512	VTables.GenerateClassData(RD);
3513	}
3514	OpportunisticVTables.clear();
3515	}
3516
3517	void CodeGenModule::EmitGlobalAnnotations() {
3518	for (const auto& [MangledName, VD] : DeferredAnnotations) {
3519	llvm::GlobalValue *GV = GetGlobalValue(Name: MangledName);
3520	if (GV)
3521	AddGlobalAnnotations(D: VD, GV);
3522	}
3523	DeferredAnnotations.clear();
3524
3525	if (Annotations.empty())
3526	return;
3527
3528	// Create a new global variable for the ConstantStruct in the Module.
3529	llvm::Constant *Array = llvm::ConstantArray::get(T: llvm::ArrayType::get(
3530	ElementType: Annotations[0]->getType(), NumElements: Annotations.size()), V: Annotations);
3531	auto *gv = new llvm::GlobalVariable(getModule(), Array->getType(), false,
3532	llvm::GlobalValue::AppendingLinkage,
3533	Array, "llvm.global.annotations");
3534	gv->setSection(AnnotationSection);
3535	}
3536
3537	llvm::Constant *CodeGenModule::EmitAnnotationString(StringRef Str) {
3538	llvm::Constant *&AStr = AnnotationStrings[Str];
3539	if (AStr)
3540	return AStr;
3541
3542	// Not found yet, create a new global.
3543	llvm::Constant *s = llvm::ConstantDataArray::getString(Context&: getLLVMContext(), Initializer: Str);
3544	auto *gv = new llvm::GlobalVariable(
3545	getModule(), s->getType(), true, llvm::GlobalValue::PrivateLinkage, s,
3546	".str", nullptr, llvm::GlobalValue::NotThreadLocal,
3547	ConstGlobalsPtrTy->getAddressSpace());
3548	gv->setSection(AnnotationSection);
3549	gv->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
3550	AStr = gv;
3551	return gv;
3552	}
3553
3554	llvm::Constant *CodeGenModule::EmitAnnotationUnit(SourceLocation Loc) {
3555	SourceManager &SM = getContext().getSourceManager();
3556	PresumedLoc PLoc = SM.getPresumedLoc(Loc);
3557	if (PLoc.isValid())
3558	return EmitAnnotationString(Str: PLoc.getFilename());
3559	return EmitAnnotationString(Str: SM.getBufferName(Loc));
3560	}
3561
3562	llvm::Constant *CodeGenModule::EmitAnnotationLineNo(SourceLocation L) {
3563	SourceManager &SM = getContext().getSourceManager();
3564	PresumedLoc PLoc = SM.getPresumedLoc(Loc: L);
3565	unsigned LineNo = PLoc.isValid() ? PLoc.getLine() :
3566	SM.getExpansionLineNumber(Loc: L);
3567	return llvm::ConstantInt::get(Ty: Int32Ty, V: LineNo);
3568	}
3569
3570	llvm::Constant *CodeGenModule::EmitAnnotationArgs(const AnnotateAttr *Attr) {
3571	ArrayRef<Expr *> Exprs = {Attr->args_begin(), Attr->args_size()};
3572	if (Exprs.empty())
3573	return llvm::ConstantPointerNull::get(T: ConstGlobalsPtrTy);
3574
3575	llvm::FoldingSetNodeID ID;
3576	for (Expr *E : Exprs) {
3577	ID.Add(x: cast<clang::ConstantExpr>(Val: E)->getAPValueResult());
3578	}
3579	llvm::Constant *&Lookup = AnnotationArgs[ID.ComputeHash()];
3580	if (Lookup)
3581	return Lookup;
3582
3583	llvm::SmallVector<llvm::Constant *, 4> LLVMArgs;
3584	LLVMArgs.reserve(N: Exprs.size());
3585	ConstantEmitter ConstEmiter(*this);
3586	llvm::transform(Range&: Exprs, d_first: std::back_inserter(x&: LLVMArgs), F: [&](const Expr *E) {
3587	const auto *CE = cast<clang::ConstantExpr>(Val: E);
3588	return ConstEmiter.emitAbstract(loc: CE->getBeginLoc(), value: CE->getAPValueResult(),
3589	T: CE->getType());
3590	});
3591	auto *Struct = llvm::ConstantStruct::getAnon(V: LLVMArgs);
3592	auto *GV = new llvm::GlobalVariable(getModule(), Struct->getType(), true,
3593	llvm::GlobalValue::PrivateLinkage, Struct,
3594	".args");
3595	GV->setSection(AnnotationSection);
3596	GV->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
3597
3598	Lookup = GV;
3599	return GV;
3600	}
3601
3602	llvm::Constant *CodeGenModule::EmitAnnotateAttr(llvm::GlobalValue *GV,
3603	const AnnotateAttr *AA,
3604	SourceLocation L) {
3605	// Get the globals for file name, annotation, and the line number.
3606	llvm::Constant *AnnoGV = EmitAnnotationString(Str: AA->getAnnotation()),
3607	*UnitGV = EmitAnnotationUnit(Loc: L),
3608	*LineNoCst = EmitAnnotationLineNo(L),
3609	*Args = EmitAnnotationArgs(Attr: AA);
3610
3611	llvm::Constant *GVInGlobalsAS = GV;
3612	if (GV->getAddressSpace() !=
3613	getDataLayout().getDefaultGlobalsAddressSpace()) {
3614	GVInGlobalsAS = llvm::ConstantExpr::getAddrSpaceCast(
3615	C: GV,
3616	Ty: llvm::PointerType::get(
3617	C&: GV->getContext(), AddressSpace: getDataLayout().getDefaultGlobalsAddressSpace()));
3618	}
3619
3620	// Create the ConstantStruct for the global annotation.
3621	llvm::Constant *Fields[] = {
3622	GVInGlobalsAS, AnnoGV, UnitGV, LineNoCst, Args,
3623	};
3624	return llvm::ConstantStruct::getAnon(V: Fields);
3625	}
3626
3627	void CodeGenModule::AddGlobalAnnotations(const ValueDecl *D,
3628	llvm::GlobalValue *GV) {
3629	assert(D->hasAttr<AnnotateAttr>() && "no annotate attribute");
3630	// Get the struct elements for these annotations.
3631	for (const auto *I : D->specific_attrs<AnnotateAttr>())
3632	Annotations.push_back(x: EmitAnnotateAttr(GV, AA: I, L: D->getLocation()));
3633	}
3634
3635	bool CodeGenModule::isInNoSanitizeList(SanitizerMask Kind, llvm::Function *Fn,
3636	SourceLocation Loc) const {
3637	const auto &NoSanitizeL = getContext().getNoSanitizeList();
3638	// NoSanitize by function name.
3639	if (NoSanitizeL.containsFunction(Mask: Kind, FunctionName: Fn->getName()))
3640	return true;
3641	// NoSanitize by location. Check "mainfile" prefix.
3642	auto &SM = Context.getSourceManager();
3643	FileEntryRef MainFile = *SM.getFileEntryRefForID(FID: SM.getMainFileID());
3644	if (NoSanitizeL.containsMainFile(Mask: Kind, FileName: MainFile.getName()))
3645	return true;
3646
3647	// Check "src" prefix.
3648	if (Loc.isValid())
3649	return NoSanitizeL.containsLocation(Mask: Kind, Loc);
3650	// If location is unknown, this may be a compiler-generated function. Assume
3651	// it's located in the main file.
3652	return NoSanitizeL.containsFile(Mask: Kind, FileName: MainFile.getName());
3653	}
3654
3655	bool CodeGenModule::isInNoSanitizeList(SanitizerMask Kind,
3656	llvm::GlobalVariable *GV,
3657	SourceLocation Loc, QualType Ty,
3658	StringRef Category) const {
3659	const auto &NoSanitizeL = getContext().getNoSanitizeList();
3660	if (NoSanitizeL.containsGlobal(Mask: Kind, GlobalName: GV->getName(), Category))
3661	return true;
3662	auto &SM = Context.getSourceManager();
3663	if (NoSanitizeL.containsMainFile(
3664	Mask: Kind, FileName: SM.getFileEntryRefForID(FID: SM.getMainFileID())->getName(),
3665	Category))
3666	return true;
3667	if (NoSanitizeL.containsLocation(Mask: Kind, Loc, Category))
3668	return true;
3669
3670	// Check global type.
3671	if (!Ty.isNull()) {
3672	// Drill down the array types: if global variable of a fixed type is
3673	// not sanitized, we also don't instrument arrays of them.
3674	while (auto AT = dyn_cast<ArrayType>(Val: Ty.getTypePtr()))
3675	Ty = AT->getElementType();
3676	Ty = Ty.getCanonicalType().getUnqualifiedType();
3677	// Only record types (classes, structs etc.) are ignored.
3678	if (Ty->isRecordType()) {
3679	std::string TypeStr = Ty.getAsString(Policy: getContext().getPrintingPolicy());
3680	if (NoSanitizeL.containsType(Mask: Kind, MangledTypeName: TypeStr, Category))
3681	return true;
3682	}
3683	}
3684	return false;
3685	}
3686
3687	bool CodeGenModule::imbueXRayAttrs(llvm::Function *Fn, SourceLocation Loc,
3688	StringRef Category) const {
3689	const auto &XRayFilter = getContext().getXRayFilter();
3690	using ImbueAttr = XRayFunctionFilter::ImbueAttribute;
3691	auto Attr = ImbueAttr::NONE;
3692	if (Loc.isValid())
3693	Attr = XRayFilter.shouldImbueLocation(Loc, Category);
3694	if (Attr == ImbueAttr::NONE)
3695	Attr = XRayFilter.shouldImbueFunction(FunctionName: Fn->getName());
3696	switch (Attr) {
3697	case ImbueAttr::NONE:
3698	return false;
3699	case ImbueAttr::ALWAYS:
3700	Fn->addFnAttr(Kind: "function-instrument", Val: "xray-always");
3701	break;
3702	case ImbueAttr::ALWAYS_ARG1:
3703	Fn->addFnAttr(Kind: "function-instrument", Val: "xray-always");
3704	Fn->addFnAttr(Kind: "xray-log-args", Val: "1");
3705	break;
3706	case ImbueAttr::NEVER:
3707	Fn->addFnAttr(Kind: "function-instrument", Val: "xray-never");
3708	break;
3709	}
3710	return true;
3711	}
3712
3713	ProfileList::ExclusionType
3714	CodeGenModule::isFunctionBlockedByProfileList(llvm::Function *Fn,
3715	SourceLocation Loc) const {
3716	const auto &ProfileList = getContext().getProfileList();
3717	// If the profile list is empty, then instrument everything.
3718	if (ProfileList.isEmpty())
3719	return ProfileList::Allow;
3720	llvm::driver::ProfileInstrKind Kind = getCodeGenOpts().getProfileInstr();
3721	// First, check the function name.
3722	if (auto V = ProfileList.isFunctionExcluded(FunctionName: Fn->getName(), Kind))
3723	return *V;
3724	// Next, check the source location.
3725	if (Loc.isValid())
3726	if (auto V = ProfileList.isLocationExcluded(Loc, Kind))
3727	return *V;
3728	// If location is unknown, this may be a compiler-generated function. Assume
3729	// it's located in the main file.
3730	auto &SM = Context.getSourceManager();
3731	if (auto MainFile = SM.getFileEntryRefForID(FID: SM.getMainFileID()))
3732	if (auto V = ProfileList.isFileExcluded(FileName: MainFile->getName(), Kind))
3733	return *V;
3734	return ProfileList.getDefault(Kind);
3735	}
3736
3737	ProfileList::ExclusionType
3738	CodeGenModule::isFunctionBlockedFromProfileInstr(llvm::Function *Fn,
3739	SourceLocation Loc) const {
3740	auto V = isFunctionBlockedByProfileList(Fn, Loc);
3741	if (V != ProfileList::Allow)
3742	return V;
3743
3744	auto NumGroups = getCodeGenOpts().ProfileTotalFunctionGroups;
3745	if (NumGroups > 1) {
3746	auto Group = llvm::crc32(Data: arrayRefFromStringRef(Input: Fn->getName())) % NumGroups;
3747	if (Group != getCodeGenOpts().ProfileSelectedFunctionGroup)
3748	return ProfileList::Skip;
3749	}
3750	return ProfileList::Allow;
3751	}
3752
3753	bool CodeGenModule::MustBeEmitted(const ValueDecl *Global) {
3754	// Never defer when EmitAllDecls is specified.
3755	if (LangOpts.EmitAllDecls)
3756	return true;
3757
3758	const auto *VD = dyn_cast<VarDecl>(Val: Global);
3759	if (VD &&
3760	((CodeGenOpts.KeepPersistentStorageVariables &&
3761	(VD->getStorageDuration() == SD_Static ||
3762	VD->getStorageDuration() == SD_Thread)) ||
3763	(CodeGenOpts.KeepStaticConsts && VD->getStorageDuration() == SD_Static &&
3764	VD->getType().isConstQualified())))
3765	return true;
3766
3767	return getContext().DeclMustBeEmitted(D: Global);
3768	}
3769
3770	bool CodeGenModule::MayBeEmittedEagerly(const ValueDecl *Global) {
3771	// In OpenMP 5.0 variables and function may be marked as
3772	// device_type(host/nohost) and we should not emit them eagerly unless we sure
3773	// that they must be emitted on the host/device. To be sure we need to have
3774	// seen a declare target with an explicit mentioning of the function, we know
3775	// we have if the level of the declare target attribute is -1. Note that we
3776	// check somewhere else if we should emit this at all.
3777	if (LangOpts.OpenMP >= 50 && !LangOpts.OpenMPSimd) {
3778	std::optional<OMPDeclareTargetDeclAttr *> ActiveAttr =
3779	OMPDeclareTargetDeclAttr::getActiveAttr(VD: Global);
3780	if (!ActiveAttr || (*ActiveAttr)->getLevel() != (unsigned)-1)
3781	return false;
3782	}
3783
3784	if (const auto *FD = dyn_cast<FunctionDecl>(Val: Global)) {
3785	if (FD->getTemplateSpecializationKind() == TSK_ImplicitInstantiation)
3786	// Implicit template instantiations may change linkage if they are later
3787	// explicitly instantiated, so they should not be emitted eagerly.
3788	return false;
3789	// Defer until all versions have been semantically checked.
3790	if (FD->hasAttr<TargetVersionAttr>() && !FD->isMultiVersion())
3791	return false;
3792	// Defer emission of SYCL kernel entry point functions during device
3793	// compilation.
3794	if (LangOpts.SYCLIsDevice && FD->hasAttr<SYCLKernelEntryPointAttr>())
3795	return false;
3796	}
3797	if (const auto *VD = dyn_cast<VarDecl>(Val: Global)) {
3798	if (Context.getInlineVariableDefinitionKind(VD) ==
3799	ASTContext::InlineVariableDefinitionKind::WeakUnknown)
3800	// A definition of an inline constexpr static data member may change
3801	// linkage later if it's redeclared outside the class.
3802	return false;
3803	if (CXX20ModuleInits && VD->getOwningModule() &&
3804	!VD->getOwningModule()->isModuleMapModule()) {
3805	// For CXX20, module-owned initializers need to be deferred, since it is
3806	// not known at this point if they will be run for the current module or
3807	// as part of the initializer for an imported one.
3808	return false;
3809	}
3810	}
3811	// If OpenMP is enabled and threadprivates must be generated like TLS, delay
3812	// codegen for global variables, because they may be marked as threadprivate.
3813	if (LangOpts.OpenMP && LangOpts.OpenMPUseTLS &&
3814	getContext().getTargetInfo().isTLSSupported() && isa<VarDecl>(Val: Global) &&
3815	!Global->getType().isConstantStorage(Ctx: getContext(), ExcludeCtor: false, ExcludeDtor: false) &&
3816	!OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD: Global))
3817	return false;
3818
3819	return true;
3820	}
3821
3822	ConstantAddress CodeGenModule::GetAddrOfMSGuidDecl(const MSGuidDecl *GD) {
3823	StringRef Name = getMangledName(GD);
3824
3825	// The UUID descriptor should be pointer aligned.
3826	CharUnits Alignment = CharUnits::fromQuantity(Quantity: PointerAlignInBytes);
3827
3828	// Look for an existing global.
3829	if (llvm::GlobalVariable *GV = getModule().getNamedGlobal(Name))
3830	return ConstantAddress(GV, GV->getValueType(), Alignment);
3831
3832	ConstantEmitter Emitter(*this);
3833	llvm::Constant *Init;
3834
3835	APValue &V = GD->getAsAPValue();
3836	if (!V.isAbsent()) {
3837	// If possible, emit the APValue version of the initializer. In particular,
3838	// this gets the type of the constant right.
3839	Init = Emitter.emitForInitializer(
3840	value: GD->getAsAPValue(), destAddrSpace: GD->getType().getAddressSpace(), destType: GD->getType());
3841	} else {
3842	// As a fallback, directly construct the constant.
3843	// FIXME: This may get padding wrong under esoteric struct layout rules.
3844	// MSVC appears to create a complete type 'struct __s_GUID' that it
3845	// presumably uses to represent these constants.
3846	MSGuidDecl::Parts Parts = GD->getParts();
3847	llvm::Constant *Fields[4] = {
3848	llvm::ConstantInt::get(Ty: Int32Ty, V: Parts.Part1),
3849	llvm::ConstantInt::get(Ty: Int16Ty, V: Parts.Part2),
3850	llvm::ConstantInt::get(Ty: Int16Ty, V: Parts.Part3),
3851	llvm::ConstantDataArray::getRaw(
3852	Data: StringRef(reinterpret_cast<char *>(Parts.Part4And5), 8), NumElements: 8,
3853	ElementTy: Int8Ty)};
3854	Init = llvm::ConstantStruct::getAnon(V: Fields);
3855	}
3856
3857	auto *GV = new llvm::GlobalVariable(
3858	getModule(), Init->getType(),
3859	/*isConstant=*/true, llvm::GlobalValue::LinkOnceODRLinkage, Init, Name);
3860	if (supportsCOMDAT())
3861	GV->setComdat(TheModule.getOrInsertComdat(Name: GV->getName()));
3862	setDSOLocal(GV);
3863
3864	if (!V.isAbsent()) {
3865	Emitter.finalize(global: GV);
3866	return ConstantAddress(GV, GV->getValueType(), Alignment);
3867	}
3868
3869	llvm::Type *Ty = getTypes().ConvertTypeForMem(T: GD->getType());
3870	return ConstantAddress(GV, Ty, Alignment);
3871	}
3872
3873	ConstantAddress CodeGenModule::GetAddrOfUnnamedGlobalConstantDecl(
3874	const UnnamedGlobalConstantDecl *GCD) {
3875	CharUnits Alignment = getContext().getTypeAlignInChars(T: GCD->getType());
3876
3877	llvm::GlobalVariable **Entry = nullptr;
3878	Entry = &UnnamedGlobalConstantDeclMap[GCD];
3879	if (*Entry)
3880	return ConstantAddress(*Entry, (*Entry)->getValueType(), Alignment);
3881
3882	ConstantEmitter Emitter(*this);
3883	llvm::Constant *Init;
3884
3885	const APValue &V = GCD->getValue();
3886
3887	assert(!V.isAbsent());
3888	Init = Emitter.emitForInitializer(value: V, destAddrSpace: GCD->getType().getAddressSpace(),
3889	destType: GCD->getType());
3890
3891	auto *GV = new llvm::GlobalVariable(getModule(), Init->getType(),
3892	/*isConstant=*/true,
3893	llvm::GlobalValue::PrivateLinkage, Init,
3894	".constant");
3895	GV->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
3896	GV->setAlignment(Alignment.getAsAlign());
3897
3898	Emitter.finalize(global: GV);
3899
3900	*Entry = GV;
3901	return ConstantAddress(GV, GV->getValueType(), Alignment);
3902	}
3903
3904	ConstantAddress CodeGenModule::GetAddrOfTemplateParamObject(
3905	const TemplateParamObjectDecl *TPO) {
3906	StringRef Name = getMangledName(GD: TPO);
3907	CharUnits Alignment = getNaturalTypeAlignment(T: TPO->getType());
3908
3909	if (llvm::GlobalVariable *GV = getModule().getNamedGlobal(Name))
3910	return ConstantAddress(GV, GV->getValueType(), Alignment);
3911
3912	ConstantEmitter Emitter(*this);
3913	llvm::Constant *Init = Emitter.emitForInitializer(
3914	value: TPO->getValue(), destAddrSpace: TPO->getType().getAddressSpace(), destType: TPO->getType());
3915
3916	if (!Init) {
3917	ErrorUnsupported(D: TPO, Type: "template parameter object");
3918	return ConstantAddress::invalid();
3919	}
3920
3921	llvm::GlobalValue::LinkageTypes Linkage =
3922	isExternallyVisible(L: TPO->getLinkageAndVisibility().getLinkage())
3923	? llvm::GlobalValue::LinkOnceODRLinkage
3924	: llvm::GlobalValue::InternalLinkage;
3925	auto *GV = new llvm::GlobalVariable(getModule(), Init->getType(),
3926	/*isConstant=*/true, Linkage, Init, Name);
3927	setGVProperties(GV, D: TPO);
3928	if (supportsCOMDAT() && Linkage == llvm::GlobalValue::LinkOnceODRLinkage)
3929	GV->setComdat(TheModule.getOrInsertComdat(Name: GV->getName()));
3930	Emitter.finalize(global: GV);
3931
3932	return ConstantAddress(GV, GV->getValueType(), Alignment);
3933	}
3934
3935	ConstantAddress CodeGenModule::GetWeakRefReference(const ValueDecl *VD) {
3936	const AliasAttr *AA = VD->getAttr<AliasAttr>();
3937	assert(AA && "No alias?");
3938
3939	CharUnits Alignment = getContext().getDeclAlign(D: VD);
3940	llvm::Type *DeclTy = getTypes().ConvertTypeForMem(T: VD->getType());
3941
3942	// See if there is already something with the target's name in the module.
3943	llvm::GlobalValue *Entry = GetGlobalValue(Name: AA->getAliasee());
3944	if (Entry)
3945	return ConstantAddress(Entry, DeclTy, Alignment);
3946
3947	llvm::Constant *Aliasee;
3948	if (isa<llvm::FunctionType>(Val: DeclTy))
3949	Aliasee = GetOrCreateLLVMFunction(MangledName: AA->getAliasee(), Ty: DeclTy,
3950	D: GlobalDecl(cast<FunctionDecl>(Val: VD)),
3951	/*ForVTable=*/false);
3952	else
3953	Aliasee = GetOrCreateLLVMGlobal(MangledName: AA->getAliasee(), Ty: DeclTy, AddrSpace: LangAS::Default,
3954	D: nullptr);
3955
3956	auto *F = cast<llvm::GlobalValue>(Val: Aliasee);
3957	F->setLinkage(llvm::Function::ExternalWeakLinkage);
3958	WeakRefReferences.insert(Ptr: F);
3959
3960	return ConstantAddress(Aliasee, DeclTy, Alignment);
3961	}
3962
3963	template <typename AttrT> static bool hasImplicitAttr(const ValueDecl *D) {
3964	if (!D)
3965	return false;
3966	if (auto *A = D->getAttr<AttrT>())
3967	return A->isImplicit();
3968	return D->isImplicit();
3969	}
3970
3971	bool CodeGenModule::shouldEmitCUDAGlobalVar(const VarDecl *Global) const {
3972	assert(LangOpts.CUDA && "Should not be called by non-CUDA languages");
3973	// We need to emit host-side 'shadows' for all global
3974	// device-side variables because the CUDA runtime needs their
3975	// size and host-side address in order to provide access to
3976	// their device-side incarnations.
3977	return !LangOpts.CUDAIsDevice || Global->hasAttr<CUDADeviceAttr>() ||
3978	Global->hasAttr<CUDAConstantAttr>() ||
3979	Global->hasAttr<CUDASharedAttr>() ||
3980	Global->getType()->isCUDADeviceBuiltinSurfaceType() ||
3981	Global->getType()->isCUDADeviceBuiltinTextureType();
3982	}
3983
3984	void CodeGenModule::EmitGlobal(GlobalDecl GD) {
3985	const auto *Global = cast<ValueDecl>(Val: GD.getDecl());
3986
3987	// Weak references don't produce any output by themselves.
3988	if (Global->hasAttr<WeakRefAttr>())
3989	return;
3990
3991	// If this is an alias definition (which otherwise looks like a declaration)
3992	// emit it now.
3993	if (Global->hasAttr<AliasAttr>())
3994	return EmitAliasDefinition(GD);
3995
3996	// IFunc like an alias whose value is resolved at runtime by calling resolver.
3997	if (Global->hasAttr<IFuncAttr>())
3998	return emitIFuncDefinition(GD);
3999
4000	// If this is a cpu_dispatch multiversion function, emit the resolver.
4001	if (Global->hasAttr<CPUDispatchAttr>())
4002	return emitCPUDispatchDefinition(GD);
4003
4004	// If this is CUDA, be selective about which declarations we emit.
4005	// Non-constexpr non-lambda implicit host device functions are not emitted
4006	// unless they are used on device side.
4007	if (LangOpts.CUDA) {
4008	assert((isa<FunctionDecl>(Global) || isa<VarDecl>(Global)) &&
4009	"Expected Variable or Function");
4010	if (const auto *VD = dyn_cast<VarDecl>(Val: Global)) {
4011	if (!shouldEmitCUDAGlobalVar(Global: VD))
4012	return;
4013	} else if (LangOpts.CUDAIsDevice) {
4014	const auto *FD = dyn_cast<FunctionDecl>(Val: Global);
4015	if ((!Global->hasAttr<CUDADeviceAttr>() ||
4016	(LangOpts.OffloadImplicitHostDeviceTemplates &&
4017	hasImplicitAttr<CUDAHostAttr>(D: FD) &&
4018	hasImplicitAttr<CUDADeviceAttr>(D: FD) && !FD->isConstexpr() &&
4019	!isLambdaCallOperator(DC: FD) &&
4020	!getContext().CUDAImplicitHostDeviceFunUsedByDevice.count(V: FD))) &&
4021	!Global->hasAttr<CUDAGlobalAttr>() &&
4022	!(LangOpts.HIPStdPar && isa<FunctionDecl>(Val: Global) &&
4023	!Global->hasAttr<CUDAHostAttr>()))
4024	return;
4025	// Device-only functions are the only things we skip.
4026	} else if (!Global->hasAttr<CUDAHostAttr>() &&
4027	Global->hasAttr<CUDADeviceAttr>())
4028	return;
4029	}
4030
4031	if (LangOpts.OpenMP) {
4032	// If this is OpenMP, check if it is legal to emit this global normally.
4033	if (OpenMPRuntime && OpenMPRuntime->emitTargetGlobal(GD))
4034	return;
4035	if (auto *DRD = dyn_cast<OMPDeclareReductionDecl>(Val: Global)) {
4036	if (MustBeEmitted(Global))
4037	EmitOMPDeclareReduction(D: DRD);
4038	return;
4039	}
4040	if (auto *DMD = dyn_cast<OMPDeclareMapperDecl>(Val: Global)) {
4041	if (MustBeEmitted(Global))
4042	EmitOMPDeclareMapper(D: DMD);
4043	return;
4044	}
4045	}
4046
4047	// Ignore declarations, they will be emitted on their first use.
4048	if (const auto *FD = dyn_cast<FunctionDecl>(Val: Global)) {
4049	if (DeviceKernelAttr::isOpenCLSpelling(A: FD->getAttr<DeviceKernelAttr>()) &&
4050	FD->doesThisDeclarationHaveABody())
4051	addDeferredDeclToEmit(GD: GlobalDecl(FD, KernelReferenceKind::Stub));
4052
4053	// Update deferred annotations with the latest declaration if the function
4054	// function was already used or defined.
4055	if (FD->hasAttr<AnnotateAttr>()) {
4056	StringRef MangledName = getMangledName(GD);
4057	if (GetGlobalValue(Name: MangledName))
4058	DeferredAnnotations[MangledName] = FD;
4059	}
4060
4061	// Forward declarations are emitted lazily on first use.
4062	if (!FD->doesThisDeclarationHaveABody()) {
4063	if (!FD->doesDeclarationForceExternallyVisibleDefinition() &&
4064	(!FD->isMultiVersion() || !getTarget().getTriple().isAArch64()))
4065	return;
4066
4067	StringRef MangledName = getMangledName(GD);
4068
4069	// Compute the function info and LLVM type.
4070	const CGFunctionInfo &FI = getTypes().arrangeGlobalDeclaration(GD);
4071	llvm::Type *Ty = getTypes().GetFunctionType(Info: FI);
4072
4073	GetOrCreateLLVMFunction(MangledName, Ty, D: GD, /*ForVTable=*/false,
4074	/*DontDefer=*/false);
4075	return;
4076	}
4077	} else {
4078	const auto *VD = cast<VarDecl>(Val: Global);
4079	assert(VD->isFileVarDecl() && "Cannot emit local var decl as global.");
4080	if (VD->isThisDeclarationADefinition() != VarDecl::Definition &&
4081	!Context.isMSStaticDataMemberInlineDefinition(VD)) {
4082	if (LangOpts.OpenMP) {
4083	// Emit declaration of the must-be-emitted declare target variable.
4084	if (std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
4085	OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD)) {
4086
4087	// If this variable has external storage and doesn't require special
4088	// link handling we defer to its canonical definition.
4089	if (VD->hasExternalStorage() &&
4090	Res != OMPDeclareTargetDeclAttr::MT_Link)
4091	return;
4092
4093	bool UnifiedMemoryEnabled =
4094	getOpenMPRuntime().hasRequiresUnifiedSharedMemory();
4095	if ((*Res == OMPDeclareTargetDeclAttr::MT_To ||
4096	*Res == OMPDeclareTargetDeclAttr::MT_Enter) &&
4097	!UnifiedMemoryEnabled) {
4098	(void)GetAddrOfGlobalVar(D: VD);
4099	} else {
4100	assert(((*Res == OMPDeclareTargetDeclAttr::MT_Link) ||
4101	((*Res == OMPDeclareTargetDeclAttr::MT_To ||
4102	*Res == OMPDeclareTargetDeclAttr::MT_Enter) &&
4103	UnifiedMemoryEnabled)) &&
4104	"Link clause or to clause with unified memory expected.");
4105	(void)getOpenMPRuntime().getAddrOfDeclareTargetVar(VD);
4106	}
4107
4108	return;
4109	}
4110	}
4111	// If this declaration may have caused an inline variable definition to
4112	// change linkage, make sure that it's emitted.
4113	if (Context.getInlineVariableDefinitionKind(VD) ==
4114	ASTContext::InlineVariableDefinitionKind::Strong)
4115	GetAddrOfGlobalVar(D: VD);
4116	return;
4117	}
4118	}
4119
4120	// Defer code generation to first use when possible, e.g. if this is an inline
4121	// function. If the global must always be emitted, do it eagerly if possible
4122	// to benefit from cache locality.
4123	if (MustBeEmitted(Global) && MayBeEmittedEagerly(Global)) {
4124	// Emit the definition if it can't be deferred.
4125	EmitGlobalDefinition(D: GD);
4126	addEmittedDeferredDecl(GD);
4127	return;
4128	}
4129
4130	// If we're deferring emission of a C++ variable with an
4131	// initializer, remember the order in which it appeared in the file.
4132	if (getLangOpts().CPlusPlus && isa<VarDecl>(Val: Global) &&
4133	cast<VarDecl>(Val: Global)->hasInit()) {
4134	DelayedCXXInitPosition[Global] = CXXGlobalInits.size();
4135	CXXGlobalInits.push_back(x: nullptr);
4136	}
4137
4138	StringRef MangledName = getMangledName(GD);
4139	if (GetGlobalValue(Name: MangledName) != nullptr) {
4140	// The value has already been used and should therefore be emitted.
4141	addDeferredDeclToEmit(GD);
4142	} else if (MustBeEmitted(Global)) {
4143	// The value must be emitted, but cannot be emitted eagerly.
4144	assert(!MayBeEmittedEagerly(Global));
4145	addDeferredDeclToEmit(GD);
4146	} else {
4147	// Otherwise, remember that we saw a deferred decl with this name. The
4148	// first use of the mangled name will cause it to move into
4149	// DeferredDeclsToEmit.
4150	DeferredDecls[MangledName] = GD;
4151	}
4152	}
4153
4154	// Check if T is a class type with a destructor that's not dllimport.
4155	static bool HasNonDllImportDtor(QualType T) {
4156	if (const auto *RT = T->getBaseElementTypeUnsafe()->getAs<RecordType>())
4157	if (CXXRecordDecl *RD = dyn_cast<CXXRecordDecl>(Val: RT->getDecl()))
4158	if (RD->getDestructor() && !RD->getDestructor()->hasAttr<DLLImportAttr>())
4159	return true;
4160
4161	return false;
4162	}
4163
4164	namespace {
4165	struct FunctionIsDirectlyRecursive
4166	: public ConstStmtVisitor<FunctionIsDirectlyRecursive, bool> {
4167	const StringRef Name;
4168	const Builtin::Context &BI;
4169	FunctionIsDirectlyRecursive(StringRef N, const Builtin::Context &C)
4170	: Name(N), BI(C) {}
4171
4172	bool VisitCallExpr(const CallExpr *E) {
4173	const FunctionDecl *FD = E->getDirectCallee();
4174	if (!FD)
4175	return false;
4176	AsmLabelAttr *Attr = FD->getAttr<AsmLabelAttr>();
4177	if (Attr && Name == Attr->getLabel())
4178	return true;
4179	unsigned BuiltinID = FD->getBuiltinID();
4180	if (!BuiltinID || !BI.isLibFunction(ID: BuiltinID))
4181	return false;
4182	std::string BuiltinNameStr = BI.getName(ID: BuiltinID);
4183	StringRef BuiltinName = BuiltinNameStr;
4184	return BuiltinName.consume_front(Prefix: "__builtin_") && Name == BuiltinName;
4185	}
4186
4187	bool VisitStmt(const Stmt *S) {
4188	for (const Stmt *Child : S->children())
4189	if (Child && this->Visit(S: Child))
4190	return true;
4191	return false;
4192	}
4193	};
4194
4195	// Make sure we're not referencing non-imported vars or functions.
4196	struct DLLImportFunctionVisitor
4197	: public RecursiveASTVisitor<DLLImportFunctionVisitor> {
4198	bool SafeToInline = true;
4199
4200	bool shouldVisitImplicitCode() const { return true; }
4201
4202	bool VisitVarDecl(VarDecl *VD) {
4203	if (VD->getTLSKind()) {
4204	// A thread-local variable cannot be imported.
4205	SafeToInline = false;
4206	return SafeToInline;
4207	}
4208
4209	// A variable definition might imply a destructor call.
4210	if (VD->isThisDeclarationADefinition())
4211	SafeToInline = !HasNonDllImportDtor(T: VD->getType());
4212
4213	return SafeToInline;
4214	}
4215
4216	bool VisitCXXBindTemporaryExpr(CXXBindTemporaryExpr *E) {
4217	if (const auto *D = E->getTemporary()->getDestructor())
4218	SafeToInline = D->hasAttr<DLLImportAttr>();
4219	return SafeToInline;
4220	}
4221
4222	bool VisitDeclRefExpr(DeclRefExpr *E) {
4223	ValueDecl *VD = E->getDecl();
4224	if (isa<FunctionDecl>(Val: VD))
4225	SafeToInline = VD->hasAttr<DLLImportAttr>();
4226	else if (VarDecl *V = dyn_cast<VarDecl>(Val: VD))
4227	SafeToInline = !V->hasGlobalStorage() || V->hasAttr<DLLImportAttr>();
4228	return SafeToInline;
4229	}
4230
4231	bool VisitCXXConstructExpr(CXXConstructExpr *E) {
4232	SafeToInline = E->getConstructor()->hasAttr<DLLImportAttr>();
4233	return SafeToInline;
4234	}
4235
4236	bool VisitCXXMemberCallExpr(CXXMemberCallExpr *E) {
4237	CXXMethodDecl *M = E->getMethodDecl();
4238	if (!M) {
4239	// Call through a pointer to member function. This is safe to inline.
4240	SafeToInline = true;
4241	} else {
4242	SafeToInline = M->hasAttr<DLLImportAttr>();
4243	}
4244	return SafeToInline;
4245	}
4246
4247	bool VisitCXXDeleteExpr(CXXDeleteExpr *E) {
4248	SafeToInline = E->getOperatorDelete()->hasAttr<DLLImportAttr>();
4249	return SafeToInline;
4250	}
4251
4252	bool VisitCXXNewExpr(CXXNewExpr *E) {
4253	SafeToInline = E->getOperatorNew()->hasAttr<DLLImportAttr>();
4254	return SafeToInline;
4255	}
4256	};
4257	}
4258
4259	// isTriviallyRecursive - Check if this function calls another
4260	// decl that, because of the asm attribute or the other decl being a builtin,
4261	// ends up pointing to itself.
4262	bool
4263	CodeGenModule::isTriviallyRecursive(const FunctionDecl *FD) {
4264	StringRef Name;
4265	if (getCXXABI().getMangleContext().shouldMangleDeclName(D: FD)) {
4266	// asm labels are a special kind of mangling we have to support.
4267	AsmLabelAttr *Attr = FD->getAttr<AsmLabelAttr>();
4268	if (!Attr)
4269	return false;
4270	Name = Attr->getLabel();
4271	} else {
4272	Name = FD->getName();
4273	}
4274
4275	FunctionIsDirectlyRecursive Walker(Name, Context.BuiltinInfo);
4276	const Stmt *Body = FD->getBody();
4277	return Body ? Walker.Visit(S: Body) : false;
4278	}
4279
4280	bool CodeGenModule::shouldEmitFunction(GlobalDecl GD) {
4281	if (getFunctionLinkage(GD) != llvm::Function::AvailableExternallyLinkage)
4282	return true;
4283
4284	const auto *F = cast<FunctionDecl>(Val: GD.getDecl());
4285	// Inline builtins declaration must be emitted. They often are fortified
4286	// functions.
4287	if (F->isInlineBuiltinDeclaration())
4288	return true;
4289
4290	if (CodeGenOpts.OptimizationLevel == 0 && !F->hasAttr<AlwaysInlineAttr>())
4291	return false;
4292
4293	// We don't import function bodies from other named module units since that
4294	// behavior may break ABI compatibility of the current unit.
4295	if (const Module *M = F->getOwningModule();
4296	M && M->getTopLevelModule()->isNamedModule() &&
4297	getContext().getCurrentNamedModule() != M->getTopLevelModule()) {
4298	// There are practices to mark template member function as always-inline
4299	// and mark the template as extern explicit instantiation but not give
4300	// the definition for member function. So we have to emit the function
4301	// from explicitly instantiation with always-inline.
4302	//
4303	// See https://github.com/llvm/llvm-project/issues/86893 for details.
4304	//
4305	// TODO: Maybe it is better to give it a warning if we call a non-inline
4306	// function from other module units which is marked as always-inline.
4307	if (!F->isTemplateInstantiation() || !F->hasAttr<AlwaysInlineAttr>()) {
4308	return false;
4309	}
4310	}
4311
4312	if (F->hasAttr<NoInlineAttr>())
4313	return false;
4314
4315	if (F->hasAttr<DLLImportAttr>() && !F->hasAttr<AlwaysInlineAttr>()) {
4316	// Check whether it would be safe to inline this dllimport function.
4317	DLLImportFunctionVisitor Visitor;
4318	Visitor.TraverseFunctionDecl(D: const_cast<FunctionDecl*>(F));
4319	if (!Visitor.SafeToInline)
4320	return false;
4321
4322	if (const CXXDestructorDecl *Dtor = dyn_cast<CXXDestructorDecl>(Val: F)) {
4323	// Implicit destructor invocations aren't captured in the AST, so the
4324	// check above can't see them. Check for them manually here.
4325	for (const Decl *Member : Dtor->getParent()->decls())
4326	if (isa<FieldDecl>(Val: Member))
4327	if (HasNonDllImportDtor(T: cast<FieldDecl>(Val: Member)->getType()))
4328	return false;
4329	for (const CXXBaseSpecifier &B : Dtor->getParent()->bases())
4330	if (HasNonDllImportDtor(T: B.getType()))
4331	return false;
4332	}
4333	}
4334
4335	// PR9614. Avoid cases where the source code is lying to us. An available
4336	// externally function should have an equivalent function somewhere else,
4337	// but a function that calls itself through asm label/`__builtin_` trickery is
4338	// clearly not equivalent to the real implementation.
4339	// This happens in glibc's btowc and in some configure checks.
4340	return !isTriviallyRecursive(FD: F);
4341	}
4342
4343	bool CodeGenModule::shouldOpportunisticallyEmitVTables() {
4344	return CodeGenOpts.OptimizationLevel > 0;
4345	}
4346
4347	void CodeGenModule::EmitMultiVersionFunctionDefinition(GlobalDecl GD,
4348	llvm::GlobalValue *GV) {
4349	const auto *FD = cast<FunctionDecl>(Val: GD.getDecl());
4350
4351	if (FD->isCPUSpecificMultiVersion()) {
4352	auto *Spec = FD->getAttr<CPUSpecificAttr>();
4353	for (unsigned I = 0; I < Spec->cpus_size(); ++I)
4354	EmitGlobalFunctionDefinition(GD: GD.getWithMultiVersionIndex(Index: I), GV: nullptr);
4355	} else if (auto *TC = FD->getAttr<TargetClonesAttr>()) {
4356	for (unsigned I = 0; I < TC->featuresStrs_size(); ++I)
4357	if (TC->isFirstOfVersion(Index: I))
4358	EmitGlobalFunctionDefinition(GD: GD.getWithMultiVersionIndex(Index: I), GV: nullptr);
4359	} else
4360	EmitGlobalFunctionDefinition(GD, GV);
4361
4362	// Ensure that the resolver function is also emitted.
4363	if (FD->isTargetVersionMultiVersion() || FD->isTargetClonesMultiVersion()) {
4364	// On AArch64 defer the resolver emission until the entire TU is processed.
4365	if (getTarget().getTriple().isAArch64())
4366	AddDeferredMultiVersionResolverToEmit(GD);
4367	else
4368	GetOrCreateMultiVersionResolver(GD);
4369	}
4370	}
4371
4372	void CodeGenModule::EmitGlobalDefinition(GlobalDecl GD, llvm::GlobalValue *GV) {
4373	const auto *D = cast<ValueDecl>(Val: GD.getDecl());
4374
4375	PrettyStackTraceDecl CrashInfo(const_cast<ValueDecl *>(D), D->getLocation(),
4376	Context.getSourceManager(),
4377	"Generating code for declaration");
4378
4379	if (const auto *FD = dyn_cast<FunctionDecl>(Val: D)) {
4380	// At -O0, don't generate IR for functions with available_externally
4381	// linkage.
4382	if (!shouldEmitFunction(GD))
4383	return;
4384
4385	llvm::TimeTraceScope TimeScope("CodeGen Function", [&]() {
4386	std::string Name;
4387	llvm::raw_string_ostream OS(Name);
4388	FD->getNameForDiagnostic(OS, Policy: getContext().getPrintingPolicy(),
4389	/*Qualified=*/true);
4390	return Name;
4391	});
4392
4393	if (const auto *Method = dyn_cast<CXXMethodDecl>(Val: D)) {
4394	// Make sure to emit the definition(s) before we emit the thunks.
4395	// This is necessary for the generation of certain thunks.
4396	if (isa<CXXConstructorDecl>(Val: Method) || isa<CXXDestructorDecl>(Val: Method))
4397	ABI->emitCXXStructor(GD);
4398	else if (FD->isMultiVersion())
4399	EmitMultiVersionFunctionDefinition(GD, GV);
4400	else
4401	EmitGlobalFunctionDefinition(GD, GV);
4402
4403	if (Method->isVirtual())
4404	getVTables().EmitThunks(GD);
4405
4406	return;
4407	}
4408
4409	if (FD->isMultiVersion())
4410	return EmitMultiVersionFunctionDefinition(GD, GV);
4411	return EmitGlobalFunctionDefinition(GD, GV);
4412	}
4413
4414	if (const auto *VD = dyn_cast<VarDecl>(Val: D))
4415	return EmitGlobalVarDefinition(D: VD, IsTentative: !VD->hasDefinition());
4416
4417	llvm_unreachable("Invalid argument to EmitGlobalDefinition()");
4418	}
4419
4420	static void ReplaceUsesOfNonProtoTypeWithRealFunction(llvm::GlobalValue *Old,
4421	llvm::Function *NewFn);
4422
4423	static uint64_t getFMVPriority(const TargetInfo &TI,
4424	const CodeGenFunction::FMVResolverOption &RO) {
4425	llvm::SmallVector<StringRef, 8> Features{RO.Features};
4426	if (RO.Architecture)
4427	Features.push_back(Elt: *RO.Architecture);
4428	return TI.getFMVPriority(Features);
4429	}
4430
4431	// Multiversion functions should be at most 'WeakODRLinkage' so that a different
4432	// TU can forward declare the function without causing problems. Particularly
4433	// in the cases of CPUDispatch, this causes issues. This also makes sure we
4434	// work with internal linkage functions, so that the same function name can be
4435	// used with internal linkage in multiple TUs.
4436	static llvm::GlobalValue::LinkageTypes
4437	getMultiversionLinkage(CodeGenModule &CGM, GlobalDecl GD) {
4438	const FunctionDecl *FD = cast<FunctionDecl>(Val: GD.getDecl());
4439	if (FD->getFormalLinkage() == Linkage::Internal)
4440	return llvm::GlobalValue::InternalLinkage;
4441	return llvm::GlobalValue::WeakODRLinkage;
4442	}
4443
4444	void CodeGenModule::emitMultiVersionFunctions() {
4445	std::vector<GlobalDecl> MVFuncsToEmit;
4446	MultiVersionFuncs.swap(x&: MVFuncsToEmit);
4447	for (GlobalDecl GD : MVFuncsToEmit) {
4448	const auto *FD = cast<FunctionDecl>(Val: GD.getDecl());
4449	assert(FD && "Expected a FunctionDecl");
4450
4451	auto createFunction = [&](const FunctionDecl *Decl, unsigned MVIdx = 0) {
4452	GlobalDecl CurGD{Decl->isDefined() ? Decl->getDefinition() : Decl, MVIdx};
4453	StringRef MangledName = getMangledName(GD: CurGD);
4454	llvm::Constant *Func = GetGlobalValue(Name: MangledName);
4455	if (!Func) {
4456	if (Decl->isDefined()) {
4457	EmitGlobalFunctionDefinition(GD: CurGD, GV: nullptr);
4458	Func = GetGlobalValue(Name: MangledName);
4459	} else {
4460	const CGFunctionInfo &FI = getTypes().arrangeGlobalDeclaration(GD: CurGD);
4461	llvm::FunctionType *Ty = getTypes().GetFunctionType(Info: FI);
4462	Func = GetAddrOfFunction(GD: CurGD, Ty, /*ForVTable=*/false,
4463	/*DontDefer=*/false, IsForDefinition: ForDefinition);
4464	}
4465	assert(Func && "This should have just been created");
4466	}
4467	return cast<llvm::Function>(Val: Func);
4468	};
4469
4470	// For AArch64, a resolver is only emitted if a function marked with
4471	// target_version("default")) or target_clones("default") is defined
4472	// in this TU. For other architectures it is always emitted.
4473	bool ShouldEmitResolver = !getTarget().getTriple().isAArch64();
4474	SmallVector<CodeGenFunction::FMVResolverOption, 10> Options;
4475
4476	getContext().forEachMultiversionedFunctionVersion(
4477	FD, Pred: [&](const FunctionDecl *CurFD) {
4478	llvm::SmallVector<StringRef, 8> Feats;
4479	bool IsDefined = CurFD->getDefinition() != nullptr;
4480
4481	if (const auto *TA = CurFD->getAttr<TargetAttr>()) {
4482	assert(getTarget().getTriple().isX86() && "Unsupported target");
4483	TA->getX86AddedFeatures(Out&: Feats);
4484	llvm::Function *Func = createFunction(CurFD);
4485	Options.emplace_back(Args&: Func, Args&: Feats, Args: TA->getX86Architecture());
4486	} else if (const auto *TVA = CurFD->getAttr<TargetVersionAttr>()) {
4487	if (TVA->isDefaultVersion() && IsDefined)
4488	ShouldEmitResolver = true;
4489	llvm::Function *Func = createFunction(CurFD);
4490	char Delim = getTarget().getTriple().isAArch64() ? '+' : ',';
4491	TVA->getFeatures(Out&: Feats, Delim);
4492	Options.emplace_back(Args&: Func, Args&: Feats);
4493	} else if (const auto *TC = CurFD->getAttr<TargetClonesAttr>()) {
4494	for (unsigned I = 0; I < TC->featuresStrs_size(); ++I) {
4495	if (!TC->isFirstOfVersion(Index: I))
4496	continue;
4497	if (TC->isDefaultVersion(Index: I) && IsDefined)
4498	ShouldEmitResolver = true;
4499	llvm::Function *Func = createFunction(CurFD, I);
4500	Feats.clear();
4501	if (getTarget().getTriple().isX86()) {
4502	TC->getX86Feature(Out&: Feats, Index: I);
4503	Options.emplace_back(Args&: Func, Args&: Feats, Args: TC->getX86Architecture(Index: I));
4504	} else {
4505	char Delim = getTarget().getTriple().isAArch64() ? '+' : ',';
4506	TC->getFeatures(Out&: Feats, Index: I, Delim);
4507	Options.emplace_back(Args&: Func, Args&: Feats);
4508	}
4509	}
4510	} else
4511	llvm_unreachable("unexpected MultiVersionKind");
4512	});
4513
4514	if (!ShouldEmitResolver)
4515	continue;
4516
4517	llvm::Constant *ResolverConstant = GetOrCreateMultiVersionResolver(GD);
4518	if (auto *IFunc = dyn_cast<llvm::GlobalIFunc>(Val: ResolverConstant)) {
4519	ResolverConstant = IFunc->getResolver();
4520	if (FD->isTargetClonesMultiVersion() &&
4521	!getTarget().getTriple().isAArch64()) {
4522	std::string MangledName = getMangledNameImpl(
4523	CGM&: *this, GD, ND: FD, /*OmitMultiVersionMangling=*/true);
4524	if (!GetGlobalValue(Name: MangledName + ".ifunc")) {
4525	const CGFunctionInfo &FI = getTypes().arrangeGlobalDeclaration(GD);
4526	llvm::FunctionType *DeclTy = getTypes().GetFunctionType(Info: FI);
4527	// In prior versions of Clang, the mangling for ifuncs incorrectly
4528	// included an .ifunc suffix. This alias is generated for backward
4529	// compatibility. It is deprecated, and may be removed in the future.
4530	auto *Alias = llvm::GlobalAlias::create(
4531	Ty: DeclTy, AddressSpace: 0, Linkage: getMultiversionLinkage(CGM&: *this, GD),
4532	Name: MangledName + ".ifunc", Aliasee: IFunc, Parent: &getModule());
4533	SetCommonAttributes(GD: FD, GV: Alias);
4534	}
4535	}
4536	}
4537	llvm::Function *ResolverFunc = cast<llvm::Function>(Val: ResolverConstant);
4538
4539	ResolverFunc->setLinkage(getMultiversionLinkage(CGM&: *this, GD));
4540
4541	if (!ResolverFunc->hasLocalLinkage() && supportsCOMDAT())
4542	ResolverFunc->setComdat(
4543	getModule().getOrInsertComdat(Name: ResolverFunc->getName()));
4544
4545	const TargetInfo &TI = getTarget();
4546	llvm::stable_sort(
4547	Range&: Options, C: [&TI](const CodeGenFunction::FMVResolverOption &LHS,
4548	const CodeGenFunction::FMVResolverOption &RHS) {
4549	return getFMVPriority(TI, RO: LHS) > getFMVPriority(TI, RO: RHS);
4550	});
4551	CodeGenFunction CGF(*this);
4552	CGF.EmitMultiVersionResolver(Resolver: ResolverFunc, Options);
4553	}
4554
4555	// Ensure that any additions to the deferred decls list caused by emitting a
4556	// variant are emitted. This can happen when the variant itself is inline and
4557	// calls a function without linkage.
4558	if (!MVFuncsToEmit.empty())
4559	EmitDeferred();
4560
4561	// Ensure that any additions to the multiversion funcs list from either the
4562	// deferred decls or the multiversion functions themselves are emitted.
4563	if (!MultiVersionFuncs.empty())
4564	emitMultiVersionFunctions();
4565	}
4566
4567	static void replaceDeclarationWith(llvm::GlobalValue *Old,
4568	llvm::Constant *New) {
4569	assert(cast<llvm::Function>(Old)->isDeclaration() && "Not a declaration");
4570	New->takeName(V: Old);
4571	Old->replaceAllUsesWith(V: New);
4572	Old->eraseFromParent();
4573	}
4574
4575	void CodeGenModule::emitCPUDispatchDefinition(GlobalDecl GD) {
4576	const auto *FD = cast<FunctionDecl>(Val: GD.getDecl());
4577	assert(FD && "Not a FunctionDecl?");
4578	assert(FD->isCPUDispatchMultiVersion() && "Not a multiversion function?");
4579	const auto *DD = FD->getAttr<CPUDispatchAttr>();
4580	assert(DD && "Not a cpu_dispatch Function?");
4581
4582	const CGFunctionInfo &FI = getTypes().arrangeGlobalDeclaration(GD);
4583	llvm::FunctionType *DeclTy = getTypes().GetFunctionType(Info: FI);
4584
4585	StringRef ResolverName = getMangledName(GD);
4586	UpdateMultiVersionNames(GD, FD, CurName&: ResolverName);
4587
4588	llvm::Type *ResolverType;
4589	GlobalDecl ResolverGD;
4590	if (getTarget().supportsIFunc()) {
4591	ResolverType = llvm::FunctionType::get(
4592	Result: llvm::PointerType::get(C&: getLLVMContext(),
4593	AddressSpace: getTypes().getTargetAddressSpace(T: FD->getType())),
4594	isVarArg: false);
4595	}
4596	else {
4597	ResolverType = DeclTy;
4598	ResolverGD = GD;
4599	}
4600
4601	auto *ResolverFunc = cast<llvm::Function>(Val: GetOrCreateLLVMFunction(
4602	MangledName: ResolverName, Ty: ResolverType, D: ResolverGD, /*ForVTable=*/false));
4603	ResolverFunc->setLinkage(getMultiversionLinkage(CGM&: *this, GD));
4604	if (supportsCOMDAT())
4605	ResolverFunc->setComdat(
4606	getModule().getOrInsertComdat(Name: ResolverFunc->getName()));
4607
4608	SmallVector<CodeGenFunction::FMVResolverOption, 10> Options;
4609	const TargetInfo &Target = getTarget();
4610	unsigned Index = 0;
4611	for (const IdentifierInfo *II : DD->cpus()) {
4612	// Get the name of the target function so we can look it up/create it.
4613	std::string MangledName = getMangledNameImpl(CGM&: *this, GD, ND: FD, OmitMultiVersionMangling: true) +
4614	getCPUSpecificMangling(CGM: *this, Name: II->getName());
4615
4616	llvm::Constant *Func = GetGlobalValue(Name: MangledName);
4617
4618	if (!Func) {
4619	GlobalDecl ExistingDecl = Manglings.lookup(Key: MangledName);
4620	if (ExistingDecl.getDecl() &&
4621	ExistingDecl.getDecl()->getAsFunction()->isDefined()) {
4622	EmitGlobalFunctionDefinition(GD: ExistingDecl, GV: nullptr);
4623	Func = GetGlobalValue(Name: MangledName);
4624	} else {
4625	if (!ExistingDecl.getDecl())
4626	ExistingDecl = GD.getWithMultiVersionIndex(Index);
4627
4628	Func = GetOrCreateLLVMFunction(
4629	MangledName, Ty: DeclTy, D: ExistingDecl,
4630	/*ForVTable=*/false, /*DontDefer=*/true,
4631	/*IsThunk=*/false, ExtraAttrs: llvm::AttributeList(), IsForDefinition: ForDefinition);
4632	}
4633	}
4634
4635	llvm::SmallVector<StringRef, 32> Features;
4636	Target.getCPUSpecificCPUDispatchFeatures(Name: II->getName(), Features);
4637	llvm::transform(Range&: Features, d_first: Features.begin(),
4638	F: [](StringRef Str) { return Str.substr(Start: 1); });
4639	llvm::erase_if(C&: Features, P: [&Target](StringRef Feat) {
4640	return !Target.validateCpuSupports(Name: Feat);
4641	});
4642	Options.emplace_back(Args: cast<llvm::Function>(Val: Func), Args&: Features);
4643	++Index;
4644	}
4645
4646	llvm::stable_sort(Range&: Options, C: [](const CodeGenFunction::FMVResolverOption &LHS,
4647	const CodeGenFunction::FMVResolverOption &RHS) {
4648	return llvm::X86::getCpuSupportsMask(FeatureStrs: LHS.Features) >
4649	llvm::X86::getCpuSupportsMask(FeatureStrs: RHS.Features);
4650	});
4651
4652	// If the list contains multiple 'default' versions, such as when it contains
4653	// 'pentium' and 'generic', don't emit the call to the generic one (since we
4654	// always run on at least a 'pentium'). We do this by deleting the 'least
4655	// advanced' (read, lowest mangling letter).
4656	while (Options.size() > 1 && llvm::all_of(Range: llvm::X86::getCpuSupportsMask(
4657	FeatureStrs: (Options.end() - 2)->Features),
4658	P: [](auto X) { return X == 0; })) {
4659	StringRef LHSName = (Options.end() - 2)->Function->getName();
4660	StringRef RHSName = (Options.end() - 1)->Function->getName();
4661	if (LHSName.compare(RHS: RHSName) < 0)
4662	Options.erase(CI: Options.end() - 2);
4663	else
4664	Options.erase(CI: Options.end() - 1);
4665	}
4666
4667	CodeGenFunction CGF(*this);
4668	CGF.EmitMultiVersionResolver(Resolver: ResolverFunc, Options);
4669
4670	if (getTarget().supportsIFunc()) {
4671	llvm::GlobalValue::LinkageTypes Linkage = getMultiversionLinkage(CGM&: *this, GD);
4672	auto *IFunc = cast<llvm::GlobalValue>(Val: GetOrCreateMultiVersionResolver(GD));
4673	unsigned AS = IFunc->getType()->getPointerAddressSpace();
4674
4675	// Fix up function declarations that were created for cpu_specific before
4676	// cpu_dispatch was known
4677	if (!isa<llvm::GlobalIFunc>(Val: IFunc)) {
4678	auto *GI = llvm::GlobalIFunc::create(Ty: DeclTy, AddressSpace: AS, Linkage, Name: "",
4679	Resolver: ResolverFunc, Parent: &getModule());
4680	replaceDeclarationWith(Old: IFunc, New: GI);
4681	IFunc = GI;
4682	}
4683
4684	std::string AliasName = getMangledNameImpl(
4685	CGM&: *this, GD, ND: FD, /*OmitMultiVersionMangling=*/true);
4686	llvm::Constant *AliasFunc = GetGlobalValue(Name: AliasName);
4687	if (!AliasFunc) {
4688	auto *GA = llvm::GlobalAlias::create(Ty: DeclTy, AddressSpace: AS, Linkage, Name: AliasName,
4689	Aliasee: IFunc, Parent: &getModule());
4690	SetCommonAttributes(GD, GV: GA);
4691	}
4692	}
4693	}
4694
4695	/// Adds a declaration to the list of multi version functions if not present.
4696	void CodeGenModule::AddDeferredMultiVersionResolverToEmit(GlobalDecl GD) {
4697	const auto *FD = cast<FunctionDecl>(Val: GD.getDecl());
4698	assert(FD && "Not a FunctionDecl?");
4699
4700	if (FD->isTargetVersionMultiVersion() || FD->isTargetClonesMultiVersion()) {
4701	std::string MangledName =
4702	getMangledNameImpl(CGM&: *this, GD, ND: FD, /*OmitMultiVersionMangling=*/true);
4703	if (!DeferredResolversToEmit.insert(key: MangledName).second)
4704	return;
4705	}
4706	MultiVersionFuncs.push_back(x: GD);
4707	}
4708
4709	/// If a dispatcher for the specified mangled name is not in the module, create
4710	/// and return it. The dispatcher is either an llvm Function with the specified
4711	/// type, or a global ifunc.
4712	llvm::Constant *CodeGenModule::GetOrCreateMultiVersionResolver(GlobalDecl GD) {
4713	const auto *FD = cast<FunctionDecl>(Val: GD.getDecl());
4714	assert(FD && "Not a FunctionDecl?");
4715
4716	std::string MangledName =
4717	getMangledNameImpl(CGM&: *this, GD, ND: FD, /*OmitMultiVersionMangling=*/true);
4718
4719	// Holds the name of the resolver, in ifunc mode this is the ifunc (which has
4720	// a separate resolver).
4721	std::string ResolverName = MangledName;
4722	if (getTarget().supportsIFunc()) {
4723	switch (FD->getMultiVersionKind()) {
4724	case MultiVersionKind::None:
4725	llvm_unreachable("unexpected MultiVersionKind::None for resolver");
4726	case MultiVersionKind::Target:
4727	case MultiVersionKind::CPUSpecific:
4728	case MultiVersionKind::CPUDispatch:
4729	ResolverName += ".ifunc";
4730	break;
4731	case MultiVersionKind::TargetClones:
4732	case MultiVersionKind::TargetVersion:
4733	break;
4734	}
4735	} else if (FD->isTargetMultiVersion()) {
4736	ResolverName += ".resolver";
4737	}
4738
4739	bool ShouldReturnIFunc =
4740	getTarget().supportsIFunc() && !FD->isCPUSpecificMultiVersion();
4741
4742	// If the resolver has already been created, just return it. This lookup may
4743	// yield a function declaration instead of a resolver on AArch64. That is
4744	// because we didn't know whether a resolver will be generated when we first
4745	// encountered a use of the symbol named after this resolver. Therefore,
4746	// targets which support ifuncs should not return here unless we actually
4747	// found an ifunc.
4748	llvm::GlobalValue *ResolverGV = GetGlobalValue(Name: ResolverName);
4749	if (ResolverGV && (isa<llvm::GlobalIFunc>(Val: ResolverGV) || !ShouldReturnIFunc))
4750	return ResolverGV;
4751
4752	const CGFunctionInfo &FI = getTypes().arrangeGlobalDeclaration(GD);
4753	llvm::FunctionType *DeclTy = getTypes().GetFunctionType(Info: FI);
4754
4755	// The resolver needs to be created. For target and target_clones, defer
4756	// creation until the end of the TU.
4757	if (FD->isTargetMultiVersion() || FD->isTargetClonesMultiVersion())
4758	AddDeferredMultiVersionResolverToEmit(GD);
4759
4760	// For cpu_specific, don't create an ifunc yet because we don't know if the
4761	// cpu_dispatch will be emitted in this translation unit.
4762	if (ShouldReturnIFunc) {
4763	unsigned AS = getTypes().getTargetAddressSpace(T: FD->getType());
4764	llvm::Type *ResolverType = llvm::FunctionType::get(
4765	Result: llvm::PointerType::get(C&: getLLVMContext(), AddressSpace: AS), isVarArg: false);
4766	llvm::Constant *Resolver = GetOrCreateLLVMFunction(
4767	MangledName: MangledName + ".resolver", Ty: ResolverType, D: GlobalDecl{},
4768	/*ForVTable=*/false);
4769	llvm::GlobalIFunc *GIF =
4770	llvm::GlobalIFunc::create(Ty: DeclTy, AddressSpace: AS, Linkage: getMultiversionLinkage(CGM&: *this, GD),
4771	Name: "", Resolver, Parent: &getModule());
4772	GIF->setName(ResolverName);
4773	SetCommonAttributes(GD: FD, GV: GIF);
4774	if (ResolverGV)
4775	replaceDeclarationWith(Old: ResolverGV, New: GIF);
4776	return GIF;
4777	}
4778
4779	llvm::Constant *Resolver = GetOrCreateLLVMFunction(
4780	MangledName: ResolverName, Ty: DeclTy, D: GlobalDecl{}, /*ForVTable=*/false);
4781	assert(isa<llvm::GlobalValue>(Resolver) && !ResolverGV &&
4782	"Resolver should be created for the first time");
4783	SetCommonAttributes(GD: FD, GV: cast<llvm::GlobalValue>(Val: Resolver));
4784	return Resolver;
4785	}
4786
4787	bool CodeGenModule::shouldDropDLLAttribute(const Decl *D,
4788	const llvm::GlobalValue *GV) const {
4789	auto SC = GV->getDLLStorageClass();
4790	if (SC == llvm::GlobalValue::DefaultStorageClass)
4791	return false;
4792	const Decl *MRD = D->getMostRecentDecl();
4793	return (((SC == llvm::GlobalValue::DLLImportStorageClass &&
4794	!MRD->hasAttr<DLLImportAttr>()) ||
4795	(SC == llvm::GlobalValue::DLLExportStorageClass &&
4796	!MRD->hasAttr<DLLExportAttr>())) &&
4797	!shouldMapVisibilityToDLLExport(D: cast<NamedDecl>(Val: MRD)));
4798	}
4799
4800	/// GetOrCreateLLVMFunction - If the specified mangled name is not in the
4801	/// module, create and return an llvm Function with the specified type. If there
4802	/// is something in the module with the specified name, return it potentially
4803	/// bitcasted to the right type.
4804	///
4805	/// If D is non-null, it specifies a decl that correspond to this. This is used
4806	/// to set the attributes on the function when it is first created.
4807	llvm::Constant *CodeGenModule::GetOrCreateLLVMFunction(
4808	StringRef MangledName, llvm::Type *Ty, GlobalDecl GD, bool ForVTable,
4809	bool DontDefer, bool IsThunk, llvm::AttributeList ExtraAttrs,
4810	ForDefinition_t IsForDefinition) {
4811	const Decl *D = GD.getDecl();
4812
4813	std::string NameWithoutMultiVersionMangling;
4814	if (const FunctionDecl *FD = cast_or_null<FunctionDecl>(Val: D)) {
4815	// For the device mark the function as one that should be emitted.
4816	if (getLangOpts().OpenMPIsTargetDevice && OpenMPRuntime &&
4817	!OpenMPRuntime->markAsGlobalTarget(GD) && FD->isDefined() &&
4818	!DontDefer && !IsForDefinition) {
4819	if (const FunctionDecl *FDDef = FD->getDefinition()) {
4820	GlobalDecl GDDef;
4821	if (const auto *CD = dyn_cast<CXXConstructorDecl>(Val: FDDef))
4822	GDDef = GlobalDecl(CD, GD.getCtorType());
4823	else if (const auto *DD = dyn_cast<CXXDestructorDecl>(Val: FDDef))
4824	GDDef = GlobalDecl(DD, GD.getDtorType());
4825	else
4826	GDDef = GlobalDecl(FDDef);
4827	EmitGlobal(GD: GDDef);
4828	}
4829	}
4830
4831	// Any attempts to use a MultiVersion function should result in retrieving
4832	// the iFunc instead. Name Mangling will handle the rest of the changes.
4833	if (FD->isMultiVersion()) {
4834	UpdateMultiVersionNames(GD, FD, CurName&: MangledName);
4835	if (!IsForDefinition) {
4836	// On AArch64 we do not immediatelly emit an ifunc resolver when a
4837	// function is used. Instead we defer the emission until we see a
4838	// default definition. In the meantime we just reference the symbol
4839	// without FMV mangling (it may or may not be replaced later).
4840	if (getTarget().getTriple().isAArch64()) {
4841	AddDeferredMultiVersionResolverToEmit(GD);
4842	NameWithoutMultiVersionMangling = getMangledNameImpl(
4843	CGM&: *this, GD, ND: FD, /*OmitMultiVersionMangling=*/true);
4844	} else
4845	return GetOrCreateMultiVersionResolver(GD);
4846	}
4847	}
4848	}
4849
4850	if (!NameWithoutMultiVersionMangling.empty())
4851	MangledName = NameWithoutMultiVersionMangling;
4852
4853	// Lookup the entry, lazily creating it if necessary.
4854	llvm::GlobalValue *Entry = GetGlobalValue(Name: MangledName);
4855	if (Entry) {
4856	if (WeakRefReferences.erase(Ptr: Entry)) {
4857	const FunctionDecl *FD = cast_or_null<FunctionDecl>(Val: D);
4858	if (FD && !FD->hasAttr<WeakAttr>())
4859	Entry->setLinkage(llvm::Function::ExternalLinkage);
4860	}
4861
4862	// Handle dropped DLL attributes.
4863	if (D && shouldDropDLLAttribute(D, GV: Entry)) {
4864	Entry->setDLLStorageClass(llvm::GlobalValue::DefaultStorageClass);
4865	setDSOLocal(Entry);
4866	}
4867
4868	// If there are two attempts to define the same mangled name, issue an
4869	// error.
4870	if (IsForDefinition && !Entry->isDeclaration()) {
4871	GlobalDecl OtherGD;
4872	// Check that GD is not yet in DiagnosedConflictingDefinitions is required
4873	// to make sure that we issue an error only once.
4874	if (lookupRepresentativeDecl(MangledName, Result&: OtherGD) &&
4875	(GD.getCanonicalDecl().getDecl() !=
4876	OtherGD.getCanonicalDecl().getDecl()) &&
4877	DiagnosedConflictingDefinitions.insert(V: GD).second) {
4878	getDiags().Report(Loc: D->getLocation(), DiagID: diag::err_duplicate_mangled_name)
4879	<< MangledName;
4880	getDiags().Report(Loc: OtherGD.getDecl()->getLocation(),
4881	DiagID: diag::note_previous_definition);
4882	}
4883	}
4884
4885	if ((isa<llvm::Function>(Val: Entry) || isa<llvm::GlobalAlias>(Val: Entry)) &&
4886	(Entry->getValueType() == Ty)) {
4887	return Entry;
4888	}
4889
4890	// Make sure the result is of the correct type.
4891	// (If function is requested for a definition, we always need to create a new
4892	// function, not just return a bitcast.)
4893	if (!IsForDefinition)
4894	return Entry;
4895	}
4896
4897	// This function doesn't have a complete type (for example, the return
4898	// type is an incomplete struct). Use a fake type instead, and make
4899	// sure not to try to set attributes.
4900	bool IsIncompleteFunction = false;
4901
4902	llvm::FunctionType *FTy;
4903	if (isa<llvm::FunctionType>(Val: Ty)) {
4904	FTy = cast<llvm::FunctionType>(Val: Ty);
4905	} else {
4906	FTy = llvm::FunctionType::get(Result: VoidTy, isVarArg: false);
4907	IsIncompleteFunction = true;
4908	}
4909
4910	llvm::Function *F =
4911	llvm::Function::Create(Ty: FTy, Linkage: llvm::Function::ExternalLinkage,
4912	N: Entry ? StringRef() : MangledName, M: &getModule());
4913
4914	// Store the declaration associated with this function so it is potentially
4915	// updated by further declarations or definitions and emitted at the end.
4916	if (D && D->hasAttr<AnnotateAttr>())
4917	DeferredAnnotations[MangledName] = cast<ValueDecl>(Val: D);
4918
4919	// If we already created a function with the same mangled name (but different
4920	// type) before, take its name and add it to the list of functions to be
4921	// replaced with F at the end of CodeGen.
4922	//
4923	// This happens if there is a prototype for a function (e.g. "int f()") and
4924	// then a definition of a different type (e.g. "int f(int x)").
4925	if (Entry) {
4926	F->takeName(V: Entry);
4927
4928	// This might be an implementation of a function without a prototype, in
4929	// which case, try to do special replacement of calls which match the new
4930	// prototype. The really key thing here is that we also potentially drop
4931	// arguments from the call site so as to make a direct call, which makes the
4932	// inliner happier and suppresses a number of optimizer warnings (!) about
4933	// dropping arguments.
4934	if (!Entry->use_empty()) {
4935	ReplaceUsesOfNonProtoTypeWithRealFunction(Old: Entry, NewFn: F);
4936	Entry->removeDeadConstantUsers();
4937	}
4938
4939	addGlobalValReplacement(GV: Entry, C: F);
4940	}
4941
4942	assert(F->getName() == MangledName && "name was uniqued!");
4943	if (D)
4944	SetFunctionAttributes(GD, F, IsIncompleteFunction, IsThunk);
4945	if (ExtraAttrs.hasFnAttrs()) {
4946	llvm::AttrBuilder B(F->getContext(), ExtraAttrs.getFnAttrs());
4947	F->addFnAttrs(Attrs: B);
4948	}
4949
4950	if (!DontDefer) {
4951	// All MSVC dtors other than the base dtor are linkonce_odr and delegate to
4952	// each other bottoming out with the base dtor. Therefore we emit non-base
4953	// dtors on usage, even if there is no dtor definition in the TU.
4954	if (isa_and_nonnull<CXXDestructorDecl>(Val: D) &&
4955	getCXXABI().useThunkForDtorVariant(Dtor: cast<CXXDestructorDecl>(Val: D),
4956	DT: GD.getDtorType()))
4957	addDeferredDeclToEmit(GD);
4958
4959	// This is the first use or definition of a mangled name. If there is a
4960	// deferred decl with this name, remember that we need to emit it at the end
4961	// of the file.
4962	auto DDI = DeferredDecls.find(Val: MangledName);
4963	if (DDI != DeferredDecls.end()) {
4964	// Move the potentially referenced deferred decl to the
4965	// DeferredDeclsToEmit list, and remove it from DeferredDecls (since we
4966	// don't need it anymore).
4967	addDeferredDeclToEmit(GD: DDI->second);
4968	DeferredDecls.erase(I: DDI);
4969
4970	// Otherwise, there are cases we have to worry about where we're
4971	// using a declaration for which we must emit a definition but where
4972	// we might not find a top-level definition:
4973	// - member functions defined inline in their classes
4974	// - friend functions defined inline in some class
4975	// - special member functions with implicit definitions
4976	// If we ever change our AST traversal to walk into class methods,
4977	// this will be unnecessary.
4978	//
4979	// We also don't emit a definition for a function if it's going to be an
4980	// entry in a vtable, unless it's already marked as used.
4981	} else if (getLangOpts().CPlusPlus && D) {
4982	// Look for a declaration that's lexically in a record.
4983	for (const auto *FD = cast<FunctionDecl>(Val: D)->getMostRecentDecl(); FD;
4984	FD = FD->getPreviousDecl()) {
4985	if (isa<CXXRecordDecl>(Val: FD->getLexicalDeclContext())) {
4986	if (FD->doesThisDeclarationHaveABody()) {
4987	addDeferredDeclToEmit(GD: GD.getWithDecl(D: FD));
4988	break;
4989	}
4990	}
4991	}
4992	}
4993	}
4994
4995	// Make sure the result is of the requested type.
4996	if (!IsIncompleteFunction) {
4997	assert(F->getFunctionType() == Ty);
4998	return F;
4999	}
5000
5001	return F;
5002	}
5003
5004	/// GetAddrOfFunction - Return the address of the given function. If Ty is
5005	/// non-null, then this function will use the specified type if it has to
5006	/// create it (this occurs when we see a definition of the function).
5007	llvm::Constant *
5008	CodeGenModule::GetAddrOfFunction(GlobalDecl GD, llvm::Type *Ty, bool ForVTable,
5009	bool DontDefer,
5010	ForDefinition_t IsForDefinition) {
5011	// If there was no specific requested type, just convert it now.
5012	if (!Ty) {
5013	const auto *FD = cast<FunctionDecl>(Val: GD.getDecl());
5014	Ty = getTypes().ConvertType(T: FD->getType());
5015	if (DeviceKernelAttr::isOpenCLSpelling(A: FD->getAttr<DeviceKernelAttr>()) &&
5016	GD.getKernelReferenceKind() == KernelReferenceKind::Stub) {
5017	const CGFunctionInfo &FI = getTypes().arrangeGlobalDeclaration(GD);
5018	Ty = getTypes().GetFunctionType(Info: FI);
5019	}
5020	}
5021
5022	// Devirtualized destructor calls may come through here instead of via
5023	// getAddrOfCXXStructor. Make sure we use the MS ABI base destructor instead
5024	// of the complete destructor when necessary.
5025	if (const auto *DD = dyn_cast<CXXDestructorDecl>(Val: GD.getDecl())) {
5026	if (getTarget().getCXXABI().isMicrosoft() &&
5027	GD.getDtorType() == Dtor_Complete &&
5028	DD->getParent()->getNumVBases() == 0)
5029	GD = GlobalDecl(DD, Dtor_Base);
5030	}
5031
5032	StringRef MangledName = getMangledName(GD);
5033	auto *F = GetOrCreateLLVMFunction(MangledName, Ty, GD, ForVTable, DontDefer,
5034	/*IsThunk=*/false, ExtraAttrs: llvm::AttributeList(),
5035	IsForDefinition);
5036	// Returns kernel handle for HIP kernel stub function.
5037	if (LangOpts.CUDA && !LangOpts.CUDAIsDevice &&
5038	cast<FunctionDecl>(Val: GD.getDecl())->hasAttr<CUDAGlobalAttr>()) {
5039	auto *Handle = getCUDARuntime().getKernelHandle(
5040	Stub: cast<llvm::Function>(Val: F->stripPointerCasts()), GD);
5041	if (IsForDefinition)
5042	return F;
5043	return Handle;
5044	}
5045	return F;
5046	}
5047
5048	llvm::Constant *CodeGenModule::GetFunctionStart(const ValueDecl *Decl) {
5049	llvm::GlobalValue *F =
5050	cast<llvm::GlobalValue>(Val: GetAddrOfFunction(GD: Decl)->stripPointerCasts());
5051
5052	return llvm::NoCFIValue::get(GV: F);
5053	}
5054
5055	static const FunctionDecl *
5056	GetRuntimeFunctionDecl(ASTContext &C, StringRef Name) {
5057	TranslationUnitDecl *TUDecl = C.getTranslationUnitDecl();
5058	DeclContext *DC = TranslationUnitDecl::castToDeclContext(D: TUDecl);
5059
5060	IdentifierInfo &CII = C.Idents.get(Name);
5061	for (const auto *Result : DC->lookup(Name: &CII))
5062	if (const auto *FD = dyn_cast<FunctionDecl>(Val: Result))
5063	return FD;
5064
5065	if (!C.getLangOpts().CPlusPlus)
5066	return nullptr;
5067
5068	// Demangle the premangled name from getTerminateFn()
5069	IdentifierInfo &CXXII =
5070	(Name == "_ZSt9terminatev" || Name == "?terminate@@YAXXZ")
5071	? C.Idents.get(Name: "terminate")
5072	: C.Idents.get(Name);
5073
5074	for (const auto &N : {"__cxxabiv1", "std"}) {
5075	IdentifierInfo &NS = C.Idents.get(Name: N);
5076	for (const auto *Result : DC->lookup(Name: &NS)) {
5077	const NamespaceDecl *ND = dyn_cast<NamespaceDecl>(Val: Result);
5078	if (auto *LSD = dyn_cast<LinkageSpecDecl>(Val: Result))
5079	for (const auto *Result : LSD->lookup(Name: &NS))
5080	if ((ND = dyn_cast<NamespaceDecl>(Val: Result)))
5081	break;
5082
5083	if (ND)
5084	for (const auto *Result : ND->lookup(Name: &CXXII))
5085	if (const auto *FD = dyn_cast<FunctionDecl>(Val: Result))
5086	return FD;
5087	}
5088	}
5089
5090	return nullptr;
5091	}
5092
5093	static void setWindowsItaniumDLLImport(CodeGenModule &CGM, bool Local,
5094	llvm::Function *F, StringRef Name) {
5095	// In Windows Itanium environments, try to mark runtime functions
5096	// dllimport. For Mingw and MSVC, don't. We don't really know if the user
5097	// will link their standard library statically or dynamically. Marking
5098	// functions imported when they are not imported can cause linker errors
5099	// and warnings.
5100	if (!Local && CGM.getTriple().isWindowsItaniumEnvironment() &&
5101	!CGM.getCodeGenOpts().LTOVisibilityPublicStd) {
5102	const FunctionDecl *FD = GetRuntimeFunctionDecl(C&: CGM.getContext(), Name);
5103	if (!FD || FD->hasAttr<DLLImportAttr>()) {
5104	F->setDLLStorageClass(llvm::GlobalValue::DLLImportStorageClass);
5105	F->setLinkage(llvm::GlobalValue::ExternalLinkage);
5106	}
5107	}
5108	}
5109
5110	llvm::FunctionCallee CodeGenModule::CreateRuntimeFunction(
5111	QualType ReturnTy, ArrayRef<QualType> ArgTys, StringRef Name,
5112	llvm::AttributeList ExtraAttrs, bool Local, bool AssumeConvergent) {
5113	if (AssumeConvergent) {
5114	ExtraAttrs =
5115	ExtraAttrs.addFnAttribute(C&: VMContext, Kind: llvm::Attribute::Convergent);
5116	}
5117
5118	QualType FTy = Context.getFunctionType(ResultTy: ReturnTy, Args: ArgTys,
5119	EPI: FunctionProtoType::ExtProtoInfo());
5120	const CGFunctionInfo &Info = getTypes().arrangeFreeFunctionType(
5121	Ty: Context.getCanonicalType(T: FTy).castAs<FunctionProtoType>());
5122	auto *ConvTy = getTypes().GetFunctionType(Info);
5123	llvm::Constant *C = GetOrCreateLLVMFunction(
5124	MangledName: Name, Ty: ConvTy, GD: GlobalDecl(), /*ForVTable=*/false,
5125	/*DontDefer=*/false, /*IsThunk=*/false, ExtraAttrs);
5126
5127	if (auto *F = dyn_cast<llvm::Function>(Val: C)) {
5128	if (F->empty()) {
5129	SetLLVMFunctionAttributes(GD: GlobalDecl(), Info, F, /*IsThunk*/ false);
5130	// FIXME: Set calling-conv properly in ExtProtoInfo
5131	F->setCallingConv(getRuntimeCC());
5132	setWindowsItaniumDLLImport(CGM&: *this, Local, F, Name);
5133	setDSOLocal(F);
5134	}
5135	}
5136	return {ConvTy, C};
5137	}
5138
5139	/// CreateRuntimeFunction - Create a new runtime function with the specified
5140	/// type and name.
5141	llvm::FunctionCallee
5142	CodeGenModule::CreateRuntimeFunction(llvm::FunctionType *FTy, StringRef Name,
5143	llvm::AttributeList ExtraAttrs, bool Local,
5144	bool AssumeConvergent) {
5145	if (AssumeConvergent) {
5146	ExtraAttrs =
5147	ExtraAttrs.addFnAttribute(C&: VMContext, Kind: llvm::Attribute::Convergent);
5148	}
5149
5150	llvm::Constant *C =
5151	GetOrCreateLLVMFunction(MangledName: Name, Ty: FTy, GD: GlobalDecl(), /*ForVTable=*/false,
5152	/*DontDefer=*/false, /*IsThunk=*/false,
5153	ExtraAttrs);
5154
5155	if (auto *F = dyn_cast<llvm::Function>(Val: C)) {
5156	if (F->empty()) {
5157	F->setCallingConv(getRuntimeCC());
5158	setWindowsItaniumDLLImport(CGM&: *this, Local, F, Name);
5159	setDSOLocal(F);
5160	// FIXME: We should use CodeGenModule::SetLLVMFunctionAttributes() instead
5161	// of trying to approximate the attributes using the LLVM function
5162	// signature. The other overload of CreateRuntimeFunction does this; it
5163	// should be used for new code.
5164	markRegisterParameterAttributes(F);
5165	}
5166	}
5167
5168	return {FTy, C};
5169	}
5170
5171	/// GetOrCreateLLVMGlobal - If the specified mangled name is not in the module,
5172	/// create and return an llvm GlobalVariable with the specified type and address
5173	/// space. If there is something in the module with the specified name, return
5174	/// it potentially bitcasted to the right type.
5175	///
5176	/// If D is non-null, it specifies a decl that correspond to this. This is used
5177	/// to set the attributes on the global when it is first created.
5178	///
5179	/// If IsForDefinition is true, it is guaranteed that an actual global with
5180	/// type Ty will be returned, not conversion of a variable with the same
5181	/// mangled name but some other type.
5182	llvm::Constant *
5183	CodeGenModule::GetOrCreateLLVMGlobal(StringRef MangledName, llvm::Type *Ty,
5184	LangAS AddrSpace, const VarDecl *D,
5185	ForDefinition_t IsForDefinition) {
5186	// Lookup the entry, lazily creating it if necessary.
5187	llvm::GlobalValue *Entry = GetGlobalValue(Name: MangledName);
5188	unsigned TargetAS = getContext().getTargetAddressSpace(AS: AddrSpace);
5189	if (Entry) {
5190	if (WeakRefReferences.erase(Ptr: Entry)) {
5191	if (D && !D->hasAttr<WeakAttr>())
5192	Entry->setLinkage(llvm::Function::ExternalLinkage);
5193	}
5194
5195	// Handle dropped DLL attributes.
5196	if (D && shouldDropDLLAttribute(D, GV: Entry))
5197	Entry->setDLLStorageClass(llvm::GlobalValue::DefaultStorageClass);
5198
5199	if (LangOpts.OpenMP && !LangOpts.OpenMPSimd && D)
5200	getOpenMPRuntime().registerTargetGlobalVariable(VD: D, Addr: Entry);
5201
5202	if (Entry->getValueType() == Ty && Entry->getAddressSpace() == TargetAS)
5203	return Entry;
5204
5205	// If there are two attempts to define the same mangled name, issue an
5206	// error.
5207	if (IsForDefinition && !Entry->isDeclaration()) {
5208	GlobalDecl OtherGD;
5209	const VarDecl *OtherD;
5210
5211	// Check that D is not yet in DiagnosedConflictingDefinitions is required
5212	// to make sure that we issue an error only once.
5213	if (D && lookupRepresentativeDecl(MangledName, Result&: OtherGD) &&
5214	(D->getCanonicalDecl() != OtherGD.getCanonicalDecl().getDecl()) &&
5215	(OtherD = dyn_cast<VarDecl>(Val: OtherGD.getDecl())) &&
5216	OtherD->hasInit() &&
5217	DiagnosedConflictingDefinitions.insert(V: D).second) {
5218	getDiags().Report(Loc: D->getLocation(), DiagID: diag::err_duplicate_mangled_name)
5219	<< MangledName;
5220	getDiags().Report(Loc: OtherGD.getDecl()->getLocation(),
5221	DiagID: diag::note_previous_definition);
5222	}
5223	}
5224
5225	// Make sure the result is of the correct type.
5226	if (Entry->getType()->getAddressSpace() != TargetAS)
5227	return llvm::ConstantExpr::getAddrSpaceCast(
5228	C: Entry, Ty: llvm::PointerType::get(C&: Ty->getContext(), AddressSpace: TargetAS));
5229
5230	// (If global is requested for a definition, we always need to create a new
5231	// global, not just return a bitcast.)
5232	if (!IsForDefinition)
5233	return Entry;
5234	}
5235
5236	auto DAddrSpace = GetGlobalVarAddressSpace(D);
5237
5238	auto *GV = new llvm::GlobalVariable(
5239	getModule(), Ty, false, llvm::GlobalValue::ExternalLinkage, nullptr,
5240	MangledName, nullptr, llvm::GlobalVariable::NotThreadLocal,
5241	getContext().getTargetAddressSpace(AS: DAddrSpace));
5242
5243	// If we already created a global with the same mangled name (but different
5244	// type) before, take its name and remove it from its parent.
5245	if (Entry) {
5246	GV->takeName(V: Entry);
5247
5248	if (!Entry->use_empty()) {
5249	Entry->replaceAllUsesWith(V: GV);
5250	}
5251
5252	Entry->eraseFromParent();
5253	}
5254
5255	// This is the first use or definition of a mangled name. If there is a
5256	// deferred decl with this name, remember that we need to emit it at the end
5257	// of the file.
5258	auto DDI = DeferredDecls.find(Val: MangledName);
5259	if (DDI != DeferredDecls.end()) {
5260	// Move the potentially referenced deferred decl to the DeferredDeclsToEmit
5261	// list, and remove it from DeferredDecls (since we don't need it anymore).
5262	addDeferredDeclToEmit(GD: DDI->second);
5263	DeferredDecls.erase(I: DDI);
5264	}
5265
5266	// Handle things which are present even on external declarations.
5267	if (D) {
5268	if (LangOpts.OpenMP && !LangOpts.OpenMPSimd)
5269	getOpenMPRuntime().registerTargetGlobalVariable(VD: D, Addr: GV);
5270
5271	// FIXME: This code is overly simple and should be merged with other global
5272	// handling.
5273	GV->setConstant(D->getType().isConstantStorage(Ctx: getContext(), ExcludeCtor: false, ExcludeDtor: false));
5274
5275	GV->setAlignment(getContext().getDeclAlign(D).getAsAlign());
5276
5277	setLinkageForGV(GV, ND: D);
5278
5279	if (D->getTLSKind()) {
5280	if (D->getTLSKind() == VarDecl::TLS_Dynamic)
5281	CXXThreadLocals.push_back(x: D);
5282	setTLSMode(GV, D: *D);
5283	}
5284
5285	setGVProperties(GV, D);
5286
5287	// If required by the ABI, treat declarations of static data members with
5288	// inline initializers as definitions.
5289	if (getContext().isMSStaticDataMemberInlineDefinition(VD: D)) {
5290	EmitGlobalVarDefinition(D);
5291	}
5292
5293	// Emit section information for extern variables.
5294	if (D->hasExternalStorage()) {
5295	if (const SectionAttr *SA = D->getAttr<SectionAttr>())
5296	GV->setSection(SA->getName());
5297	}
5298
5299	// Handle XCore specific ABI requirements.
5300	if (getTriple().getArch() == llvm::Triple::xcore &&
5301	D->getLanguageLinkage() == CLanguageLinkage &&
5302	D->getType().isConstant(Ctx: Context) &&
5303	isExternallyVisible(L: D->getLinkageAndVisibility().getLinkage()))
5304	GV->setSection(".cp.rodata");
5305
5306	// Handle code model attribute
5307	if (const auto *CMA = D->getAttr<CodeModelAttr>())
5308	GV->setCodeModel(CMA->getModel());
5309
5310	// Check if we a have a const declaration with an initializer, we may be
5311	// able to emit it as available_externally to expose it's value to the
5312	// optimizer.
5313	if (Context.getLangOpts().CPlusPlus && GV->hasExternalLinkage() &&
5314	D->getType().isConstQualified() && !GV->hasInitializer() &&
5315	!D->hasDefinition() && D->hasInit() && !D->hasAttr<DLLImportAttr>()) {
5316	const auto *Record =
5317	Context.getBaseElementType(QT: D->getType())->getAsCXXRecordDecl();
5318	bool HasMutableFields = Record && Record->hasMutableFields();
5319	if (!HasMutableFields) {
5320	const VarDecl *InitDecl;
5321	const Expr *InitExpr = D->getAnyInitializer(D&: InitDecl);
5322	if (InitExpr) {
5323	ConstantEmitter emitter(*this);
5324	llvm::Constant *Init = emitter.tryEmitForInitializer(D: *InitDecl);
5325	if (Init) {
5326	auto *InitType = Init->getType();
5327	if (GV->getValueType() != InitType) {
5328	// The type of the initializer does not match the definition.
5329	// This happens when an initializer has a different type from
5330	// the type of the global (because of padding at the end of a
5331	// structure for instance).
5332	GV->setName(StringRef());
5333	// Make a new global with the correct type, this is now guaranteed
5334	// to work.
5335	auto *NewGV = cast<llvm::GlobalVariable>(
5336	Val: GetAddrOfGlobalVar(D, Ty: InitType, IsForDefinition)
5337	->stripPointerCasts());
5338
5339	// Erase the old global, since it is no longer used.
5340	GV->eraseFromParent();
5341	GV = NewGV;
5342	} else {
5343	GV->setInitializer(Init);
5344	GV->setConstant(true);
5345	GV->setLinkage(llvm::GlobalValue::AvailableExternallyLinkage);
5346	}
5347	emitter.finalize(global: GV);
5348	}
5349	}
5350	}
5351	}
5352	}
5353
5354	if (D &&
5355	D->isThisDeclarationADefinition(Context) == VarDecl::DeclarationOnly) {
5356	getTargetCodeGenInfo().setTargetAttributes(D, GV, M&: *this);
5357	// External HIP managed variables needed to be recorded for transformation
5358	// in both device and host compilations.
5359	if (getLangOpts().CUDA && D && D->hasAttr<HIPManagedAttr>() &&
5360	D->hasExternalStorage())
5361	getCUDARuntime().handleVarRegistration(VD: D, Var&: *GV);
5362	}
5363
5364	if (D)
5365	SanitizerMD->reportGlobal(GV, D: *D);
5366
5367	LangAS ExpectedAS =
5368	D ? D->getType().getAddressSpace()
5369	: (LangOpts.OpenCL ? LangAS::opencl_global : LangAS::Default);
5370	assert(getContext().getTargetAddressSpace(ExpectedAS) == TargetAS);
5371	if (DAddrSpace != ExpectedAS) {
5372	return getTargetCodeGenInfo().performAddrSpaceCast(
5373	CGM&: *this, V: GV, SrcAddr: DAddrSpace,
5374	DestTy: llvm::PointerType::get(C&: getLLVMContext(), AddressSpace: TargetAS));
5375	}
5376
5377	return GV;
5378	}
5379
5380	llvm::Constant *
5381	CodeGenModule::GetAddrOfGlobal(GlobalDecl GD, ForDefinition_t IsForDefinition) {
5382	const Decl *D = GD.getDecl();
5383
5384	if (isa<CXXConstructorDecl>(Val: D) || isa<CXXDestructorDecl>(Val: D))
5385	return getAddrOfCXXStructor(GD, /*FnInfo=*/nullptr, /*FnType=*/nullptr,
5386	/*DontDefer=*/false, IsForDefinition);
5387
5388	if (isa<CXXMethodDecl>(Val: D)) {
5389	auto FInfo =
5390	&getTypes().arrangeCXXMethodDeclaration(MD: cast<CXXMethodDecl>(Val: D));
5391	auto Ty = getTypes().GetFunctionType(Info: *FInfo);
5392	return GetAddrOfFunction(GD, Ty, /*ForVTable=*/false, /*DontDefer=*/false,
5393	IsForDefinition);
5394	}
5395
5396	if (isa<FunctionDecl>(Val: D)) {
5397	const CGFunctionInfo &FI = getTypes().arrangeGlobalDeclaration(GD);
5398	llvm::FunctionType *Ty = getTypes().GetFunctionType(Info: FI);
5399	return GetAddrOfFunction(GD, Ty, /*ForVTable=*/false, /*DontDefer=*/false,
5400	IsForDefinition);
5401	}
5402
5403	return GetAddrOfGlobalVar(D: cast<VarDecl>(Val: D), /*Ty=*/nullptr, IsForDefinition);
5404	}
5405
5406	llvm::GlobalVariable *CodeGenModule::CreateOrReplaceCXXRuntimeVariable(
5407	StringRef Name, llvm::Type *Ty, llvm::GlobalValue::LinkageTypes Linkage,
5408	llvm::Align Alignment) {
5409	llvm::GlobalVariable *GV = getModule().getNamedGlobal(Name);
5410	llvm::GlobalVariable *OldGV = nullptr;
5411
5412	if (GV) {
5413	// Check if the variable has the right type.
5414	if (GV->getValueType() == Ty)
5415	return GV;
5416
5417	// Because C++ name mangling, the only way we can end up with an already
5418	// existing global with the same name is if it has been declared extern "C".
5419	assert(GV->isDeclaration() && "Declaration has wrong type!");
5420	OldGV = GV;
5421	}
5422
5423	// Create a new variable.
5424	GV = new llvm::GlobalVariable(getModule(), Ty, /*isConstant=*/true,
5425	Linkage, nullptr, Name);
5426
5427	if (OldGV) {
5428	// Replace occurrences of the old variable if needed.
5429	GV->takeName(V: OldGV);
5430
5431	if (!OldGV->use_empty()) {
5432	OldGV->replaceAllUsesWith(V: GV);
5433	}
5434
5435	OldGV->eraseFromParent();
5436	}
5437
5438	if (supportsCOMDAT() && GV->isWeakForLinker() &&
5439	!GV->hasAvailableExternallyLinkage())
5440	GV->setComdat(TheModule.getOrInsertComdat(Name: GV->getName()));
5441
5442	GV->setAlignment(Alignment);
5443
5444	return GV;
5445	}
5446
5447	/// GetAddrOfGlobalVar - Return the llvm::Constant for the address of the
5448	/// given global variable. If Ty is non-null and if the global doesn't exist,
5449	/// then it will be created with the specified type instead of whatever the
5450	/// normal requested type would be. If IsForDefinition is true, it is guaranteed
5451	/// that an actual global with type Ty will be returned, not conversion of a
5452	/// variable with the same mangled name but some other type.
5453	llvm::Constant *CodeGenModule::GetAddrOfGlobalVar(const VarDecl *D,
5454	llvm::Type *Ty,
5455	ForDefinition_t IsForDefinition) {
5456	assert(D->hasGlobalStorage() && "Not a global variable");
5457	QualType ASTTy = D->getType();
5458	if (!Ty)
5459	Ty = getTypes().ConvertTypeForMem(T: ASTTy);
5460
5461	StringRef MangledName = getMangledName(GD: D);
5462	return GetOrCreateLLVMGlobal(MangledName, Ty, AddrSpace: ASTTy.getAddressSpace(), D,
5463	IsForDefinition);
5464	}
5465
5466	/// CreateRuntimeVariable - Create a new runtime global variable with the
5467	/// specified type and name.
5468	llvm::Constant *
5469	CodeGenModule::CreateRuntimeVariable(llvm::Type *Ty,
5470	StringRef Name) {
5471	LangAS AddrSpace = getContext().getLangOpts().OpenCL ? LangAS::opencl_global
5472	: LangAS::Default;
5473	auto *Ret = GetOrCreateLLVMGlobal(MangledName: Name, Ty, AddrSpace, D: nullptr);
5474	setDSOLocal(cast<llvm::GlobalValue>(Val: Ret->stripPointerCasts()));
5475	return Ret;
5476	}
5477
5478	void CodeGenModule::EmitTentativeDefinition(const VarDecl *D) {
5479	assert(!D->getInit() && "Cannot emit definite definitions here!");
5480
5481	StringRef MangledName = getMangledName(GD: D);
5482	llvm::GlobalValue *GV = GetGlobalValue(Name: MangledName);
5483
5484	// We already have a definition, not declaration, with the same mangled name.
5485	// Emitting of declaration is not required (and actually overwrites emitted
5486	// definition).
5487	if (GV && !GV->isDeclaration())
5488	return;
5489
5490	// If we have not seen a reference to this variable yet, place it into the
5491	// deferred declarations table to be emitted if needed later.
5492	if (!MustBeEmitted(Global: D) && !GV) {
5493	DeferredDecls[MangledName] = D;
5494	return;
5495	}
5496
5497	// The tentative definition is the only definition.
5498	EmitGlobalVarDefinition(D);
5499	}
5500
5501	// Return a GlobalDecl. Use the base variants for destructors and constructors.
5502	static GlobalDecl getBaseVariantGlobalDecl(const NamedDecl *D) {
5503	if (auto const *CD = dyn_cast<const CXXConstructorDecl>(Val: D))
5504	return GlobalDecl(CD, CXXCtorType::Ctor_Base);
5505	else if (auto const *DD = dyn_cast<const CXXDestructorDecl>(Val: D))
5506	return GlobalDecl(DD, CXXDtorType::Dtor_Base);
5507	return GlobalDecl(D);
5508	}
5509
5510	void CodeGenModule::EmitExternalDeclaration(const DeclaratorDecl *D) {
5511	CGDebugInfo *DI = getModuleDebugInfo();
5512	if (!DI || !getCodeGenOpts().hasReducedDebugInfo())
5513	return;
5514
5515	GlobalDecl GD = getBaseVariantGlobalDecl(D);
5516	if (!GD)
5517	return;
5518
5519	llvm::Constant *Addr = GetAddrOfGlobal(GD)->stripPointerCasts();
5520	if (const auto *VD = dyn_cast<VarDecl>(Val: D)) {
5521	DI->EmitExternalVariable(
5522	GV: cast<llvm::GlobalVariable>(Val: Addr->stripPointerCasts()), Decl: VD);
5523	} else if (const auto *FD = dyn_cast<FunctionDecl>(Val: D)) {
5524	llvm::Function *Fn = cast<llvm::Function>(Val: Addr);
5525	if (!Fn->getSubprogram())
5526	DI->EmitFunctionDecl(GD, Loc: FD->getLocation(), FnType: FD->getType(), Fn);
5527	}
5528	}
5529
5530	CharUnits CodeGenModule::GetTargetTypeStoreSize(llvm::Type *Ty) const {
5531	return Context.toCharUnitsFromBits(
5532	BitSize: getDataLayout().getTypeStoreSizeInBits(Ty));
5533	}
5534
5535	LangAS CodeGenModule::GetGlobalVarAddressSpace(const VarDecl *D) {
5536	if (LangOpts.OpenCL) {
5537	LangAS AS = D ? D->getType().getAddressSpace() : LangAS::opencl_global;
5538	assert(AS == LangAS::opencl_global ||
5539	AS == LangAS::opencl_global_device ||
5540	AS == LangAS::opencl_global_host ||
5541	AS == LangAS::opencl_constant ||
5542	AS == LangAS::opencl_local ||
5543	AS >= LangAS::FirstTargetAddressSpace);
5544	return AS;
5545	}
5546
5547	if (LangOpts.SYCLIsDevice &&
5548	(!D || D->getType().getAddressSpace() == LangAS::Default))
5549	return LangAS::sycl_global;
5550
5551	if (LangOpts.CUDA && LangOpts.CUDAIsDevice) {
5552	if (D) {
5553	if (D->hasAttr<CUDAConstantAttr>())
5554	return LangAS::cuda_constant;
5555	if (D->hasAttr<CUDASharedAttr>())
5556	return LangAS::cuda_shared;
5557	if (D->hasAttr<CUDADeviceAttr>())
5558	return LangAS::cuda_device;
5559	if (D->getType().isConstQualified())
5560	return LangAS::cuda_constant;
5561	}
5562	return LangAS::cuda_device;
5563	}
5564
5565	if (LangOpts.OpenMP) {
5566	LangAS AS;
5567	if (OpenMPRuntime->hasAllocateAttributeForGlobalVar(VD: D, AS))
5568	return AS;
5569	}
5570	return getTargetCodeGenInfo().getGlobalVarAddressSpace(CGM&: *this, D);
5571	}
5572
5573	LangAS CodeGenModule::GetGlobalConstantAddressSpace() const {
5574	// OpenCL v1.2 s6.5.3: a string literal is in the constant address space.
5575	if (LangOpts.OpenCL)
5576	return LangAS::opencl_constant;
5577	if (LangOpts.SYCLIsDevice)
5578	return LangAS::sycl_global;
5579	if (LangOpts.HIP && LangOpts.CUDAIsDevice && getTriple().isSPIRV())
5580	// For HIPSPV map literals to cuda_device (maps to CrossWorkGroup in SPIR-V)
5581	// instead of default AS (maps to Generic in SPIR-V). Otherwise, we end up
5582	// with OpVariable instructions with Generic storage class which is not
5583	// allowed (SPIR-V V1.6 s3.42.8). Also, mapping literals to SPIR-V
5584	// UniformConstant storage class is not viable as pointers to it may not be
5585	// casted to Generic pointers which are used to model HIP's "flat" pointers.
5586	return LangAS::cuda_device;
5587	if (auto AS = getTarget().getConstantAddressSpace())
5588	return *AS;
5589	return LangAS::Default;
5590	}
5591
5592	// In address space agnostic languages, string literals are in default address
5593	// space in AST. However, certain targets (e.g. amdgcn) request them to be
5594	// emitted in constant address space in LLVM IR. To be consistent with other
5595	// parts of AST, string literal global variables in constant address space
5596	// need to be casted to default address space before being put into address
5597	// map and referenced by other part of CodeGen.
5598	// In OpenCL, string literals are in constant address space in AST, therefore
5599	// they should not be casted to default address space.
5600	static llvm::Constant *
5601	castStringLiteralToDefaultAddressSpace(CodeGenModule &CGM,
5602	llvm::GlobalVariable *GV) {
5603	llvm::Constant *Cast = GV;
5604	if (!CGM.getLangOpts().OpenCL) {
5605	auto AS = CGM.GetGlobalConstantAddressSpace();
5606	if (AS != LangAS::Default)
5607	Cast = CGM.getTargetCodeGenInfo().performAddrSpaceCast(
5608	CGM, V: GV, SrcAddr: AS,
5609	DestTy: llvm::PointerType::get(
5610	C&: CGM.getLLVMContext(),
5611	AddressSpace: CGM.getContext().getTargetAddressSpace(AS: LangAS::Default)));
5612	}
5613	return Cast;
5614	}
5615
5616	template<typename SomeDecl>
5617	void CodeGenModule::MaybeHandleStaticInExternC(const SomeDecl *D,
5618	llvm::GlobalValue *GV) {
5619	if (!getLangOpts().CPlusPlus)
5620	return;
5621
5622	// Must have 'used' attribute, or else inline assembly can't rely on
5623	// the name existing.
5624	if (!D->template hasAttr<UsedAttr>())
5625	return;
5626
5627	// Must have internal linkage and an ordinary name.
5628	if (!D->getIdentifier() || D->getFormalLinkage() != Linkage::Internal)
5629	return;
5630
5631	// Must be in an extern "C" context. Entities declared directly within
5632	// a record are not extern "C" even if the record is in such a context.
5633	const SomeDecl *First = D->getFirstDecl();
5634	if (First->getDeclContext()->isRecord() || !First->isInExternCContext())
5635	return;
5636
5637	// OK, this is an internal linkage entity inside an extern "C" linkage
5638	// specification. Make a note of that so we can give it the "expected"
5639	// mangled name if nothing else is using that name.
5640	std::pair<StaticExternCMap::iterator, bool> R =
5641	StaticExternCValues.insert(std::make_pair(D->getIdentifier(), GV));
5642
5643	// If we have multiple internal linkage entities with the same name
5644	// in extern "C" regions, none of them gets that name.
5645	if (!R.second)
5646	R.first->second = nullptr;
5647	}
5648
5649	static bool shouldBeInCOMDAT(CodeGenModule &CGM, const Decl &D) {
5650	if (!CGM.supportsCOMDAT())
5651	return false;
5652
5653	if (D.hasAttr<SelectAnyAttr>())
5654	return true;
5655
5656	GVALinkage Linkage;
5657	if (auto *VD = dyn_cast<VarDecl>(Val: &D))
5658	Linkage = CGM.getContext().GetGVALinkageForVariable(VD);
5659	else
5660	Linkage = CGM.getContext().GetGVALinkageForFunction(FD: cast<FunctionDecl>(Val: &D));
5661
5662	switch (Linkage) {
5663	case GVA_Internal:
5664	case GVA_AvailableExternally:
5665	case GVA_StrongExternal:
5666	return false;
5667	case GVA_DiscardableODR:
5668	case GVA_StrongODR:
5669	return true;
5670	}
5671	llvm_unreachable("No such linkage");
5672	}
5673
5674	bool CodeGenModule::supportsCOMDAT() const {
5675	return getTriple().supportsCOMDAT();
5676	}
5677
5678	void CodeGenModule::maybeSetTrivialComdat(const Decl &D,
5679	llvm::GlobalObject &GO) {
5680	if (!shouldBeInCOMDAT(CGM&: *this, D))
5681	return;
5682	GO.setComdat(TheModule.getOrInsertComdat(Name: GO.getName()));
5683	}
5684
5685	const ABIInfo &CodeGenModule::getABIInfo() {
5686	return getTargetCodeGenInfo().getABIInfo();
5687	}
5688
5689	/// Pass IsTentative as true if you want to create a tentative definition.
5690	void CodeGenModule::EmitGlobalVarDefinition(const VarDecl *D,
5691	bool IsTentative) {
5692	// OpenCL global variables of sampler type are translated to function calls,
5693	// therefore no need to be translated.
5694	QualType ASTTy = D->getType();
5695	if (getLangOpts().OpenCL && ASTTy->isSamplerT())
5696	return;
5697
5698	// HLSL default buffer constants will be emitted during HLSLBufferDecl codegen
5699	if (getLangOpts().HLSL &&
5700	D->getType().getAddressSpace() == LangAS::hlsl_constant)
5701	return;
5702
5703	// If this is OpenMP device, check if it is legal to emit this global
5704	// normally.
5705	if (LangOpts.OpenMPIsTargetDevice && OpenMPRuntime &&
5706	OpenMPRuntime->emitTargetGlobalVariable(GD: D))
5707	return;
5708
5709	llvm::TrackingVH<llvm::Constant> Init;
5710	bool NeedsGlobalCtor = false;
5711	// Whether the definition of the variable is available externally.
5712	// If yes, we shouldn't emit the GloablCtor and GlobalDtor for the variable
5713	// since this is the job for its original source.
5714	bool IsDefinitionAvailableExternally =
5715	getContext().GetGVALinkageForVariable(VD: D) == GVA_AvailableExternally;
5716	bool NeedsGlobalDtor =
5717	!IsDefinitionAvailableExternally &&
5718	D->needsDestruction(Ctx: getContext()) == QualType::DK_cxx_destructor;
5719
5720	// It is helpless to emit the definition for an available_externally variable
5721	// which can't be marked as const.
5722	// We don't need to check if it needs global ctor or dtor. See the above
5723	// comment for ideas.
5724	if (IsDefinitionAvailableExternally &&
5725	(!D->hasConstantInitialization() ||
5726	// TODO: Update this when we have interface to check constexpr
5727	// destructor.
5728	D->needsDestruction(Ctx: getContext()) ||
5729	!D->getType().isConstantStorage(Ctx: getContext(), ExcludeCtor: true, ExcludeDtor: true)))
5730	return;
5731
5732	const VarDecl *InitDecl;
5733	const Expr *InitExpr = D->getAnyInitializer(D&: InitDecl);
5734
5735	std::optional<ConstantEmitter> emitter;
5736
5737	// CUDA E.2.4.1 "__shared__ variables cannot have an initialization
5738	// as part of their declaration." Sema has already checked for
5739	// error cases, so we just need to set Init to UndefValue.
5740	bool IsCUDASharedVar =
5741	getLangOpts().CUDAIsDevice && D->hasAttr<CUDASharedAttr>();
5742	// Shadows of initialized device-side global variables are also left
5743	// undefined.
5744	// Managed Variables should be initialized on both host side and device side.
5745	bool IsCUDAShadowVar =
5746	!getLangOpts().CUDAIsDevice && !D->hasAttr<HIPManagedAttr>() &&
5747	(D->hasAttr<CUDAConstantAttr>() || D->hasAttr<CUDADeviceAttr>() ||
5748	D->hasAttr<CUDASharedAttr>());
5749	bool IsCUDADeviceShadowVar =
5750	getLangOpts().CUDAIsDevice && !D->hasAttr<HIPManagedAttr>() &&
5751	(D->getType()->isCUDADeviceBuiltinSurfaceType() ||
5752	D->getType()->isCUDADeviceBuiltinTextureType());
5753	if (getLangOpts().CUDA &&
5754	(IsCUDASharedVar || IsCUDAShadowVar || IsCUDADeviceShadowVar))
5755	Init = llvm::UndefValue::get(T: getTypes().ConvertTypeForMem(T: ASTTy));
5756	else if (D->hasAttr<LoaderUninitializedAttr>())
5757	Init = llvm::UndefValue::get(T: getTypes().ConvertTypeForMem(T: ASTTy));
5758	else if (!InitExpr) {
5759	// This is a tentative definition; tentative definitions are
5760	// implicitly initialized with { 0 }.
5761	//
5762	// Note that tentative definitions are only emitted at the end of
5763	// a translation unit, so they should never have incomplete
5764	// type. In addition, EmitTentativeDefinition makes sure that we
5765	// never attempt to emit a tentative definition if a real one
5766	// exists. A use may still exists, however, so we still may need
5767	// to do a RAUW.
5768	assert(!ASTTy->isIncompleteType() && "Unexpected incomplete type");
5769	Init = EmitNullConstant(T: D->getType());
5770	} else {
5771	initializedGlobalDecl = GlobalDecl(D);
5772	emitter.emplace(args&: *this);
5773	llvm::Constant *Initializer = emitter->tryEmitForInitializer(D: *InitDecl);
5774	if (!Initializer) {
5775	QualType T = InitExpr->getType();
5776	if (D->getType()->isReferenceType())
5777	T = D->getType();
5778
5779	if (getLangOpts().HLSL &&
5780	D->getType().getTypePtr()->isHLSLResourceRecord()) {
5781	Init = llvm::PoisonValue::get(T: getTypes().ConvertType(T: ASTTy));
5782	NeedsGlobalCtor = true;
5783	} else if (getLangOpts().CPlusPlus) {
5784	Init = EmitNullConstant(T);
5785	if (!IsDefinitionAvailableExternally)
5786	NeedsGlobalCtor = true;
5787	if (InitDecl->hasFlexibleArrayInit(Ctx: getContext())) {
5788	ErrorUnsupported(D, Type: "flexible array initializer");
5789	// We cannot create ctor for flexible array initializer
5790	NeedsGlobalCtor = false;
5791	}
5792	} else {
5793	ErrorUnsupported(D, Type: "static initializer");
5794	Init = llvm::PoisonValue::get(T: getTypes().ConvertType(T));
5795	}
5796	} else {
5797	Init = Initializer;
5798	// We don't need an initializer, so remove the entry for the delayed
5799	// initializer position (just in case this entry was delayed) if we
5800	// also don't need to register a destructor.
5801	if (getLangOpts().CPlusPlus && !NeedsGlobalDtor)
5802	DelayedCXXInitPosition.erase(Val: D);
5803
5804	#ifndef NDEBUG
5805	CharUnits VarSize = getContext().getTypeSizeInChars(ASTTy) +
5806	InitDecl->getFlexibleArrayInitChars(getContext());
5807	CharUnits CstSize = CharUnits::fromQuantity(
5808	getDataLayout().getTypeAllocSize(Init->getType()));
5809	assert(VarSize == CstSize && "Emitted constant has unexpected size");
5810	#endif
5811	}
5812	}
5813
5814	llvm::Type* InitType = Init->getType();
5815	llvm::Constant *Entry =
5816	GetAddrOfGlobalVar(D, Ty: InitType, IsForDefinition: ForDefinition_t(!IsTentative));
5817
5818	// Strip off pointer casts if we got them.
5819	Entry = Entry->stripPointerCasts();
5820
5821	// Entry is now either a Function or GlobalVariable.
5822	auto *GV = dyn_cast<llvm::GlobalVariable>(Val: Entry);
5823
5824	// We have a definition after a declaration with the wrong type.
5825	// We must make a new GlobalVariable* and update everything that used OldGV
5826	// (a declaration or tentative definition) with the new GlobalVariable*
5827	// (which will be a definition).
5828	//
5829	// This happens if there is a prototype for a global (e.g.
5830	// "extern int x[];") and then a definition of a different type (e.g.
5831	// "int x[10];"). This also happens when an initializer has a different type
5832	// from the type of the global (this happens with unions).
5833	if (!GV || GV->getValueType() != InitType ||
5834	GV->getType()->getAddressSpace() !=
5835	getContext().getTargetAddressSpace(AS: GetGlobalVarAddressSpace(D))) {
5836
5837	// Move the old entry aside so that we'll create a new one.
5838	Entry->setName(StringRef());
5839
5840	// Make a new global with the correct type, this is now guaranteed to work.
5841	GV = cast<llvm::GlobalVariable>(
5842	Val: GetAddrOfGlobalVar(D, Ty: InitType, IsForDefinition: ForDefinition_t(!IsTentative))
5843	->stripPointerCasts());
5844
5845	// Replace all uses of the old global with the new global
5846	llvm::Constant *NewPtrForOldDecl =
5847	llvm::ConstantExpr::getPointerBitCastOrAddrSpaceCast(C: GV,
5848	Ty: Entry->getType());
5849	Entry->replaceAllUsesWith(V: NewPtrForOldDecl);
5850
5851	// Erase the old global, since it is no longer used.
5852	cast<llvm::GlobalValue>(Val: Entry)->eraseFromParent();
5853	}
5854
5855	MaybeHandleStaticInExternC(D, GV);
5856
5857	if (D->hasAttr<AnnotateAttr>())
5858	AddGlobalAnnotations(D, GV);
5859
5860	// Set the llvm linkage type as appropriate.
5861	llvm::GlobalValue::LinkageTypes Linkage = getLLVMLinkageVarDefinition(VD: D);
5862
5863	// CUDA B.2.1 "The __device__ qualifier declares a variable that resides on
5864	// the device. [...]"
5865	// CUDA B.2.2 "The __constant__ qualifier, optionally used together with
5866	// __device__, declares a variable that: [...]
5867	// Is accessible from all the threads within the grid and from the host
5868	// through the runtime library (cudaGetSymbolAddress() / cudaGetSymbolSize()
5869	// / cudaMemcpyToSymbol() / cudaMemcpyFromSymbol())."
5870	if (LangOpts.CUDA) {
5871	if (LangOpts.CUDAIsDevice) {
5872	if (Linkage != llvm::GlobalValue::InternalLinkage &&
5873	(D->hasAttr<CUDADeviceAttr>() || D->hasAttr<CUDAConstantAttr>() ||
5874	D->getType()->isCUDADeviceBuiltinSurfaceType() ||
5875	D->getType()->isCUDADeviceBuiltinTextureType()))
5876	GV->setExternallyInitialized(true);
5877	} else {
5878	getCUDARuntime().internalizeDeviceSideVar(D, Linkage);
5879	}
5880	getCUDARuntime().handleVarRegistration(VD: D, Var&: *GV);
5881	}
5882
5883	if (LangOpts.HLSL && GetGlobalVarAddressSpace(D) == LangAS::hlsl_input) {
5884	// HLSL Input variables are considered to be set by the driver/pipeline, but
5885	// only visible to a single thread/wave.
5886	GV->setExternallyInitialized(true);
5887	} else {
5888	GV->setInitializer(Init);
5889	}
5890
5891	if (LangOpts.HLSL)
5892	getHLSLRuntime().handleGlobalVarDefinition(VD: D, Var: GV);
5893
5894	if (emitter)
5895	emitter->finalize(global: GV);
5896
5897	// If it is safe to mark the global 'constant', do so now.
5898	GV->setConstant((D->hasAttr<CUDAConstantAttr>() && LangOpts.CUDAIsDevice) ||
5899	(!NeedsGlobalCtor && !NeedsGlobalDtor &&
5900	D->getType().isConstantStorage(Ctx: getContext(), ExcludeCtor: true, ExcludeDtor: true)));
5901
5902	// If it is in a read-only section, mark it 'constant'.
5903	if (const SectionAttr *SA = D->getAttr<SectionAttr>()) {
5904	const ASTContext::SectionInfo &SI = Context.SectionInfos[SA->getName()];
5905	if ((SI.SectionFlags & ASTContext::PSF_Write) == 0)
5906	GV->setConstant(true);
5907	}
5908
5909	CharUnits AlignVal = getContext().getDeclAlign(D);
5910	// Check for alignment specifed in an 'omp allocate' directive.
5911	if (std::optional<CharUnits> AlignValFromAllocate =
5912	getOMPAllocateAlignment(VD: D))
5913	AlignVal = *AlignValFromAllocate;
5914	GV->setAlignment(AlignVal.getAsAlign());
5915
5916	// On Darwin, unlike other Itanium C++ ABI platforms, the thread-wrapper
5917	// function is only defined alongside the variable, not also alongside
5918	// callers. Normally, all accesses to a thread_local go through the
5919	// thread-wrapper in order to ensure initialization has occurred, underlying
5920	// variable will never be used other than the thread-wrapper, so it can be
5921	// converted to internal linkage.
5922	//
5923	// However, if the variable has the 'constinit' attribute, it _can_ be
5924	// referenced directly, without calling the thread-wrapper, so the linkage
5925	// must not be changed.
5926	//
5927	// Additionally, if the variable isn't plain external linkage, e.g. if it's
5928	// weak or linkonce, the de-duplication semantics are important to preserve,
5929	// so we don't change the linkage.
5930	if (D->getTLSKind() == VarDecl::TLS_Dynamic &&
5931	Linkage == llvm::GlobalValue::ExternalLinkage &&
5932	Context.getTargetInfo().getTriple().isOSDarwin() &&
5933	!D->hasAttr<ConstInitAttr>())
5934	Linkage = llvm::GlobalValue::InternalLinkage;
5935
5936	// HLSL variables in the input address space maps like memory-mapped
5937	// variables. Even if they are 'static', they are externally initialized and
5938	// read/write by the hardware/driver/pipeline.
5939	if (LangOpts.HLSL && GetGlobalVarAddressSpace(D) == LangAS::hlsl_input)
5940	Linkage = llvm::GlobalValue::ExternalLinkage;
5941
5942	GV->setLinkage(Linkage);
5943	if (D->hasAttr<DLLImportAttr>())
5944	GV->setDLLStorageClass(llvm::GlobalVariable::DLLImportStorageClass);
5945	else if (D->hasAttr<DLLExportAttr>())
5946	GV->setDLLStorageClass(llvm::GlobalVariable::DLLExportStorageClass);
5947	else
5948	GV->setDLLStorageClass(llvm::GlobalVariable::DefaultStorageClass);
5949
5950	if (Linkage == llvm::GlobalVariable::CommonLinkage) {
5951	// common vars aren't constant even if declared const.
5952	GV->setConstant(false);
5953	// Tentative definition of global variables may be initialized with
5954	// non-zero null pointers. In this case they should have weak linkage
5955	// since common linkage must have zero initializer and must not have
5956	// explicit section therefore cannot have non-zero initial value.
5957	if (!GV->getInitializer()->isNullValue())
5958	GV->setLinkage(llvm::GlobalVariable::WeakAnyLinkage);
5959	}
5960
5961	setNonAliasAttributes(GD: D, GO: GV);
5962
5963	if (D->getTLSKind() && !GV->isThreadLocal()) {
5964	if (D->getTLSKind() == VarDecl::TLS_Dynamic)
5965	CXXThreadLocals.push_back(x: D);
5966	setTLSMode(GV, D: *D);
5967	}
5968
5969	maybeSetTrivialComdat(D: *D, GO&: *GV);
5970
5971	// Emit the initializer function if necessary.
5972	if (NeedsGlobalCtor || NeedsGlobalDtor)
5973	EmitCXXGlobalVarDeclInitFunc(D, Addr: GV, PerformInit: NeedsGlobalCtor);
5974
5975	SanitizerMD->reportGlobal(GV, D: *D, IsDynInit: NeedsGlobalCtor);
5976
5977	// Emit global variable debug information.
5978	if (CGDebugInfo *DI = getModuleDebugInfo())
5979	if (getCodeGenOpts().hasReducedDebugInfo())
5980	DI->EmitGlobalVariable(GV, Decl: D);
5981	}
5982
5983	static bool isVarDeclStrongDefinition(const ASTContext &Context,
5984	CodeGenModule &CGM, const VarDecl *D,
5985	bool NoCommon) {
5986	// Don't give variables common linkage if -fno-common was specified unless it
5987	// was overridden by a NoCommon attribute.
5988	if ((NoCommon || D->hasAttr<NoCommonAttr>()) && !D->hasAttr<CommonAttr>())
5989	return true;
5990
5991	// C11 6.9.2/2:
5992	// A declaration of an identifier for an object that has file scope without
5993	// an initializer, and without a storage-class specifier or with the
5994	// storage-class specifier static, constitutes a tentative definition.
5995	if (D->getInit() || D->hasExternalStorage())
5996	return true;
5997
5998	// A variable cannot be both common and exist in a section.
5999	if (D->hasAttr<SectionAttr>())
6000	return true;
6001
6002	// A variable cannot be both common and exist in a section.
6003	// We don't try to determine which is the right section in the front-end.
6004	// If no specialized section name is applicable, it will resort to default.
6005	if (D->hasAttr<PragmaClangBSSSectionAttr>() ||
6006	D->hasAttr<PragmaClangDataSectionAttr>() ||
6007	D->hasAttr<PragmaClangRelroSectionAttr>() ||
6008	D->hasAttr<PragmaClangRodataSectionAttr>())
6009	return true;
6010
6011	// Thread local vars aren't considered common linkage.
6012	if (D->getTLSKind())
6013	return true;
6014
6015	// Tentative definitions marked with WeakImportAttr are true definitions.
6016	if (D->hasAttr<WeakImportAttr>())
6017	return true;
6018
6019	// A variable cannot be both common and exist in a comdat.
6020	if (shouldBeInCOMDAT(CGM, D: *D))
6021	return true;
6022
6023	// Declarations with a required alignment do not have common linkage in MSVC
6024	// mode.
6025	if (Context.getTargetInfo().getCXXABI().isMicrosoft()) {
6026	if (D->hasAttr<AlignedAttr>())
6027	return true;
6028	QualType VarType = D->getType();
6029	if (Context.isAlignmentRequired(T: VarType))
6030	return true;
6031
6032	if (const auto *RT = VarType->getAs<RecordType>()) {
6033	const RecordDecl *RD = RT->getDecl();
6034	for (const FieldDecl *FD : RD->fields()) {
6035	if (FD->isBitField())
6036	continue;
6037	if (FD->hasAttr<AlignedAttr>())
6038	return true;
6039	if (Context.isAlignmentRequired(T: FD->getType()))
6040	return true;
6041	}
6042	}
6043	}
6044
6045	// Microsoft's link.exe doesn't support alignments greater than 32 bytes for
6046	// common symbols, so symbols with greater alignment requirements cannot be
6047	// common.
6048	// Other COFF linkers (ld.bfd and LLD) support arbitrary power-of-two
6049	// alignments for common symbols via the aligncomm directive, so this
6050	// restriction only applies to MSVC environments.
6051	if (Context.getTargetInfo().getTriple().isKnownWindowsMSVCEnvironment() &&
6052	Context.getTypeAlignIfKnown(T: D->getType()) >
6053	Context.toBits(CharSize: CharUnits::fromQuantity(Quantity: 32)))
6054	return true;
6055
6056	return false;
6057	}
6058
6059	llvm::GlobalValue::LinkageTypes
6060	CodeGenModule::getLLVMLinkageForDeclarator(const DeclaratorDecl *D,
6061	GVALinkage Linkage) {
6062	if (Linkage == GVA_Internal)
6063	return llvm::Function::InternalLinkage;
6064
6065	if (D->hasAttr<WeakAttr>())
6066	return llvm::GlobalVariable::WeakAnyLinkage;
6067
6068	if (const auto *FD = D->getAsFunction())
6069	if (FD->isMultiVersion() && Linkage == GVA_AvailableExternally)
6070	return llvm::GlobalVariable::LinkOnceAnyLinkage;
6071
6072	// We are guaranteed to have a strong definition somewhere else,
6073	// so we can use available_externally linkage.
6074	if (Linkage == GVA_AvailableExternally)
6075	return llvm::GlobalValue::AvailableExternallyLinkage;
6076
6077	// Note that Apple's kernel linker doesn't support symbol
6078	// coalescing, so we need to avoid linkonce and weak linkages there.
6079	// Normally, this means we just map to internal, but for explicit
6080	// instantiations we'll map to external.
6081
6082	// In C++, the compiler has to emit a definition in every translation unit
6083	// that references the function. We should use linkonce_odr because
6084	// a) if all references in this translation unit are optimized away, we
6085	// don't need to codegen it. b) if the function persists, it needs to be
6086	// merged with other definitions. c) C++ has the ODR, so we know the
6087	// definition is dependable.
6088	if (Linkage == GVA_DiscardableODR)
6089	return !Context.getLangOpts().AppleKext ? llvm::Function::LinkOnceODRLinkage
6090	: llvm::Function::InternalLinkage;
6091
6092	// An explicit instantiation of a template has weak linkage, since
6093	// explicit instantiations can occur in multiple translation units
6094	// and must all be equivalent. However, we are not allowed to
6095	// throw away these explicit instantiations.
6096	//
6097	// CUDA/HIP: For -fno-gpu-rdc case, device code is limited to one TU,
6098	// so say that CUDA templates are either external (for kernels) or internal.
6099	// This lets llvm perform aggressive inter-procedural optimizations. For
6100	// -fgpu-rdc case, device function calls across multiple TU's are allowed,
6101	// therefore we need to follow the normal linkage paradigm.
6102	if (Linkage == GVA_StrongODR) {
6103	if (getLangOpts().AppleKext)
6104	return llvm::Function::ExternalLinkage;
6105	if (getLangOpts().CUDA && getLangOpts().CUDAIsDevice &&
6106	!getLangOpts().GPURelocatableDeviceCode)
6107	return D->hasAttr<CUDAGlobalAttr>() ? llvm::Function::ExternalLinkage
6108	: llvm::Function::InternalLinkage;
6109	return llvm::Function::WeakODRLinkage;
6110	}
6111
6112	// C++ doesn't have tentative definitions and thus cannot have common
6113	// linkage.
6114	if (!getLangOpts().CPlusPlus && isa<VarDecl>(Val: D) &&
6115	!isVarDeclStrongDefinition(Context, CGM&: *this, D: cast<VarDecl>(Val: D),
6116	NoCommon: CodeGenOpts.NoCommon))
6117	return llvm::GlobalVariable::CommonLinkage;
6118
6119	// selectany symbols are externally visible, so use weak instead of
6120	// linkonce. MSVC optimizes away references to const selectany globals, so
6121	// all definitions should be the same and ODR linkage should be used.
6122	// http://msdn.microsoft.com/en-us/library/5tkz6s71.aspx
6123	if (D->hasAttr<SelectAnyAttr>())
6124	return llvm::GlobalVariable::WeakODRLinkage;
6125
6126	// Otherwise, we have strong external linkage.
6127	assert(Linkage == GVA_StrongExternal);
6128	return llvm::GlobalVariable::ExternalLinkage;
6129	}
6130
6131	llvm::GlobalValue::LinkageTypes
6132	CodeGenModule::getLLVMLinkageVarDefinition(const VarDecl *VD) {
6133	GVALinkage Linkage = getContext().GetGVALinkageForVariable(VD);
6134	return getLLVMLinkageForDeclarator(D: VD, Linkage);
6135	}
6136
6137	/// Replace the uses of a function that was declared with a non-proto type.
6138	/// We want to silently drop extra arguments from call sites
6139	static void replaceUsesOfNonProtoConstant(llvm::Constant *old,
6140	llvm::Function *newFn) {
6141	// Fast path.
6142	if (old->use_empty())
6143	return;
6144
6145	llvm::Type *newRetTy = newFn->getReturnType();
6146	SmallVector<llvm::Value *, 4> newArgs;
6147
6148	SmallVector<llvm::CallBase *> callSitesToBeRemovedFromParent;
6149
6150	for (llvm::Value::use_iterator ui = old->use_begin(), ue = old->use_end();
6151	ui != ue; ui++) {
6152	llvm::User *user = ui->getUser();
6153
6154	// Recognize and replace uses of bitcasts. Most calls to
6155	// unprototyped functions will use bitcasts.
6156	if (auto *bitcast = dyn_cast<llvm::ConstantExpr>(Val: user)) {
6157	if (bitcast->getOpcode() == llvm::Instruction::BitCast)
6158	replaceUsesOfNonProtoConstant(old: bitcast, newFn);
6159	continue;
6160	}
6161
6162	// Recognize calls to the function.
6163	llvm::CallBase *callSite = dyn_cast<llvm::CallBase>(Val: user);
6164	if (!callSite)
6165	continue;
6166	if (!callSite->isCallee(U: &*ui))
6167	continue;
6168
6169	// If the return types don't match exactly, then we can't
6170	// transform this call unless it's dead.
6171	if (callSite->getType() != newRetTy && !callSite->use_empty())
6172	continue;
6173
6174	// Get the call site's attribute list.
6175	SmallVector<llvm::AttributeSet, 8> newArgAttrs;
6176	llvm::AttributeList oldAttrs = callSite->getAttributes();
6177
6178	// If the function was passed too few arguments, don't transform.
6179	unsigned newNumArgs = newFn->arg_size();
6180	if (callSite->arg_size() < newNumArgs)
6181	continue;
6182
6183	// If extra arguments were passed, we silently drop them.
6184	// If any of the types mismatch, we don't transform.
6185	unsigned argNo = 0;
6186	bool dontTransform = false;
6187	for (llvm::Argument &A : newFn->args()) {
6188	if (callSite->getArgOperand(i: argNo)->getType() != A.getType()) {
6189	dontTransform = true;
6190	break;
6191	}
6192
6193	// Add any parameter attributes.
6194	newArgAttrs.push_back(Elt: oldAttrs.getParamAttrs(ArgNo: argNo));
6195	argNo++;
6196	}
6197	if (dontTransform)
6198	continue;
6199
6200	// Okay, we can transform this. Create the new call instruction and copy
6201	// over the required information.
6202	newArgs.append(in_start: callSite->arg_begin(), in_end: callSite->arg_begin() + argNo);
6203
6204	// Copy over any operand bundles.
6205	SmallVector<llvm::OperandBundleDef, 1> newBundles;
6206	callSite->getOperandBundlesAsDefs(Defs&: newBundles);
6207
6208	llvm::CallBase *newCall;
6209	if (isa<llvm::CallInst>(Val: callSite)) {
6210	newCall = llvm::CallInst::Create(Func: newFn, Args: newArgs, Bundles: newBundles, NameStr: "",
6211	InsertBefore: callSite->getIterator());
6212	} else {
6213	auto *oldInvoke = cast<llvm::InvokeInst>(Val: callSite);
6214	newCall = llvm::InvokeInst::Create(
6215	Func: newFn, IfNormal: oldInvoke->getNormalDest(), IfException: oldInvoke->getUnwindDest(),
6216	Args: newArgs, Bundles: newBundles, NameStr: "", InsertBefore: callSite->getIterator());
6217	}
6218	newArgs.clear(); // for the next iteration
6219
6220	if (!newCall->getType()->isVoidTy())
6221	newCall->takeName(V: callSite);
6222	newCall->setAttributes(
6223	llvm::AttributeList::get(C&: newFn->getContext(), FnAttrs: oldAttrs.getFnAttrs(),
6224	RetAttrs: oldAttrs.getRetAttrs(), ArgAttrs: newArgAttrs));
6225	newCall->setCallingConv(callSite->getCallingConv());
6226
6227	// Finally, remove the old call, replacing any uses with the new one.
6228	if (!callSite->use_empty())
6229	callSite->replaceAllUsesWith(V: newCall);
6230
6231	// Copy debug location attached to CI.
6232	if (callSite->getDebugLoc())
6233	newCall->setDebugLoc(callSite->getDebugLoc());
6234
6235	callSitesToBeRemovedFromParent.push_back(Elt: callSite);
6236	}
6237
6238	for (auto *callSite : callSitesToBeRemovedFromParent) {
6239	callSite->eraseFromParent();
6240	}
6241	}
6242
6243	/// ReplaceUsesOfNonProtoTypeWithRealFunction - This function is called when we
6244	/// implement a function with no prototype, e.g. "int foo() {}". If there are
6245	/// existing call uses of the old function in the module, this adjusts them to
6246	/// call the new function directly.
6247	///
6248	/// This is not just a cleanup: the always_inline pass requires direct calls to
6249	/// functions to be able to inline them. If there is a bitcast in the way, it
6250	/// won't inline them. Instcombine normally deletes these calls, but it isn't
6251	/// run at -O0.
6252	static void ReplaceUsesOfNonProtoTypeWithRealFunction(llvm::GlobalValue *Old,
6253	llvm::Function *NewFn) {
6254	// If we're redefining a global as a function, don't transform it.
6255	if (!isa<llvm::Function>(Val: Old)) return;
6256
6257	replaceUsesOfNonProtoConstant(old: Old, newFn: NewFn);
6258	}
6259
6260	void CodeGenModule::HandleCXXStaticMemberVarInstantiation(VarDecl *VD) {
6261	auto DK = VD->isThisDeclarationADefinition();
6262	if ((DK == VarDecl::Definition && VD->hasAttr<DLLImportAttr>()) ||
6263	(LangOpts.CUDA && !shouldEmitCUDAGlobalVar(Global: VD)))
6264	return;
6265
6266	TemplateSpecializationKind TSK = VD->getTemplateSpecializationKind();
6267	// If we have a definition, this might be a deferred decl. If the
6268	// instantiation is explicit, make sure we emit it at the end.
6269	if (VD->getDefinition() && TSK == TSK_ExplicitInstantiationDefinition)
6270	GetAddrOfGlobalVar(D: VD);
6271
6272	EmitTopLevelDecl(D: VD);
6273	}
6274
6275	void CodeGenModule::EmitGlobalFunctionDefinition(GlobalDecl GD,
6276	llvm::GlobalValue *GV) {
6277	const auto *D = cast<FunctionDecl>(Val: GD.getDecl());
6278
6279	// Compute the function info and LLVM type.
6280	const CGFunctionInfo &FI = getTypes().arrangeGlobalDeclaration(GD);
6281	llvm::FunctionType *Ty = getTypes().GetFunctionType(Info: FI);
6282
6283	// Get or create the prototype for the function.
6284	if (!GV || (GV->getValueType() != Ty))
6285	GV = cast<llvm::GlobalValue>(Val: GetAddrOfFunction(GD, Ty, /*ForVTable=*/false,
6286	/*DontDefer=*/true,
6287	IsForDefinition: ForDefinition));
6288
6289	// Already emitted.
6290	if (!GV->isDeclaration())
6291	return;
6292
6293	// We need to set linkage and visibility on the function before
6294	// generating code for it because various parts of IR generation
6295	// want to propagate this information down (e.g. to local static
6296	// declarations).
6297	auto *Fn = cast<llvm::Function>(Val: GV);
6298	setFunctionLinkage(GD, F: Fn);
6299
6300	// FIXME: this is redundant with part of setFunctionDefinitionAttributes
6301	setGVProperties(GV: Fn, GD);
6302
6303	MaybeHandleStaticInExternC(D, GV: Fn);
6304
6305	maybeSetTrivialComdat(D: *D, GO&: *Fn);
6306
6307	CodeGenFunction(*this).GenerateCode(GD, Fn, FnInfo: FI);
6308
6309	setNonAliasAttributes(GD, GO: Fn);
6310
6311	bool ShouldAddOptNone = !CodeGenOpts.DisableO0ImplyOptNone &&
6312	(CodeGenOpts.OptimizationLevel == 0) &&
6313	!D->hasAttr<MinSizeAttr>();
6314
6315	if (DeviceKernelAttr::isOpenCLSpelling(A: D->getAttr<DeviceKernelAttr>())) {
6316	if (GD.getKernelReferenceKind() == KernelReferenceKind::Stub &&
6317	!D->hasAttr<NoInlineAttr>() &&
6318	!Fn->hasFnAttribute(Kind: llvm::Attribute::NoInline) &&
6319	!D->hasAttr<OptimizeNoneAttr>() &&
6320	!Fn->hasFnAttribute(Kind: llvm::Attribute::OptimizeNone) &&
6321	!ShouldAddOptNone) {
6322	Fn->addFnAttr(Kind: llvm::Attribute::AlwaysInline);
6323	}
6324	}
6325
6326	SetLLVMFunctionAttributesForDefinition(D, F: Fn);
6327
6328	if (const ConstructorAttr *CA = D->getAttr<ConstructorAttr>())
6329	AddGlobalCtor(Ctor: Fn, Priority: CA->getPriority());
6330	if (const DestructorAttr *DA = D->getAttr<DestructorAttr>())
6331	AddGlobalDtor(Dtor: Fn, Priority: DA->getPriority(), IsDtorAttrFunc: true);
6332	if (getLangOpts().OpenMP && D->hasAttr<OMPDeclareTargetDeclAttr>())
6333	getOpenMPRuntime().emitDeclareTargetFunction(FD: D, GV);
6334	}
6335
6336	void CodeGenModule::EmitAliasDefinition(GlobalDecl GD) {
6337	const auto *D = cast<ValueDecl>(Val: GD.getDecl());
6338	const AliasAttr *AA = D->getAttr<AliasAttr>();
6339	assert(AA && "Not an alias?");
6340
6341	StringRef MangledName = getMangledName(GD);
6342
6343	if (AA->getAliasee() == MangledName) {
6344	Diags.Report(Loc: AA->getLocation(), DiagID: diag::err_cyclic_alias) << 0;
6345	return;
6346	}
6347
6348	// If there is a definition in the module, then it wins over the alias.
6349	// This is dubious, but allow it to be safe. Just ignore the alias.
6350	llvm::GlobalValue *Entry = GetGlobalValue(Name: MangledName);
6351	if (Entry && !Entry->isDeclaration())
6352	return;
6353
6354	Aliases.push_back(x: GD);
6355
6356	llvm::Type *DeclTy = getTypes().ConvertTypeForMem(T: D->getType());
6357
6358	// Create a reference to the named value. This ensures that it is emitted
6359	// if a deferred decl.
6360	llvm::Constant *Aliasee;
6361	llvm::GlobalValue::LinkageTypes LT;
6362	if (isa<llvm::FunctionType>(Val: DeclTy)) {
6363	Aliasee = GetOrCreateLLVMFunction(MangledName: AA->getAliasee(), Ty: DeclTy, GD,
6364	/*ForVTable=*/false);
6365	LT = getFunctionLinkage(GD);
6366	} else {
6367	Aliasee = GetOrCreateLLVMGlobal(MangledName: AA->getAliasee(), Ty: DeclTy, AddrSpace: LangAS::Default,
6368	/*D=*/nullptr);
6369	if (const auto *VD = dyn_cast<VarDecl>(Val: GD.getDecl()))
6370	LT = getLLVMLinkageVarDefinition(VD);
6371	else
6372	LT = getFunctionLinkage(GD);
6373	}
6374
6375	// Create the new alias itself, but don't set a name yet.
6376	unsigned AS = Aliasee->getType()->getPointerAddressSpace();
6377	auto *GA =
6378	llvm::GlobalAlias::create(Ty: DeclTy, AddressSpace: AS, Linkage: LT, Name: "", Aliasee, Parent: &getModule());
6379
6380	if (Entry) {
6381	if (GA->getAliasee() == Entry) {
6382	Diags.Report(Loc: AA->getLocation(), DiagID: diag::err_cyclic_alias) << 0;
6383	return;
6384	}
6385
6386	assert(Entry->isDeclaration());
6387
6388	// If there is a declaration in the module, then we had an extern followed
6389	// by the alias, as in:
6390	// extern int test6();
6391	// ...
6392	// int test6() __attribute__((alias("test7")));
6393	//
6394	// Remove it and replace uses of it with the alias.
6395	GA->takeName(V: Entry);
6396
6397	Entry->replaceAllUsesWith(V: GA);
6398	Entry->eraseFromParent();
6399	} else {
6400	GA->setName(MangledName);
6401	}
6402
6403	// Set attributes which are particular to an alias; this is a
6404	// specialization of the attributes which may be set on a global
6405	// variable/function.
6406	if (D->hasAttr<WeakAttr>() || D->hasAttr<WeakRefAttr>() ||
6407	D->isWeakImported()) {
6408	GA->setLinkage(llvm::Function::WeakAnyLinkage);
6409	}
6410
6411	if (const auto *VD = dyn_cast<VarDecl>(Val: D))
6412	if (VD->getTLSKind())
6413	setTLSMode(GV: GA, D: *VD);
6414
6415	SetCommonAttributes(GD, GV: GA);
6416
6417	// Emit global alias debug information.
6418	if (isa<VarDecl>(Val: D))
6419	if (CGDebugInfo *DI = getModuleDebugInfo())
6420	DI->EmitGlobalAlias(GV: cast<llvm::GlobalValue>(Val: GA->getAliasee()->stripPointerCasts()), Decl: GD);
6421	}
6422
6423	void CodeGenModule::emitIFuncDefinition(GlobalDecl GD) {
6424	const auto *D = cast<ValueDecl>(Val: GD.getDecl());
6425	const IFuncAttr *IFA = D->getAttr<IFuncAttr>();
6426	assert(IFA && "Not an ifunc?");
6427
6428	StringRef MangledName = getMangledName(GD);
6429
6430	if (IFA->getResolver() == MangledName) {
6431	Diags.Report(Loc: IFA->getLocation(), DiagID: diag::err_cyclic_alias) << 1;
6432	return;
6433	}
6434
6435	// Report an error if some definition overrides ifunc.
6436	llvm::GlobalValue *Entry = GetGlobalValue(Name: MangledName);
6437	if (Entry && !Entry->isDeclaration()) {
6438	GlobalDecl OtherGD;
6439	if (lookupRepresentativeDecl(MangledName, Result&: OtherGD) &&
6440	DiagnosedConflictingDefinitions.insert(V: GD).second) {
6441	Diags.Report(Loc: D->getLocation(), DiagID: diag::err_duplicate_mangled_name)
6442	<< MangledName;
6443	Diags.Report(Loc: OtherGD.getDecl()->getLocation(),
6444	DiagID: diag::note_previous_definition);
6445	}
6446	return;
6447	}
6448
6449	Aliases.push_back(x: GD);
6450
6451	// The resolver might not be visited yet. Specify a dummy non-function type to
6452	// indicate IsIncompleteFunction. Either the type is ignored (if the resolver
6453	// was emitted) or the whole function will be replaced (if the resolver has
6454	// not been emitted).
6455	llvm::Constant *Resolver =
6456	GetOrCreateLLVMFunction(MangledName: IFA->getResolver(), Ty: VoidTy, GD: {},
6457	/*ForVTable=*/false);
6458	llvm::Type *DeclTy = getTypes().ConvertTypeForMem(T: D->getType());
6459	unsigned AS = getTypes().getTargetAddressSpace(T: D->getType());
6460	llvm::GlobalIFunc *GIF = llvm::GlobalIFunc::create(
6461	Ty: DeclTy, AddressSpace: AS, Linkage: llvm::Function::ExternalLinkage, Name: "", Resolver, Parent: &getModule());
6462	if (Entry) {
6463	if (GIF->getResolver() == Entry) {
6464	Diags.Report(Loc: IFA->getLocation(), DiagID: diag::err_cyclic_alias) << 1;
6465	return;
6466	}
6467	assert(Entry->isDeclaration());
6468
6469	// If there is a declaration in the module, then we had an extern followed
6470	// by the ifunc, as in:
6471	// extern int test();
6472	// ...
6473	// int test() __attribute__((ifunc("resolver")));
6474	//
6475	// Remove it and replace uses of it with the ifunc.
6476	GIF->takeName(V: Entry);
6477
6478	Entry->replaceAllUsesWith(V: GIF);
6479	Entry->eraseFromParent();
6480	} else
6481	GIF->setName(MangledName);
6482	SetCommonAttributes(GD, GV: GIF);
6483	}
6484
6485	llvm::Function *CodeGenModule::getIntrinsic(unsigned IID,
6486	ArrayRef<llvm::Type*> Tys) {
6487	return llvm::Intrinsic::getOrInsertDeclaration(M: &getModule(),
6488	id: (llvm::Intrinsic::ID)IID, Tys);
6489	}
6490
6491	static llvm::StringMapEntry<llvm::GlobalVariable *> &
6492	GetConstantCFStringEntry(llvm::StringMap<llvm::GlobalVariable *> &Map,
6493	const StringLiteral *Literal, bool TargetIsLSB,
6494	bool &IsUTF16, unsigned &StringLength) {
6495	StringRef String = Literal->getString();
6496	unsigned NumBytes = String.size();
6497
6498	// Check for simple case.
6499	if (!Literal->containsNonAsciiOrNull()) {
6500	StringLength = NumBytes;
6501	return *Map.insert(KV: std::make_pair(x&: String, y: nullptr)).first;
6502	}
6503
6504	// Otherwise, convert the UTF8 literals into a string of shorts.
6505	IsUTF16 = true;
6506
6507	SmallVector<llvm::UTF16, 128> ToBuf(NumBytes + 1); // +1 for ending nulls.
6508	const llvm::UTF8 *FromPtr = (const llvm::UTF8 *)String.data();
6509	llvm::UTF16 *ToPtr = &ToBuf[0];
6510
6511	(void)llvm::ConvertUTF8toUTF16(sourceStart: &FromPtr, sourceEnd: FromPtr + NumBytes, targetStart: &ToPtr,
6512	targetEnd: ToPtr + NumBytes, flags: llvm::strictConversion);
6513
6514	// ConvertUTF8toUTF16 returns the length in ToPtr.
6515	StringLength = ToPtr - &ToBuf[0];
6516
6517	// Add an explicit null.
6518	*ToPtr = 0;
6519	return *Map.insert(KV: std::make_pair(
6520	x: StringRef(reinterpret_cast<const char *>(ToBuf.data()),
6521	(StringLength + 1) * 2),
6522	y: nullptr)).first;
6523	}
6524
6525	ConstantAddress
6526	CodeGenModule::GetAddrOfConstantCFString(const StringLiteral *Literal) {
6527	unsigned StringLength = 0;
6528	bool isUTF16 = false;
6529	llvm::StringMapEntry<llvm::GlobalVariable *> &Entry =
6530	GetConstantCFStringEntry(Map&: CFConstantStringMap, Literal,
6531	TargetIsLSB: getDataLayout().isLittleEndian(), IsUTF16&: isUTF16,
6532	StringLength);
6533
6534	if (auto *C = Entry.second)
6535	return ConstantAddress(
6536	C, C->getValueType(), CharUnits::fromQuantity(Quantity: C->getAlignment()));
6537
6538	const ASTContext &Context = getContext();
6539	const llvm::Triple &Triple = getTriple();
6540
6541	const auto CFRuntime = getLangOpts().CFRuntime;
6542	const bool IsSwiftABI =
6543	static_cast<unsigned>(CFRuntime) >=
6544	static_cast<unsigned>(LangOptions::CoreFoundationABI::Swift);
6545	const bool IsSwift4_1 = CFRuntime == LangOptions::CoreFoundationABI::Swift4_1;
6546
6547	// If we don't already have it, get __CFConstantStringClassReference.
6548	if (!CFConstantStringClassRef) {
6549	const char *CFConstantStringClassName = "__CFConstantStringClassReference";
6550	llvm::Type *Ty = getTypes().ConvertType(T: getContext().IntTy);
6551	Ty = llvm::ArrayType::get(ElementType: Ty, NumElements: 0);
6552
6553	switch (CFRuntime) {
6554	default: break;
6555	case LangOptions::CoreFoundationABI::Swift: [[fallthrough]];
6556	case LangOptions::CoreFoundationABI::Swift5_0:
6557	CFConstantStringClassName =
6558	Triple.isOSDarwin() ? "$s15SwiftFoundation19_NSCFConstantStringCN"
6559	: "$s10Foundation19_NSCFConstantStringCN";
6560	Ty = IntPtrTy;
6561	break;
6562	case LangOptions::CoreFoundationABI::Swift4_2:
6563	CFConstantStringClassName =
6564	Triple.isOSDarwin() ? "$S15SwiftFoundation19_NSCFConstantStringCN"
6565	: "$S10Foundation19_NSCFConstantStringCN";
6566	Ty = IntPtrTy;
6567	break;
6568	case LangOptions::CoreFoundationABI::Swift4_1:
6569	CFConstantStringClassName =
6570	Triple.isOSDarwin() ? "__T015SwiftFoundation19_NSCFConstantStringCN"
6571	: "__T010Foundation19_NSCFConstantStringCN";
6572	Ty = IntPtrTy;
6573	break;
6574	}
6575
6576	llvm::Constant *C = CreateRuntimeVariable(Ty, Name: CFConstantStringClassName);
6577
6578	if (Triple.isOSBinFormatELF() || Triple.isOSBinFormatCOFF()) {
6579	llvm::GlobalValue *GV = nullptr;
6580
6581	if ((GV = dyn_cast<llvm::GlobalValue>(Val: C))) {
6582	IdentifierInfo &II = Context.Idents.get(Name: GV->getName());
6583	TranslationUnitDecl *TUDecl = Context.getTranslationUnitDecl();
6584	DeclContext *DC = TranslationUnitDecl::castToDeclContext(D: TUDecl);
6585
6586	const VarDecl *VD = nullptr;
6587	for (const auto *Result : DC->lookup(Name: &II))
6588	if ((VD = dyn_cast<VarDecl>(Val: Result)))
6589	break;
6590
6591	if (Triple.isOSBinFormatELF()) {
6592	if (!VD)
6593	GV->setLinkage(llvm::GlobalValue::ExternalLinkage);
6594	} else {
6595	GV->setLinkage(llvm::GlobalValue::ExternalLinkage);
6596	if (!VD || !VD->hasAttr<DLLExportAttr>())
6597	GV->setDLLStorageClass(llvm::GlobalValue::DLLImportStorageClass);
6598	else
6599	GV->setDLLStorageClass(llvm::GlobalValue::DLLExportStorageClass);
6600	}
6601
6602	setDSOLocal(GV);
6603	}
6604	}
6605
6606	// Decay array -> ptr
6607	CFConstantStringClassRef =
6608	IsSwiftABI ? llvm::ConstantExpr::getPtrToInt(C, Ty) : C;
6609	}
6610
6611	QualType CFTy = Context.getCFConstantStringType();
6612
6613	auto *STy = cast<llvm::StructType>(Val: getTypes().ConvertType(T: CFTy));
6614
6615	ConstantInitBuilder Builder(*this);
6616	auto Fields = Builder.beginStruct(structTy: STy);
6617
6618	// Class pointer.
6619	Fields.addSignedPointer(Pointer: cast<llvm::Constant>(Val&: CFConstantStringClassRef),
6620	Schema: getCodeGenOpts().PointerAuth.ObjCIsaPointers,
6621	CalleeDecl: GlobalDecl(), CalleeType: QualType());
6622
6623	// Flags.
6624	if (IsSwiftABI) {
6625	Fields.addInt(intTy: IntPtrTy, value: IsSwift4_1 ? 0x05 : 0x01);
6626	Fields.addInt(intTy: Int64Ty, value: isUTF16 ? 0x07d0 : 0x07c8);
6627	} else {
6628	Fields.addInt(intTy: IntTy, value: isUTF16 ? 0x07d0 : 0x07C8);
6629	}
6630
6631	// String pointer.
6632	llvm::Constant *C = nullptr;
6633	if (isUTF16) {
6634	auto Arr = llvm::ArrayRef(
6635	reinterpret_cast<uint16_t *>(const_cast<char *>(Entry.first().data())),
6636	Entry.first().size() / 2);
6637	C = llvm::ConstantDataArray::get(Context&: VMContext, Elts: Arr);
6638	} else {
6639	C = llvm::ConstantDataArray::getString(Context&: VMContext, Initializer: Entry.first());
6640	}
6641
6642	// Note: -fwritable-strings doesn't make the backing store strings of
6643	// CFStrings writable.
6644	auto *GV =
6645	new llvm::GlobalVariable(getModule(), C->getType(), /*isConstant=*/true,
6646	llvm::GlobalValue::PrivateLinkage, C, ".str");
6647	GV->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
6648	// Don't enforce the target's minimum global alignment, since the only use
6649	// of the string is via this class initializer.
6650	CharUnits Align = isUTF16 ? Context.getTypeAlignInChars(T: Context.ShortTy)
6651	: Context.getTypeAlignInChars(T: Context.CharTy);
6652	GV->setAlignment(Align.getAsAlign());
6653
6654	// FIXME: We set the section explicitly to avoid a bug in ld64 224.1.
6655	// Without it LLVM can merge the string with a non unnamed_addr one during
6656	// LTO. Doing that changes the section it ends in, which surprises ld64.
6657	if (Triple.isOSBinFormatMachO())
6658	GV->setSection(isUTF16 ? "__TEXT,__ustring"
6659	: "__TEXT,__cstring,cstring_literals");
6660	// Make sure the literal ends up in .rodata to allow for safe ICF and for
6661	// the static linker to adjust permissions to read-only later on.
6662	else if (Triple.isOSBinFormatELF())
6663	GV->setSection(".rodata");
6664
6665	// String.
6666	Fields.add(value: GV);
6667
6668	// String length.
6669	llvm::IntegerType *LengthTy =
6670	llvm::IntegerType::get(C&: getModule().getContext(),
6671	NumBits: Context.getTargetInfo().getLongWidth());
6672	if (IsSwiftABI) {
6673	if (CFRuntime == LangOptions::CoreFoundationABI::Swift4_1 ||
6674	CFRuntime == LangOptions::CoreFoundationABI::Swift4_2)
6675	LengthTy = Int32Ty;
6676	else
6677	LengthTy = IntPtrTy;
6678	}
6679	Fields.addInt(intTy: LengthTy, value: StringLength);
6680
6681	// Swift ABI requires 8-byte alignment to ensure that the _Atomic(uint64_t) is
6682	// properly aligned on 32-bit platforms.
6683	CharUnits Alignment =
6684	IsSwiftABI ? Context.toCharUnitsFromBits(BitSize: 64) : getPointerAlign();
6685
6686	// The struct.
6687	GV = Fields.finishAndCreateGlobal(args: "_unnamed_cfstring_", args&: Alignment,
6688	/*isConstant=*/args: false,
6689	args: llvm::GlobalVariable::PrivateLinkage);
6690	GV->addAttribute(Kind: "objc_arc_inert");
6691	switch (Triple.getObjectFormat()) {
6692	case llvm::Triple::UnknownObjectFormat:
6693	llvm_unreachable("unknown file format");
6694	case llvm::Triple::DXContainer:
6695	case llvm::Triple::GOFF:
6696	case llvm::Triple::SPIRV:
6697	case llvm::Triple::XCOFF:
6698	llvm_unreachable("unimplemented");
6699	case llvm::Triple::COFF:
6700	case llvm::Triple::ELF:
6701	case llvm::Triple::Wasm:
6702	GV->setSection("cfstring");
6703	break;
6704	case llvm::Triple::MachO:
6705	GV->setSection("__DATA,__cfstring");
6706	break;
6707	}
6708	Entry.second = GV;
6709
6710	return ConstantAddress(GV, GV->getValueType(), Alignment);
6711	}
6712
6713	bool CodeGenModule::getExpressionLocationsEnabled() const {
6714	return !CodeGenOpts.EmitCodeView || CodeGenOpts.DebugColumnInfo;
6715	}
6716
6717	QualType CodeGenModule::getObjCFastEnumerationStateType() {
6718	if (ObjCFastEnumerationStateType.isNull()) {
6719	RecordDecl *D = Context.buildImplicitRecord(Name: "__objcFastEnumerationState");
6720	D->startDefinition();
6721
6722	QualType FieldTypes[] = {
6723	Context.UnsignedLongTy, Context.getPointerType(T: Context.getObjCIdType()),
6724	Context.getPointerType(T: Context.UnsignedLongTy),
6725	Context.getConstantArrayType(EltTy: Context.UnsignedLongTy, ArySize: llvm::APInt(32, 5),
6726	SizeExpr: nullptr, ASM: ArraySizeModifier::Normal, IndexTypeQuals: 0)};
6727
6728	for (size_t i = 0; i < 4; ++i) {
6729	FieldDecl *Field = FieldDecl::Create(C: Context,
6730	DC: D,
6731	StartLoc: SourceLocation(),
6732	IdLoc: SourceLocation(), Id: nullptr,
6733	T: FieldTypes[i], /*TInfo=*/nullptr,
6734	/*BitWidth=*/BW: nullptr,
6735	/*Mutable=*/false,
6736	InitStyle: ICIS_NoInit);
6737	Field->setAccess(AS_public);
6738	D->addDecl(D: Field);
6739	}
6740
6741	D->completeDefinition();
6742	ObjCFastEnumerationStateType = Context.getTagDeclType(Decl: D);
6743	}
6744
6745	return ObjCFastEnumerationStateType;
6746	}
6747
6748	llvm::Constant *
6749	CodeGenModule::GetConstantArrayFromStringLiteral(const StringLiteral *E) {
6750	assert(!E->getType()->isPointerType() && "Strings are always arrays");
6751
6752	// Don't emit it as the address of the string, emit the string data itself
6753	// as an inline array.
6754	if (E->getCharByteWidth() == 1) {
6755	SmallString<64> Str(E->getString());
6756
6757	// Resize the string to the right size, which is indicated by its type.
6758	const ConstantArrayType *CAT = Context.getAsConstantArrayType(T: E->getType());
6759	assert(CAT && "String literal not of constant array type!");
6760	Str.resize(N: CAT->getZExtSize());
6761	return llvm::ConstantDataArray::getString(Context&: VMContext, Initializer: Str, AddNull: false);
6762	}
6763
6764	auto *AType = cast<llvm::ArrayType>(Val: getTypes().ConvertType(T: E->getType()));
6765	llvm::Type *ElemTy = AType->getElementType();
6766	unsigned NumElements = AType->getNumElements();
6767
6768	// Wide strings have either 2-byte or 4-byte elements.
6769	if (ElemTy->getPrimitiveSizeInBits() == 16) {
6770	SmallVector<uint16_t, 32> Elements;
6771	Elements.reserve(N: NumElements);
6772
6773	for(unsigned i = 0, e = E->getLength(); i != e; ++i)
6774	Elements.push_back(Elt: E->getCodeUnit(i));
6775	Elements.resize(N: NumElements);
6776	return llvm::ConstantDataArray::get(Context&: VMContext, Elts&: Elements);
6777	}
6778
6779	assert(ElemTy->getPrimitiveSizeInBits() == 32);
6780	SmallVector<uint32_t, 32> Elements;
6781	Elements.reserve(N: NumElements);
6782
6783	for(unsigned i = 0, e = E->getLength(); i != e; ++i)
6784	Elements.push_back(Elt: E->getCodeUnit(i));
6785	Elements.resize(N: NumElements);
6786	return llvm::ConstantDataArray::get(Context&: VMContext, Elts&: Elements);
6787	}
6788
6789	static llvm::GlobalVariable *
6790	GenerateStringLiteral(llvm::Constant *C, llvm::GlobalValue::LinkageTypes LT,
6791	CodeGenModule &CGM, StringRef GlobalName,
6792	CharUnits Alignment) {
6793	unsigned AddrSpace = CGM.getContext().getTargetAddressSpace(
6794	AS: CGM.GetGlobalConstantAddressSpace());
6795
6796	llvm::Module &M = CGM.getModule();
6797	// Create a global variable for this string
6798	auto *GV = new llvm::GlobalVariable(
6799	M, C->getType(), !CGM.getLangOpts().WritableStrings, LT, C, GlobalName,
6800	nullptr, llvm::GlobalVariable::NotThreadLocal, AddrSpace);
6801	GV->setAlignment(Alignment.getAsAlign());
6802	GV->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
6803	if (GV->isWeakForLinker()) {
6804	assert(CGM.supportsCOMDAT() && "Only COFF uses weak string literals");
6805	GV->setComdat(M.getOrInsertComdat(Name: GV->getName()));
6806	}
6807	CGM.setDSOLocal(GV);
6808
6809	return GV;
6810	}
6811
6812	/// GetAddrOfConstantStringFromLiteral - Return a pointer to a
6813	/// constant array for the given string literal.
6814	ConstantAddress
6815	CodeGenModule::GetAddrOfConstantStringFromLiteral(const StringLiteral *S,
6816	StringRef Name) {
6817	CharUnits Alignment =
6818	getContext().getAlignOfGlobalVarInChars(T: S->getType(), /*VD=*/nullptr);
6819
6820	llvm::Constant *C = GetConstantArrayFromStringLiteral(E: S);
6821	llvm::GlobalVariable **Entry = nullptr;
6822	if (!LangOpts.WritableStrings) {
6823	Entry = &ConstantStringMap[C];
6824	if (auto GV = *Entry) {
6825	if (uint64_t(Alignment.getQuantity()) > GV->getAlignment())
6826	GV->setAlignment(Alignment.getAsAlign());
6827	return ConstantAddress(castStringLiteralToDefaultAddressSpace(CGM&: *this, GV),
6828	GV->getValueType(), Alignment);
6829	}
6830	}
6831
6832	SmallString<256> MangledNameBuffer;
6833	StringRef GlobalVariableName;
6834	llvm::GlobalValue::LinkageTypes LT;
6835
6836	// Mangle the string literal if that's how the ABI merges duplicate strings.
6837	// Don't do it if they are writable, since we don't want writes in one TU to
6838	// affect strings in another.
6839	if (getCXXABI().getMangleContext().shouldMangleStringLiteral(SL: S) &&
6840	!LangOpts.WritableStrings) {
6841	llvm::raw_svector_ostream Out(MangledNameBuffer);
6842	getCXXABI().getMangleContext().mangleStringLiteral(SL: S, Out);
6843	LT = llvm::GlobalValue::LinkOnceODRLinkage;
6844	GlobalVariableName = MangledNameBuffer;
6845	} else {
6846	LT = llvm::GlobalValue::PrivateLinkage;
6847	GlobalVariableName = Name;
6848	}
6849
6850	auto GV = GenerateStringLiteral(C, LT, CGM&: *this, GlobalName: GlobalVariableName, Alignment);
6851
6852	CGDebugInfo *DI = getModuleDebugInfo();
6853	if (DI && getCodeGenOpts().hasReducedDebugInfo())
6854	DI->AddStringLiteralDebugInfo(GV, S);
6855
6856	if (Entry)
6857	*Entry = GV;
6858
6859	SanitizerMD->reportGlobal(GV, Loc: S->getStrTokenLoc(TokNum: 0), Name: "<string literal>");
6860
6861	return ConstantAddress(castStringLiteralToDefaultAddressSpace(CGM&: *this, GV),
6862	GV->getValueType(), Alignment);
6863	}
6864
6865	/// GetAddrOfConstantStringFromObjCEncode - Return a pointer to a constant
6866	/// array for the given ObjCEncodeExpr node.
6867	ConstantAddress
6868	CodeGenModule::GetAddrOfConstantStringFromObjCEncode(const ObjCEncodeExpr *E) {
6869	std::string Str;
6870	getContext().getObjCEncodingForType(T: E->getEncodedType(), S&: Str);
6871
6872	return GetAddrOfConstantCString(Str);
6873	}
6874
6875	/// GetAddrOfConstantCString - Returns a pointer to a character array containing
6876	/// the literal and a terminating '\0' character.
6877	/// The result has pointer to array type.
6878	ConstantAddress CodeGenModule::GetAddrOfConstantCString(
6879	const std::string &Str, const char *GlobalName) {
6880	StringRef StrWithNull(Str.c_str(), Str.size() + 1);
6881	CharUnits Alignment = getContext().getAlignOfGlobalVarInChars(
6882	T: getContext().CharTy, /*VD=*/nullptr);
6883
6884	llvm::Constant *C =
6885	llvm::ConstantDataArray::getString(Context&: getLLVMContext(), Initializer: StrWithNull, AddNull: false);
6886
6887	// Don't share any string literals if strings aren't constant.
6888	llvm::GlobalVariable **Entry = nullptr;
6889	if (!LangOpts.WritableStrings) {
6890	Entry = &ConstantStringMap[C];
6891	if (auto GV = *Entry) {
6892	if (uint64_t(Alignment.getQuantity()) > GV->getAlignment())
6893	GV->setAlignment(Alignment.getAsAlign());
6894	return ConstantAddress(castStringLiteralToDefaultAddressSpace(CGM&: *this, GV),
6895	GV->getValueType(), Alignment);
6896	}
6897	}
6898
6899	// Get the default prefix if a name wasn't specified.
6900	if (!GlobalName)
6901	GlobalName = ".str";
6902	// Create a global variable for this.
6903	auto GV = GenerateStringLiteral(C, LT: llvm::GlobalValue::PrivateLinkage, CGM&: *this,
6904	GlobalName, Alignment);
6905	if (Entry)
6906	*Entry = GV;
6907
6908	return ConstantAddress(castStringLiteralToDefaultAddressSpace(CGM&: *this, GV),
6909	GV->getValueType(), Alignment);
6910	}
6911
6912	ConstantAddress CodeGenModule::GetAddrOfGlobalTemporary(
6913	const MaterializeTemporaryExpr *E, const Expr *Init) {
6914	assert((E->getStorageDuration() == SD_Static ||
6915	E->getStorageDuration() == SD_Thread) && "not a global temporary");
6916	const auto *VD = cast<VarDecl>(Val: E->getExtendingDecl());
6917
6918	// If we're not materializing a subobject of the temporary, keep the
6919	// cv-qualifiers from the type of the MaterializeTemporaryExpr.
6920	QualType MaterializedType = Init->getType();
6921	if (Init == E->getSubExpr())
6922	MaterializedType = E->getType();
6923
6924	CharUnits Align = getContext().getTypeAlignInChars(T: MaterializedType);
6925
6926	auto InsertResult = MaterializedGlobalTemporaryMap.insert(KV: {E, nullptr});
6927	if (!InsertResult.second) {
6928	// We've seen this before: either we already created it or we're in the
6929	// process of doing so.
6930	if (!InsertResult.first->second) {
6931	// We recursively re-entered this function, probably during emission of
6932	// the initializer. Create a placeholder. We'll clean this up in the
6933	// outer call, at the end of this function.
6934	llvm::Type *Type = getTypes().ConvertTypeForMem(T: MaterializedType);
6935	InsertResult.first->second = new llvm::GlobalVariable(
6936	getModule(), Type, false, llvm::GlobalVariable::InternalLinkage,
6937	nullptr);
6938	}
6939	return ConstantAddress(InsertResult.first->second,
6940	llvm::cast<llvm::GlobalVariable>(
6941	Val: InsertResult.first->second->stripPointerCasts())
6942	->getValueType(),
6943	Align);
6944	}
6945
6946	// FIXME: If an externally-visible declaration extends multiple temporaries,
6947	// we need to give each temporary the same name in every translation unit (and
6948	// we also need to make the temporaries externally-visible).
6949	SmallString<256> Name;
6950	llvm::raw_svector_ostream Out(Name);
6951	getCXXABI().getMangleContext().mangleReferenceTemporary(
6952	D: VD, ManglingNumber: E->getManglingNumber(), Out);
6953
6954	APValue *Value = nullptr;
6955	if (E->getStorageDuration() == SD_Static && VD->evaluateValue()) {
6956	// If the initializer of the extending declaration is a constant
6957	// initializer, we should have a cached constant initializer for this
6958	// temporary. Note that this might have a different value from the value
6959	// computed by evaluating the initializer if the surrounding constant
6960	// expression modifies the temporary.
6961	Value = E->getOrCreateValue(MayCreate: false);
6962	}
6963
6964	// Try evaluating it now, it might have a constant initializer.
6965	Expr::EvalResult EvalResult;
6966	if (!Value && Init->EvaluateAsRValue(Result&: EvalResult, Ctx: getContext()) &&
6967	!EvalResult.hasSideEffects())
6968	Value = &EvalResult.Val;
6969
6970	LangAS AddrSpace = GetGlobalVarAddressSpace(D: VD);
6971
6972	std::optional<ConstantEmitter> emitter;
6973	llvm::Constant *InitialValue = nullptr;
6974	bool Constant = false;
6975	llvm::Type *Type;
6976	if (Value) {
6977	// The temporary has a constant initializer, use it.
6978	emitter.emplace(args&: *this);
6979	InitialValue = emitter->emitForInitializer(value: *Value, destAddrSpace: AddrSpace,
6980	destType: MaterializedType);
6981	Constant =
6982	MaterializedType.isConstantStorage(Ctx: getContext(), /*ExcludeCtor*/ Value,
6983	/*ExcludeDtor*/ false);
6984	Type = InitialValue->getType();
6985	} else {
6986	// No initializer, the initialization will be provided when we
6987	// initialize the declaration which performed lifetime extension.
6988	Type = getTypes().ConvertTypeForMem(T: MaterializedType);
6989	}
6990
6991	// Create a global variable for this lifetime-extended temporary.
6992	llvm::GlobalValue::LinkageTypes Linkage = getLLVMLinkageVarDefinition(VD);
6993	if (Linkage == llvm::GlobalVariable::ExternalLinkage) {
6994	const VarDecl *InitVD;
6995	if (VD->isStaticDataMember() && VD->getAnyInitializer(D&: InitVD) &&
6996	isa<CXXRecordDecl>(Val: InitVD->getLexicalDeclContext())) {
6997	// Temporaries defined inside a class get linkonce_odr linkage because the
6998	// class can be defined in multiple translation units.
6999	Linkage = llvm::GlobalVariable::LinkOnceODRLinkage;
7000	} else {
7001	// There is no need for this temporary to have external linkage if the
7002	// VarDecl has external linkage.
7003	Linkage = llvm::GlobalVariable::InternalLinkage;
7004	}
7005	}
7006	auto TargetAS = getContext().getTargetAddressSpace(AS: AddrSpace);
7007	auto *GV = new llvm::GlobalVariable(
7008	getModule(), Type, Constant, Linkage, InitialValue, Name.c_str(),
7009	/*InsertBefore=*/nullptr, llvm::GlobalVariable::NotThreadLocal, TargetAS);
7010	if (emitter) emitter->finalize(global: GV);
7011	// Don't assign dllimport or dllexport to local linkage globals.
7012	if (!llvm::GlobalValue::isLocalLinkage(Linkage)) {
7013	setGVProperties(GV, D: VD);
7014	if (GV->getDLLStorageClass() == llvm::GlobalVariable::DLLExportStorageClass)
7015	// The reference temporary should never be dllexport.
7016	GV->setDLLStorageClass(llvm::GlobalVariable::DefaultStorageClass);
7017	}
7018	GV->setAlignment(Align.getAsAlign());
7019	if (supportsCOMDAT() && GV->isWeakForLinker())
7020	GV->setComdat(TheModule.getOrInsertComdat(Name: GV->getName()));
7021	if (VD->getTLSKind())
7022	setTLSMode(GV, D: *VD);
7023	llvm::Constant *CV = GV;
7024	if (AddrSpace != LangAS::Default)
7025	CV = getTargetCodeGenInfo().performAddrSpaceCast(
7026	CGM&: *this, V: GV, SrcAddr: AddrSpace,
7027	DestTy: llvm::PointerType::get(
7028	C&: getLLVMContext(),
7029	AddressSpace: getContext().getTargetAddressSpace(AS: LangAS::Default)));
7030
7031	// Update the map with the new temporary. If we created a placeholder above,
7032	// replace it with the new global now.
7033	llvm::Constant *&Entry = MaterializedGlobalTemporaryMap[E];
7034	if (Entry) {
7035	Entry->replaceAllUsesWith(V: CV);
7036	llvm::cast<llvm::GlobalVariable>(Val: Entry)->eraseFromParent();
7037	}
7038	Entry = CV;
7039
7040	return ConstantAddress(CV, Type, Align);
7041	}
7042
7043	/// EmitObjCPropertyImplementations - Emit information for synthesized
7044	/// properties for an implementation.
7045	void CodeGenModule::EmitObjCPropertyImplementations(const
7046	ObjCImplementationDecl *D) {
7047	for (const auto *PID : D->property_impls()) {
7048	// Dynamic is just for type-checking.
7049	if (PID->getPropertyImplementation() == ObjCPropertyImplDecl::Synthesize) {
7050	ObjCPropertyDecl *PD = PID->getPropertyDecl();
7051
7052	// Determine which methods need to be implemented, some may have
7053	// been overridden. Note that ::isPropertyAccessor is not the method
7054	// we want, that just indicates if the decl came from a
7055	// property. What we want to know is if the method is defined in
7056	// this implementation.
7057	auto *Getter = PID->getGetterMethodDecl();
7058	if (!Getter || Getter->isSynthesizedAccessorStub())
7059	CodeGenFunction(*this).GenerateObjCGetter(
7060	IMP: const_cast<ObjCImplementationDecl *>(D), PID);
7061	auto *Setter = PID->getSetterMethodDecl();
7062	if (!PD->isReadOnly() && (!Setter || Setter->isSynthesizedAccessorStub()))
7063	CodeGenFunction(*this).GenerateObjCSetter(
7064	IMP: const_cast<ObjCImplementationDecl *>(D), PID);
7065	}
7066	}
7067	}
7068
7069	static bool needsDestructMethod(ObjCImplementationDecl *impl) {
7070	const ObjCInterfaceDecl *iface = impl->getClassInterface();
7071	for (const ObjCIvarDecl *ivar = iface->all_declared_ivar_begin();
7072	ivar; ivar = ivar->getNextIvar())
7073	if (ivar->getType().isDestructedType())
7074	return true;
7075
7076	return false;
7077	}
7078
7079	static bool AllTrivialInitializers(CodeGenModule &CGM,
7080	ObjCImplementationDecl *D) {
7081	CodeGenFunction CGF(CGM);
7082	for (ObjCImplementationDecl::init_iterator B = D->init_begin(),
7083	E = D->init_end(); B != E; ++B) {
7084	CXXCtorInitializer *CtorInitExp = *B;
7085	Expr *Init = CtorInitExp->getInit();
7086	if (!CGF.isTrivialInitializer(Init))
7087	return false;
7088	}
7089	return true;
7090	}
7091
7092	/// EmitObjCIvarInitializations - Emit information for ivar initialization
7093	/// for an implementation.
7094	void CodeGenModule::EmitObjCIvarInitializations(ObjCImplementationDecl *D) {
7095	// We might need a .cxx_destruct even if we don't have any ivar initializers.
7096	if (needsDestructMethod(impl: D)) {
7097	const IdentifierInfo *II = &getContext().Idents.get(Name: ".cxx_destruct");
7098	Selector cxxSelector = getContext().Selectors.getSelector(NumArgs: 0, IIV: &II);
7099	ObjCMethodDecl *DTORMethod = ObjCMethodDecl::Create(
7100	C&: getContext(), beginLoc: D->getLocation(), endLoc: D->getLocation(), SelInfo: cxxSelector,
7101	T: getContext().VoidTy, ReturnTInfo: nullptr, contextDecl: D,
7102	/*isInstance=*/true, /*isVariadic=*/false,
7103	/*isPropertyAccessor=*/true, /*isSynthesizedAccessorStub=*/false,
7104	/*isImplicitlyDeclared=*/true,
7105	/*isDefined=*/false, impControl: ObjCImplementationControl::Required);
7106	D->addInstanceMethod(method: DTORMethod);
7107	CodeGenFunction(*this).GenerateObjCCtorDtorMethod(IMP: D, MD: DTORMethod, ctor: false);
7108	D->setHasDestructors(true);
7109	}
7110
7111	// If the implementation doesn't have any ivar initializers, we don't need
7112	// a .cxx_construct.
7113	if (D->getNumIvarInitializers() == 0 ||
7114	AllTrivialInitializers(CGM&: *this, D))
7115	return;
7116
7117	const IdentifierInfo *II = &getContext().Idents.get(Name: ".cxx_construct");
7118	Selector cxxSelector = getContext().Selectors.getSelector(NumArgs: 0, IIV: &II);
7119	// The constructor returns 'self'.
7120	ObjCMethodDecl *CTORMethod = ObjCMethodDecl::Create(
7121	C&: getContext(), beginLoc: D->getLocation(), endLoc: D->getLocation(), SelInfo: cxxSelector,
7122	T: getContext().getObjCIdType(), ReturnTInfo: nullptr, contextDecl: D, /*isInstance=*/true,
7123	/*isVariadic=*/false,
7124	/*isPropertyAccessor=*/true, /*isSynthesizedAccessorStub=*/false,
7125	/*isImplicitlyDeclared=*/true,
7126	/*isDefined=*/false, impControl: ObjCImplementationControl::Required);
7127	D->addInstanceMethod(method: CTORMethod);
7128	CodeGenFunction(*this).GenerateObjCCtorDtorMethod(IMP: D, MD: CTORMethod, ctor: true);
7129	D->setHasNonZeroConstructors(true);
7130	}
7131
7132	// EmitLinkageSpec - Emit all declarations in a linkage spec.
7133	void CodeGenModule::EmitLinkageSpec(const LinkageSpecDecl *LSD) {
7134	if (LSD->getLanguage() != LinkageSpecLanguageIDs::C &&
7135	LSD->getLanguage() != LinkageSpecLanguageIDs::CXX) {
7136	ErrorUnsupported(D: LSD, Type: "linkage spec");
7137	return;
7138	}
7139
7140	EmitDeclContext(DC: LSD);
7141	}
7142
7143	void CodeGenModule::EmitTopLevelStmt(const TopLevelStmtDecl *D) {
7144	// Device code should not be at top level.
7145	if (LangOpts.CUDA && LangOpts.CUDAIsDevice)
7146	return;
7147
7148	std::unique_ptr<CodeGenFunction> &CurCGF =
7149	GlobalTopLevelStmtBlockInFlight.first;
7150
7151	// We emitted a top-level stmt but after it there is initialization.
7152	// Stop squashing the top-level stmts into a single function.
7153	if (CurCGF && CXXGlobalInits.back() != CurCGF->CurFn) {
7154	CurCGF->FinishFunction(EndLoc: D->getEndLoc());
7155	CurCGF = nullptr;
7156	}
7157
7158	if (!CurCGF) {
7159	// void __stmts__N(void)
7160	// FIXME: Ask the ABI name mangler to pick a name.
7161	std::string Name = "__stmts__" + llvm::utostr(X: CXXGlobalInits.size());
7162	FunctionArgList Args;
7163	QualType RetTy = getContext().VoidTy;
7164	const CGFunctionInfo &FnInfo =
7165	getTypes().arrangeBuiltinFunctionDeclaration(resultType: RetTy, args: Args);
7166	llvm::FunctionType *FnTy = getTypes().GetFunctionType(Info: FnInfo);
7167	llvm::Function *Fn = llvm::Function::Create(
7168	Ty: FnTy, Linkage: llvm::GlobalValue::InternalLinkage, N: Name, M: &getModule());
7169
7170	CurCGF.reset(p: new CodeGenFunction(*this));
7171	GlobalTopLevelStmtBlockInFlight.second = D;
7172	CurCGF->StartFunction(GD: GlobalDecl(), RetTy, Fn, FnInfo, Args,
7173	Loc: D->getBeginLoc(), StartLoc: D->getBeginLoc());
7174	CXXGlobalInits.push_back(x: Fn);
7175	}
7176
7177	CurCGF->EmitStmt(S: D->getStmt());
7178	}
7179
7180	void CodeGenModule::EmitDeclContext(const DeclContext *DC) {
7181	for (auto *I : DC->decls()) {
7182	// Unlike other DeclContexts, the contents of an ObjCImplDecl at TU scope
7183	// are themselves considered "top-level", so EmitTopLevelDecl on an
7184	// ObjCImplDecl does not recursively visit them. We need to do that in
7185	// case they're nested inside another construct (LinkageSpecDecl /
7186	// ExportDecl) that does stop them from being considered "top-level".
7187	if (auto *OID = dyn_cast<ObjCImplDecl>(Val: I)) {
7188	for (auto *M : OID->methods())
7189	EmitTopLevelDecl(D: M);
7190	}
7191
7192	EmitTopLevelDecl(D: I);
7193	}
7194	}
7195
7196	/// EmitTopLevelDecl - Emit code for a single top level declaration.
7197	void CodeGenModule::EmitTopLevelDecl(Decl *D) {
7198	// Ignore dependent declarations.
7199	if (D->isTemplated())
7200	return;
7201
7202	// Consteval function shouldn't be emitted.
7203	if (auto *FD = dyn_cast<FunctionDecl>(Val: D); FD && FD->isImmediateFunction())
7204	return;
7205
7206	switch (D->getKind()) {
7207	case Decl::CXXConversion:
7208	case Decl::CXXMethod:
7209	case Decl::Function:
7210	EmitGlobal(GD: cast<FunctionDecl>(Val: D));
7211	// Always provide some coverage mapping
7212	// even for the functions that aren't emitted.
7213	AddDeferredUnusedCoverageMapping(D);
7214	break;
7215
7216	case Decl::CXXDeductionGuide:
7217	// Function-like, but does not result in code emission.
7218	break;
7219
7220	case Decl::Var:
7221	case Decl::Decomposition:
7222	case Decl::VarTemplateSpecialization:
7223	EmitGlobal(GD: cast<VarDecl>(Val: D));
7224	if (auto *DD = dyn_cast<DecompositionDecl>(Val: D))
7225	for (auto *B : DD->flat_bindings())
7226	if (auto *HD = B->getHoldingVar())
7227	EmitGlobal(GD: HD);
7228
7229	break;
7230
7231	// Indirect fields from global anonymous structs and unions can be
7232	// ignored; only the actual variable requires IR gen support.
7233	case Decl::IndirectField:
7234	break;
7235
7236	// C++ Decls
7237	case Decl::Namespace:
7238	EmitDeclContext(DC: cast<NamespaceDecl>(Val: D));
7239	break;
7240	case Decl::ClassTemplateSpecialization: {
7241	const auto *Spec = cast<ClassTemplateSpecializationDecl>(Val: D);
7242	if (CGDebugInfo *DI = getModuleDebugInfo())
7243	if (Spec->getSpecializationKind() ==
7244	TSK_ExplicitInstantiationDefinition &&
7245	Spec->hasDefinition())
7246	DI->completeTemplateDefinition(SD: *Spec);
7247	} [[fallthrough]];
7248	case Decl::CXXRecord: {
7249	CXXRecordDecl *CRD = cast<CXXRecordDecl>(Val: D);
7250	if (CGDebugInfo *DI = getModuleDebugInfo()) {
7251	if (CRD->hasDefinition())
7252	DI->EmitAndRetainType(Ty: getContext().getRecordType(Decl: cast<RecordDecl>(Val: D)));
7253	if (auto *ES = D->getASTContext().getExternalSource())
7254	if (ES->hasExternalDefinitions(D) == ExternalASTSource::EK_Never)
7255	DI->completeUnusedClass(D: *CRD);
7256	}
7257	// Emit any static data members, they may be definitions.
7258	for (auto *I : CRD->decls())
7259	if (isa<VarDecl>(Val: I) || isa<CXXRecordDecl>(Val: I) || isa<EnumDecl>(Val: I))
7260	EmitTopLevelDecl(D: I);
7261	break;
7262	}
7263	// No code generation needed.
7264	case Decl::UsingShadow:
7265	case Decl::ClassTemplate:
7266	case Decl::VarTemplate:
7267	case Decl::Concept:
7268	case Decl::VarTemplatePartialSpecialization:
7269	case Decl::FunctionTemplate:
7270	case Decl::TypeAliasTemplate:
7271	case Decl::Block:
7272	case Decl::Empty:
7273	case Decl::Binding:
7274	break;
7275	case Decl::Using: // using X; [C++]
7276	if (CGDebugInfo *DI = getModuleDebugInfo())
7277	DI->EmitUsingDecl(UD: cast<UsingDecl>(Val&: *D));
7278	break;
7279	case Decl::UsingEnum: // using enum X; [C++]
7280	if (CGDebugInfo *DI = getModuleDebugInfo())
7281	DI->EmitUsingEnumDecl(UD: cast<UsingEnumDecl>(Val&: *D));
7282	break;
7283	case Decl::NamespaceAlias:
7284	if (CGDebugInfo *DI = getModuleDebugInfo())
7285	DI->EmitNamespaceAlias(NA: cast<NamespaceAliasDecl>(Val&: *D));
7286	break;
7287	case Decl::UsingDirective: // using namespace X; [C++]
7288	if (CGDebugInfo *DI = getModuleDebugInfo())
7289	DI->EmitUsingDirective(UD: cast<UsingDirectiveDecl>(Val&: *D));
7290	break;
7291	case Decl::CXXConstructor:
7292	getCXXABI().EmitCXXConstructors(D: cast<CXXConstructorDecl>(Val: D));
7293	break;
7294	case Decl::CXXDestructor:
7295	getCXXABI().EmitCXXDestructors(D: cast<CXXDestructorDecl>(Val: D));
7296	break;
7297
7298	case Decl::StaticAssert:
7299	// Nothing to do.
7300	break;
7301
7302	// Objective-C Decls
7303
7304	// Forward declarations, no (immediate) code generation.
7305	case Decl::ObjCInterface:
7306	case Decl::ObjCCategory:
7307	break;
7308
7309	case Decl::ObjCProtocol: {
7310	auto *Proto = cast<ObjCProtocolDecl>(Val: D);
7311	if (Proto->isThisDeclarationADefinition())
7312	ObjCRuntime->GenerateProtocol(OPD: Proto);
7313	break;
7314	}
7315
7316	case Decl::ObjCCategoryImpl:
7317	// Categories have properties but don't support synthesize so we
7318	// can ignore them here.
7319	ObjCRuntime->GenerateCategory(OCD: cast<ObjCCategoryImplDecl>(Val: D));
7320	break;
7321
7322	case Decl::ObjCImplementation: {
7323	auto *OMD = cast<ObjCImplementationDecl>(Val: D);
7324	EmitObjCPropertyImplementations(D: OMD);
7325	EmitObjCIvarInitializations(D: OMD);
7326	ObjCRuntime->GenerateClass(OID: OMD);
7327	// Emit global variable debug information.
7328	if (CGDebugInfo *DI = getModuleDebugInfo())
7329	if (getCodeGenOpts().hasReducedDebugInfo())
7330	DI->getOrCreateInterfaceType(Ty: getContext().getObjCInterfaceType(
7331	Decl: OMD->getClassInterface()), Loc: OMD->getLocation());
7332	break;
7333	}
7334	case Decl::ObjCMethod: {
7335	auto *OMD = cast<ObjCMethodDecl>(Val: D);
7336	// If this is not a prototype, emit the body.
7337	if (OMD->getBody())
7338	CodeGenFunction(*this).GenerateObjCMethod(OMD);
7339	break;
7340	}
7341	case Decl::ObjCCompatibleAlias:
7342	ObjCRuntime->RegisterAlias(OAD: cast<ObjCCompatibleAliasDecl>(Val: D));
7343	break;
7344
7345	case Decl::PragmaComment: {
7346	const auto *PCD = cast<PragmaCommentDecl>(Val: D);
7347	switch (PCD->getCommentKind()) {
7348	case PCK_Unknown:
7349	llvm_unreachable("unexpected pragma comment kind");
7350	case PCK_Linker:
7351	AppendLinkerOptions(Opts: PCD->getArg());
7352	break;
7353	case PCK_Lib:
7354	AddDependentLib(Lib: PCD->getArg());
7355	break;
7356	case PCK_Compiler:
7357	case PCK_ExeStr:
7358	case PCK_User:
7359	break; // We ignore all of these.
7360	}
7361	break;
7362	}
7363
7364	case Decl::PragmaDetectMismatch: {
7365	const auto *PDMD = cast<PragmaDetectMismatchDecl>(Val: D);
7366	AddDetectMismatch(Name: PDMD->getName(), Value: PDMD->getValue());
7367	break;
7368	}
7369
7370	case Decl::LinkageSpec:
7371	EmitLinkageSpec(LSD: cast<LinkageSpecDecl>(Val: D));
7372	break;
7373
7374	case Decl::FileScopeAsm: {
7375	// File-scope asm is ignored during device-side CUDA compilation.
7376	if (LangOpts.CUDA && LangOpts.CUDAIsDevice)
7377	break;
7378	// File-scope asm is ignored during device-side OpenMP compilation.
7379	if (LangOpts.OpenMPIsTargetDevice)
7380	break;
7381	// File-scope asm is ignored during device-side SYCL compilation.
7382	if (LangOpts.SYCLIsDevice)
7383	break;
7384	auto *AD = cast<FileScopeAsmDecl>(Val: D);
7385	getModule().appendModuleInlineAsm(Asm: AD->getAsmString());
7386	break;
7387	}
7388
7389	case Decl::TopLevelStmt:
7390	EmitTopLevelStmt(D: cast<TopLevelStmtDecl>(Val: D));
7391	break;
7392
7393	case Decl::Import: {
7394	auto *Import = cast<ImportDecl>(Val: D);
7395
7396	// If we've already imported this module, we're done.
7397	if (!ImportedModules.insert(X: Import->getImportedModule()))
7398	break;
7399
7400	// Emit debug information for direct imports.
7401	if (!Import->getImportedOwningModule()) {
7402	if (CGDebugInfo *DI = getModuleDebugInfo())
7403	DI->EmitImportDecl(ID: *Import);
7404	}
7405
7406	// For C++ standard modules we are done - we will call the module
7407	// initializer for imported modules, and that will likewise call those for
7408	// any imports it has.
7409	if (CXX20ModuleInits && Import->getImportedModule() &&
7410	Import->getImportedModule()->isNamedModule())
7411	break;
7412
7413	// For clang C++ module map modules the initializers for sub-modules are
7414	// emitted here.
7415
7416	// Find all of the submodules and emit the module initializers.
7417	llvm::SmallPtrSet<clang::Module *, 16> Visited;
7418	SmallVector<clang::Module *, 16> Stack;
7419	Visited.insert(Ptr: Import->getImportedModule());
7420	Stack.push_back(Elt: Import->getImportedModule());
7421
7422	while (!Stack.empty()) {
7423	clang::Module *Mod = Stack.pop_back_val();
7424	if (!EmittedModuleInitializers.insert(Ptr: Mod).second)
7425	continue;
7426
7427	for (auto *D : Context.getModuleInitializers(M: Mod))
7428	EmitTopLevelDecl(D);
7429
7430	// Visit the submodules of this module.
7431	for (auto *Submodule : Mod->submodules()) {
7432	// Skip explicit children; they need to be explicitly imported to emit
7433	// the initializers.
7434	if (Submodule->IsExplicit)
7435	continue;
7436
7437	if (Visited.insert(Ptr: Submodule).second)
7438	Stack.push_back(Elt: Submodule);
7439	}
7440	}
7441	break;
7442	}
7443
7444	case Decl::Export:
7445	EmitDeclContext(DC: cast<ExportDecl>(Val: D));
7446	break;
7447
7448	case Decl::OMPThreadPrivate:
7449	EmitOMPThreadPrivateDecl(D: cast<OMPThreadPrivateDecl>(Val: D));
7450	break;
7451
7452	case Decl::OMPAllocate:
7453	EmitOMPAllocateDecl(D: cast<OMPAllocateDecl>(Val: D));
7454	break;
7455
7456	case Decl::OMPDeclareReduction:
7457	EmitOMPDeclareReduction(D: cast<OMPDeclareReductionDecl>(Val: D));
7458	break;
7459
7460	case Decl::OMPDeclareMapper:
7461	EmitOMPDeclareMapper(D: cast<OMPDeclareMapperDecl>(Val: D));
7462	break;
7463
7464	case Decl::OMPRequires:
7465	EmitOMPRequiresDecl(D: cast<OMPRequiresDecl>(Val: D));
7466	break;
7467
7468	case Decl::Typedef:
7469	case Decl::TypeAlias: // using foo = bar; [C++11]
7470	if (CGDebugInfo *DI = getModuleDebugInfo())
7471	DI->EmitAndRetainType(
7472	Ty: getContext().getTypedefType(Decl: cast<TypedefNameDecl>(Val: D)));
7473	break;
7474
7475	case Decl::Record:
7476	if (CGDebugInfo *DI = getModuleDebugInfo())
7477	if (cast<RecordDecl>(Val: D)->getDefinition())
7478	DI->EmitAndRetainType(Ty: getContext().getRecordType(Decl: cast<RecordDecl>(Val: D)));
7479	break;
7480
7481	case Decl::Enum:
7482	if (CGDebugInfo *DI = getModuleDebugInfo())
7483	if (cast<EnumDecl>(Val: D)->getDefinition())
7484	DI->EmitAndRetainType(Ty: getContext().getEnumType(Decl: cast<EnumDecl>(Val: D)));
7485	break;
7486
7487	case Decl::HLSLBuffer:
7488	getHLSLRuntime().addBuffer(D: cast<HLSLBufferDecl>(Val: D));
7489	break;
7490
7491	case Decl::OpenACCDeclare:
7492	EmitOpenACCDeclare(D: cast<OpenACCDeclareDecl>(Val: D));
7493	break;
7494	case Decl::OpenACCRoutine:
7495	EmitOpenACCRoutine(D: cast<OpenACCRoutineDecl>(Val: D));
7496	break;
7497
7498	default:
7499	// Make sure we handled everything we should, every other kind is a
7500	// non-top-level decl. FIXME: Would be nice to have an isTopLevelDeclKind
7501	// function. Need to recode Decl::Kind to do that easily.
7502	assert(isa<TypeDecl>(D) && "Unsupported decl kind");
7503	break;
7504	}
7505	}
7506
7507	void CodeGenModule::AddDeferredUnusedCoverageMapping(Decl *D) {
7508	// Do we need to generate coverage mapping?
7509	if (!CodeGenOpts.CoverageMapping)
7510	return;
7511	switch (D->getKind()) {
7512	case Decl::CXXConversion:
7513	case Decl::CXXMethod:
7514	case Decl::Function:
7515	case Decl::ObjCMethod:
7516	case Decl::CXXConstructor:
7517	case Decl::CXXDestructor: {
7518	if (!cast<FunctionDecl>(Val: D)->doesThisDeclarationHaveABody())
7519	break;
7520	SourceManager &SM = getContext().getSourceManager();
7521	if (LimitedCoverage && SM.getMainFileID() != SM.getFileID(SpellingLoc: D->getBeginLoc()))
7522	break;
7523	if (!llvm::coverage::SystemHeadersCoverage &&
7524	SM.isInSystemHeader(Loc: D->getBeginLoc()))
7525	break;
7526	DeferredEmptyCoverageMappingDecls.try_emplace(Key: D, Args: true);
7527	break;
7528	}
7529	default:
7530	break;
7531	};
7532	}
7533
7534	void CodeGenModule::ClearUnusedCoverageMapping(const Decl *D) {
7535	// Do we need to generate coverage mapping?
7536	if (!CodeGenOpts.CoverageMapping)
7537	return;
7538	if (const auto *Fn = dyn_cast<FunctionDecl>(Val: D)) {
7539	if (Fn->isTemplateInstantiation())
7540	ClearUnusedCoverageMapping(D: Fn->getTemplateInstantiationPattern());
7541	}
7542	DeferredEmptyCoverageMappingDecls.insert_or_assign(Key: D, Val: false);
7543	}
7544
7545	void CodeGenModule::EmitDeferredUnusedCoverageMappings() {
7546	// We call takeVector() here to avoid use-after-free.
7547	// FIXME: DeferredEmptyCoverageMappingDecls is getting mutated because
7548	// we deserialize function bodies to emit coverage info for them, and that
7549	// deserializes more declarations. How should we handle that case?
7550	for (const auto &Entry : DeferredEmptyCoverageMappingDecls.takeVector()) {
7551	if (!Entry.second)
7552	continue;
7553	const Decl *D = Entry.first;
7554	switch (D->getKind()) {
7555	case Decl::CXXConversion:
7556	case Decl::CXXMethod:
7557	case Decl::Function:
7558	case Decl::ObjCMethod: {
7559	CodeGenPGO PGO(*this);
7560	GlobalDecl GD(cast<FunctionDecl>(Val: D));
7561	PGO.emitEmptyCounterMapping(D, FuncName: getMangledName(GD),
7562	Linkage: getFunctionLinkage(GD));
7563	break;
7564	}
7565	case Decl::CXXConstructor: {
7566	CodeGenPGO PGO(*this);
7567	GlobalDecl GD(cast<CXXConstructorDecl>(Val: D), Ctor_Base);
7568	PGO.emitEmptyCounterMapping(D, FuncName: getMangledName(GD),
7569	Linkage: getFunctionLinkage(GD));
7570	break;
7571	}
7572	case Decl::CXXDestructor: {
7573	CodeGenPGO PGO(*this);
7574	GlobalDecl GD(cast<CXXDestructorDecl>(Val: D), Dtor_Base);
7575	PGO.emitEmptyCounterMapping(D, FuncName: getMangledName(GD),
7576	Linkage: getFunctionLinkage(GD));
7577	break;
7578	}
7579	default:
7580	break;
7581	};
7582	}
7583	}
7584
7585	void CodeGenModule::EmitMainVoidAlias() {
7586	// In order to transition away from "__original_main" gracefully, emit an
7587	// alias for "main" in the no-argument case so that libc can detect when
7588	// new-style no-argument main is in used.
7589	if (llvm::Function *F = getModule().getFunction(Name: "main")) {
7590	if (!F->isDeclaration() && F->arg_size() == 0 && !F->isVarArg() &&
7591	F->getReturnType()->isIntegerTy(Bitwidth: Context.getTargetInfo().getIntWidth())) {
7592	auto *GA = llvm::GlobalAlias::create(Name: "__main_void", Aliasee: F);
7593	GA->setVisibility(llvm::GlobalValue::HiddenVisibility);
7594	}
7595	}
7596	}
7597
7598	/// Turns the given pointer into a constant.
7599	static llvm::Constant *GetPointerConstant(llvm::LLVMContext &Context,
7600	const void *Ptr) {
7601	uintptr_t PtrInt = reinterpret_cast<uintptr_t>(Ptr);
7602	llvm::Type *i64 = llvm::Type::getInt64Ty(C&: Context);
7603	return llvm::ConstantInt::get(Ty: i64, V: PtrInt);
7604	}
7605
7606	static void EmitGlobalDeclMetadata(CodeGenModule &CGM,
7607	llvm::NamedMDNode *&GlobalMetadata,
7608	GlobalDecl D,
7609	llvm::GlobalValue *Addr) {
7610	if (!GlobalMetadata)
7611	GlobalMetadata =
7612	CGM.getModule().getOrInsertNamedMetadata(Name: "clang.global.decl.ptrs");
7613
7614	// TODO: should we report variant information for ctors/dtors?
7615	llvm::Metadata *Ops[] = {llvm::ConstantAsMetadata::get(C: Addr),
7616	llvm::ConstantAsMetadata::get(C: GetPointerConstant(
7617	Context&: CGM.getLLVMContext(), Ptr: D.getDecl()))};
7618	GlobalMetadata->addOperand(M: llvm::MDNode::get(Context&: CGM.getLLVMContext(), MDs: Ops));
7619	}
7620
7621	bool CodeGenModule::CheckAndReplaceExternCIFuncs(llvm::GlobalValue *Elem,
7622	llvm::GlobalValue *CppFunc) {
7623	// Store the list of ifuncs we need to replace uses in.
7624	llvm::SmallVector<llvm::GlobalIFunc *> IFuncs;
7625	// List of ConstantExprs that we should be able to delete when we're done
7626	// here.
7627	llvm::SmallVector<llvm::ConstantExpr *> CEs;
7628
7629	// It isn't valid to replace the extern-C ifuncs if all we find is itself!
7630	if (Elem == CppFunc)
7631	return false;
7632
7633	// First make sure that all users of this are ifuncs (or ifuncs via a
7634	// bitcast), and collect the list of ifuncs and CEs so we can work on them
7635	// later.
7636	for (llvm::User *User : Elem->users()) {
7637	// Users can either be a bitcast ConstExpr that is used by the ifuncs, OR an
7638	// ifunc directly. In any other case, just give up, as we don't know what we
7639	// could break by changing those.
7640	if (auto *ConstExpr = dyn_cast<llvm::ConstantExpr>(Val: User)) {
7641	if (ConstExpr->getOpcode() != llvm::Instruction::BitCast)
7642	return false;
7643
7644	for (llvm::User *CEUser : ConstExpr->users()) {
7645	if (auto *IFunc = dyn_cast<llvm::GlobalIFunc>(Val: CEUser)) {
7646	IFuncs.push_back(Elt: IFunc);
7647	} else {
7648	return false;
7649	}
7650	}
7651	CEs.push_back(Elt: ConstExpr);
7652	} else if (auto *IFunc = dyn_cast<llvm::GlobalIFunc>(Val: User)) {
7653	IFuncs.push_back(Elt: IFunc);
7654	} else {
7655	// This user is one we don't know how to handle, so fail redirection. This
7656	// will result in an ifunc retaining a resolver name that will ultimately
7657	// fail to be resolved to a defined function.
7658	return false;
7659	}
7660	}
7661
7662	// Now we know this is a valid case where we can do this alias replacement, we
7663	// need to remove all of the references to Elem (and the bitcasts!) so we can
7664	// delete it.
7665	for (llvm::GlobalIFunc *IFunc : IFuncs)
7666	IFunc->setResolver(nullptr);
7667	for (llvm::ConstantExpr *ConstExpr : CEs)
7668	ConstExpr->destroyConstant();
7669
7670	// We should now be out of uses for the 'old' version of this function, so we
7671	// can erase it as well.
7672	Elem->eraseFromParent();
7673
7674	for (llvm::GlobalIFunc *IFunc : IFuncs) {
7675	// The type of the resolver is always just a function-type that returns the
7676	// type of the IFunc, so create that here. If the type of the actual
7677	// resolver doesn't match, it just gets bitcast to the right thing.
7678	auto *ResolverTy =
7679	llvm::FunctionType::get(Result: IFunc->getType(), /*isVarArg*/ false);
7680	llvm::Constant *Resolver = GetOrCreateLLVMFunction(
7681	MangledName: CppFunc->getName(), Ty: ResolverTy, GD: {}, /*ForVTable*/ false);
7682	IFunc->setResolver(Resolver);
7683	}
7684	return true;
7685	}
7686
7687	/// For each function which is declared within an extern "C" region and marked
7688	/// as 'used', but has internal linkage, create an alias from the unmangled
7689	/// name to the mangled name if possible. People expect to be able to refer
7690	/// to such functions with an unmangled name from inline assembly within the
7691	/// same translation unit.
7692	void CodeGenModule::EmitStaticExternCAliases() {
7693	if (!getTargetCodeGenInfo().shouldEmitStaticExternCAliases())
7694	return;
7695	for (auto &I : StaticExternCValues) {
7696	const IdentifierInfo *Name = I.first;
7697	llvm::GlobalValue *Val = I.second;
7698
7699	// If Val is null, that implies there were multiple declarations that each
7700	// had a claim to the unmangled name. In this case, generation of the alias
7701	// is suppressed. See CodeGenModule::MaybeHandleStaticInExternC.
7702	if (!Val)
7703	break;
7704
7705	llvm::GlobalValue *ExistingElem =
7706	getModule().getNamedValue(Name: Name->getName());
7707
7708	// If there is either not something already by this name, or we were able to
7709	// replace all uses from IFuncs, create the alias.
7710	if (!ExistingElem || CheckAndReplaceExternCIFuncs(Elem: ExistingElem, CppFunc: Val))
7711	addCompilerUsedGlobal(GV: llvm::GlobalAlias::create(Name: Name->getName(), Aliasee: Val));
7712	}
7713	}
7714
7715	bool CodeGenModule::lookupRepresentativeDecl(StringRef MangledName,
7716	GlobalDecl &Result) const {
7717	auto Res = Manglings.find(Key: MangledName);
7718	if (Res == Manglings.end())
7719	return false;
7720	Result = Res->getValue();
7721	return true;
7722	}
7723
7724	/// Emits metadata nodes associating all the global values in the
7725	/// current module with the Decls they came from. This is useful for
7726	/// projects using IR gen as a subroutine.
7727	///
7728	/// Since there's currently no way to associate an MDNode directly
7729	/// with an llvm::GlobalValue, we create a global named metadata
7730	/// with the name 'clang.global.decl.ptrs'.
7731	void CodeGenModule::EmitDeclMetadata() {
7732	llvm::NamedMDNode *GlobalMetadata = nullptr;
7733
7734	for (auto &I : MangledDeclNames) {
7735	llvm::GlobalValue *Addr = getModule().getNamedValue(Name: I.second);
7736	// Some mangled names don't necessarily have an associated GlobalValue
7737	// in this module, e.g. if we mangled it for DebugInfo.
7738	if (Addr)
7739	EmitGlobalDeclMetadata(CGM&: *this, GlobalMetadata, D: I.first, Addr);
7740	}
7741	}
7742
7743	/// Emits metadata nodes for all the local variables in the current
7744	/// function.
7745	void CodeGenFunction::EmitDeclMetadata() {
7746	if (LocalDeclMap.empty()) return;
7747
7748	llvm::LLVMContext &Context = getLLVMContext();
7749
7750	// Find the unique metadata ID for this name.
7751	unsigned DeclPtrKind = Context.getMDKindID(Name: "clang.decl.ptr");
7752
7753	llvm::NamedMDNode *GlobalMetadata = nullptr;
7754
7755	for (auto &I : LocalDeclMap) {
7756	const Decl *D = I.first;
7757	llvm::Value *Addr = I.second.emitRawPointer(CGF&: *this);
7758	if (auto *Alloca = dyn_cast<llvm::AllocaInst>(Val: Addr)) {
7759	llvm::Value *DAddr = GetPointerConstant(Context&: getLLVMContext(), Ptr: D);
7760	Alloca->setMetadata(
7761	KindID: DeclPtrKind, Node: llvm::MDNode::get(
7762	Context, MDs: llvm::ValueAsMetadata::getConstant(C: DAddr)));
7763	} else if (auto *GV = dyn_cast<llvm::GlobalValue>(Val: Addr)) {
7764	GlobalDecl GD = GlobalDecl(cast<VarDecl>(Val: D));
7765	EmitGlobalDeclMetadata(CGM, GlobalMetadata, D: GD, Addr: GV);
7766	}
7767	}
7768	}
7769
7770	void CodeGenModule::EmitVersionIdentMetadata() {
7771	llvm::NamedMDNode *IdentMetadata =
7772	TheModule.getOrInsertNamedMetadata(Name: "llvm.ident");
7773	std::string Version = getClangFullVersion();
7774	llvm::LLVMContext &Ctx = TheModule.getContext();
7775
7776	llvm::Metadata *IdentNode[] = {llvm::MDString::get(Context&: Ctx, Str: Version)};
7777	IdentMetadata->addOperand(M: llvm::MDNode::get(Context&: Ctx, MDs: IdentNode));
7778	}
7779
7780	void CodeGenModule::EmitCommandLineMetadata() {
7781	llvm::NamedMDNode *CommandLineMetadata =
7782	TheModule.getOrInsertNamedMetadata(Name: "llvm.commandline");
7783	std::string CommandLine = getCodeGenOpts().RecordCommandLine;
7784	llvm::LLVMContext &Ctx = TheModule.getContext();
7785
7786	llvm::Metadata *CommandLineNode[] = {llvm::MDString::get(Context&: Ctx, Str: CommandLine)};
7787	CommandLineMetadata->addOperand(M: llvm::MDNode::get(Context&: Ctx, MDs: CommandLineNode));
7788	}
7789
7790	void CodeGenModule::EmitCoverageFile() {
7791	llvm::NamedMDNode *CUNode = TheModule.getNamedMetadata(Name: "llvm.dbg.cu");
7792	if (!CUNode)
7793	return;
7794
7795	llvm::NamedMDNode *GCov = TheModule.getOrInsertNamedMetadata(Name: "llvm.gcov");
7796	llvm::LLVMContext &Ctx = TheModule.getContext();
7797	auto *CoverageDataFile =
7798	llvm::MDString::get(Context&: Ctx, Str: getCodeGenOpts().CoverageDataFile);
7799	auto *CoverageNotesFile =
7800	llvm::MDString::get(Context&: Ctx, Str: getCodeGenOpts().CoverageNotesFile);
7801	for (int i = 0, e = CUNode->getNumOperands(); i != e; ++i) {
7802	llvm::MDNode *CU = CUNode->getOperand(i);
7803	llvm::Metadata *Elts[] = {CoverageNotesFile, CoverageDataFile, CU};
7804	GCov->addOperand(M: llvm::MDNode::get(Context&: Ctx, MDs: Elts));
7805	}
7806	}
7807
7808	llvm::Constant *CodeGenModule::GetAddrOfRTTIDescriptor(QualType Ty,
7809	bool ForEH) {
7810	// Return a bogus pointer if RTTI is disabled, unless it's for EH.
7811	// FIXME: should we even be calling this method if RTTI is disabled
7812	// and it's not for EH?
7813	if (!shouldEmitRTTI(ForEH))
7814	return llvm::Constant::getNullValue(Ty: GlobalsInt8PtrTy);
7815
7816	if (ForEH && Ty->isObjCObjectPointerType() &&
7817	LangOpts.ObjCRuntime.isGNUFamily())
7818	return ObjCRuntime->GetEHType(T: Ty);
7819
7820	return getCXXABI().getAddrOfRTTIDescriptor(Ty);
7821	}
7822
7823	void CodeGenModule::EmitOMPThreadPrivateDecl(const OMPThreadPrivateDecl *D) {
7824	// Do not emit threadprivates in simd-only mode.
7825	if (LangOpts.OpenMP && LangOpts.OpenMPSimd)
7826	return;
7827	for (auto RefExpr : D->varlist()) {
7828	auto *VD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: RefExpr)->getDecl());
7829	bool PerformInit =
7830	VD->getAnyInitializer() &&
7831	!VD->getAnyInitializer()->isConstantInitializer(Ctx&: getContext(),
7832	/*ForRef=*/false);
7833
7834	Address Addr(GetAddrOfGlobalVar(D: VD),
7835	getTypes().ConvertTypeForMem(T: VD->getType()),
7836	getContext().getDeclAlign(D: VD));
7837	if (auto InitFunction = getOpenMPRuntime().emitThreadPrivateVarDefinition(
7838	VD, VDAddr: Addr, Loc: RefExpr->getBeginLoc(), PerformInit))
7839	CXXGlobalInits.push_back(x: InitFunction);
7840	}
7841	}
7842
7843	llvm::Metadata *
7844	CodeGenModule::CreateMetadataIdentifierImpl(QualType T, MetadataTypeMap &Map,
7845	StringRef Suffix) {
7846	if (auto *FnType = T->getAs<FunctionProtoType>())
7847	T = getContext().getFunctionType(
7848	ResultTy: FnType->getReturnType(), Args: FnType->getParamTypes(),
7849	EPI: FnType->getExtProtoInfo().withExceptionSpec(ESI: EST_None));
7850
7851	llvm::Metadata *&InternalId = Map[T.getCanonicalType()];
7852	if (InternalId)
7853	return InternalId;
7854
7855	if (isExternallyVisible(L: T->getLinkage())) {
7856	std::string OutName;
7857	llvm::raw_string_ostream Out(OutName);
7858	getCXXABI().getMangleContext().mangleCanonicalTypeName(
7859	T, Out, NormalizeIntegers: getCodeGenOpts().SanitizeCfiICallNormalizeIntegers);
7860
7861	if (getCodeGenOpts().SanitizeCfiICallNormalizeIntegers)
7862	Out << ".normalized";
7863
7864	Out << Suffix;
7865
7866	InternalId = llvm::MDString::get(Context&: getLLVMContext(), Str: Out.str());
7867	} else {
7868	InternalId = llvm::MDNode::getDistinct(Context&: getLLVMContext(),
7869	MDs: llvm::ArrayRef<llvm::Metadata *>());
7870	}
7871
7872	return InternalId;
7873	}
7874
7875	llvm::Metadata *CodeGenModule::CreateMetadataIdentifierForType(QualType T) {
7876	return CreateMetadataIdentifierImpl(T, Map&: MetadataIdMap, Suffix: "");
7877	}
7878
7879	llvm::Metadata *
7880	CodeGenModule::CreateMetadataIdentifierForVirtualMemPtrType(QualType T) {
7881	return CreateMetadataIdentifierImpl(T, Map&: VirtualMetadataIdMap, Suffix: ".virtual");
7882	}
7883
7884	// Generalize pointer types to a void pointer with the qualifiers of the
7885	// originally pointed-to type, e.g. 'const char *' and 'char * const *'
7886	// generalize to 'const void *' while 'char *' and 'const char **' generalize to
7887	// 'void *'.
7888	static QualType GeneralizeType(ASTContext &Ctx, QualType Ty) {
7889	if (!Ty->isPointerType())
7890	return Ty;
7891
7892	return Ctx.getPointerType(
7893	T: QualType(Ctx.VoidTy).withCVRQualifiers(
7894	CVR: Ty->getPointeeType().getCVRQualifiers()));
7895	}
7896
7897	// Apply type generalization to a FunctionType's return and argument types
7898	static QualType GeneralizeFunctionType(ASTContext &Ctx, QualType Ty) {
7899	if (auto *FnType = Ty->getAs<FunctionProtoType>()) {
7900	SmallVector<QualType, 8> GeneralizedParams;
7901	for (auto &Param : FnType->param_types())
7902	GeneralizedParams.push_back(Elt: GeneralizeType(Ctx, Ty: Param));
7903
7904	return Ctx.getFunctionType(
7905	ResultTy: GeneralizeType(Ctx, Ty: FnType->getReturnType()),
7906	Args: GeneralizedParams, EPI: FnType->getExtProtoInfo());
7907	}
7908
7909	if (auto *FnType = Ty->getAs<FunctionNoProtoType>())
7910	return Ctx.getFunctionNoProtoType(
7911	ResultTy: GeneralizeType(Ctx, Ty: FnType->getReturnType()));
7912
7913	llvm_unreachable("Encountered unknown FunctionType");
7914	}
7915
7916	llvm::Metadata *CodeGenModule::CreateMetadataIdentifierGeneralized(QualType T) {
7917	return CreateMetadataIdentifierImpl(T: GeneralizeFunctionType(Ctx&: getContext(), Ty: T),
7918	Map&: GeneralizedMetadataIdMap, Suffix: ".generalized");
7919	}
7920
7921	/// Returns whether this module needs the "all-vtables" type identifier.
7922	bool CodeGenModule::NeedAllVtablesTypeId() const {
7923	// Returns true if at least one of vtable-based CFI checkers is enabled and
7924	// is not in the trapping mode.
7925	return ((LangOpts.Sanitize.has(K: SanitizerKind::CFIVCall) &&
7926	!CodeGenOpts.SanitizeTrap.has(K: SanitizerKind::CFIVCall)) ||
7927	(LangOpts.Sanitize.has(K: SanitizerKind::CFINVCall) &&
7928	!CodeGenOpts.SanitizeTrap.has(K: SanitizerKind::CFINVCall)) ||
7929	(LangOpts.Sanitize.has(K: SanitizerKind::CFIDerivedCast) &&
7930	!CodeGenOpts.SanitizeTrap.has(K: SanitizerKind::CFIDerivedCast)) ||
7931	(LangOpts.Sanitize.has(K: SanitizerKind::CFIUnrelatedCast) &&
7932	!CodeGenOpts.SanitizeTrap.has(K: SanitizerKind::CFIUnrelatedCast)));
7933	}
7934
7935	void CodeGenModule::AddVTableTypeMetadata(llvm::GlobalVariable *VTable,
7936	CharUnits Offset,
7937	const CXXRecordDecl *RD) {
7938	llvm::Metadata *MD =
7939	CreateMetadataIdentifierForType(T: QualType(RD->getTypeForDecl(), 0));
7940	VTable->addTypeMetadata(Offset: Offset.getQuantity(), TypeID: MD);
7941
7942	if (CodeGenOpts.SanitizeCfiCrossDso)
7943	if (auto CrossDsoTypeId = CreateCrossDsoCfiTypeId(MD))
7944	VTable->addTypeMetadata(Offset: Offset.getQuantity(),
7945	TypeID: llvm::ConstantAsMetadata::get(C: CrossDsoTypeId));
7946
7947	if (NeedAllVtablesTypeId()) {
7948	llvm::Metadata *MD = llvm::MDString::get(Context&: getLLVMContext(), Str: "all-vtables");
7949	VTable->addTypeMetadata(Offset: Offset.getQuantity(), TypeID: MD);
7950	}
7951	}
7952
7953	llvm::SanitizerStatReport &CodeGenModule::getSanStats() {
7954	if (!SanStats)
7955	SanStats = std::make_unique<llvm::SanitizerStatReport>(args: &getModule());
7956
7957	return *SanStats;
7958	}
7959
7960	llvm::Value *
7961	CodeGenModule::createOpenCLIntToSamplerConversion(const Expr *E,
7962	CodeGenFunction &CGF) {
7963	llvm::Constant *C = ConstantEmitter(CGF).emitAbstract(E, T: E->getType());
7964	auto *SamplerT = getOpenCLRuntime().getSamplerType(T: E->getType().getTypePtr());
7965	auto *FTy = llvm::FunctionType::get(Result: SamplerT, Params: {C->getType()}, isVarArg: false);
7966	auto *Call = CGF.EmitRuntimeCall(
7967	callee: CreateRuntimeFunction(FTy, Name: "__translate_sampler_initializer"), args: {C});
7968	return Call;
7969	}
7970
7971	CharUnits CodeGenModule::getNaturalPointeeTypeAlignment(
7972	QualType T, LValueBaseInfo *BaseInfo, TBAAAccessInfo *TBAAInfo) {
7973	return getNaturalTypeAlignment(T: T->getPointeeType(), BaseInfo, TBAAInfo,
7974	/* forPointeeType= */ true);
7975	}
7976
7977	CharUnits CodeGenModule::getNaturalTypeAlignment(QualType T,
7978	LValueBaseInfo *BaseInfo,
7979	TBAAAccessInfo *TBAAInfo,
7980	bool forPointeeType) {
7981	if (TBAAInfo)
7982	*TBAAInfo = getTBAAAccessInfo(AccessType: T);
7983
7984	// FIXME: This duplicates logic in ASTContext::getTypeAlignIfKnown. But
7985	// that doesn't return the information we need to compute BaseInfo.
7986
7987	// Honor alignment typedef attributes even on incomplete types.
7988	// We also honor them straight for C++ class types, even as pointees;
7989	// there's an expressivity gap here.
7990	if (auto TT = T->getAs<TypedefType>()) {
7991	if (auto Align = TT->getDecl()->getMaxAlignment()) {
7992	if (BaseInfo)
7993	*BaseInfo = LValueBaseInfo(AlignmentSource::AttributedType);
7994	return getContext().toCharUnitsFromBits(BitSize: Align);
7995	}
7996	}
7997
7998	bool AlignForArray = T->isArrayType();
7999
8000	// Analyze the base element type, so we don't get confused by incomplete
8001	// array types.
8002	T = getContext().getBaseElementType(QT: T);
8003
8004	if (T->isIncompleteType()) {
8005	// We could try to replicate the logic from
8006	// ASTContext::getTypeAlignIfKnown, but nothing uses the alignment if the
8007	// type is incomplete, so it's impossible to test. We could try to reuse
8008	// getTypeAlignIfKnown, but that doesn't return the information we need
8009	// to set BaseInfo. So just ignore the possibility that the alignment is
8010	// greater than one.
8011	if (BaseInfo)
8012	*BaseInfo = LValueBaseInfo(AlignmentSource::Type);
8013	return CharUnits::One();
8014	}
8015
8016	if (BaseInfo)
8017	*BaseInfo = LValueBaseInfo(AlignmentSource::Type);
8018
8019	CharUnits Alignment;
8020	const CXXRecordDecl *RD;
8021	if (T.getQualifiers().hasUnaligned()) {
8022	Alignment = CharUnits::One();
8023	} else if (forPointeeType && !AlignForArray &&
8024	(RD = T->getAsCXXRecordDecl())) {
8025	// For C++ class pointees, we don't know whether we're pointing at a
8026	// base or a complete object, so we generally need to use the
8027	// non-virtual alignment.
8028	Alignment = getClassPointerAlignment(CD: RD);
8029	} else {
8030	Alignment = getContext().getTypeAlignInChars(T);
8031	}
8032
8033	// Cap to the global maximum type alignment unless the alignment
8034	// was somehow explicit on the type.
8035	if (unsigned MaxAlign = getLangOpts().MaxTypeAlign) {
8036	if (Alignment.getQuantity() > MaxAlign &&
8037	!getContext().isAlignmentRequired(T))
8038	Alignment = CharUnits::fromQuantity(Quantity: MaxAlign);
8039	}
8040	return Alignment;
8041	}
8042
8043	bool CodeGenModule::stopAutoInit() {
8044	unsigned StopAfter = getContext().getLangOpts().TrivialAutoVarInitStopAfter;
8045	if (StopAfter) {
8046	// This number is positive only when -ftrivial-auto-var-init-stop-after=* is
8047	// used
8048	if (NumAutoVarInit >= StopAfter) {
8049	return true;
8050	}
8051	if (!NumAutoVarInit) {
8052	unsigned DiagID = getDiags().getCustomDiagID(
8053	L: DiagnosticsEngine::Warning,
8054	FormatString: "-ftrivial-auto-var-init-stop-after=%0 has been enabled to limit the "
8055	"number of times ftrivial-auto-var-init=%1 gets applied.");
8056	getDiags().Report(DiagID)
8057	<< StopAfter
8058	<< (getContext().getLangOpts().getTrivialAutoVarInit() ==
8059	LangOptions::TrivialAutoVarInitKind::Zero
8060	? "zero"
8061	: "pattern");
8062	}
8063	++NumAutoVarInit;
8064	}
8065	return false;
8066	}
8067
8068	void CodeGenModule::printPostfixForExternalizedDecl(llvm::raw_ostream &OS,
8069	const Decl *D) const {
8070	// ptxas does not allow '.' in symbol names. On the other hand, HIP prefers
8071	// postfix beginning with '.' since the symbol name can be demangled.
8072	if (LangOpts.HIP)
8073	OS << (isa<VarDecl>(Val: D) ? ".static." : ".intern.");
8074	else
8075	OS << (isa<VarDecl>(Val: D) ? "__static__" : "__intern__");
8076
8077	// If the CUID is not specified we try to generate a unique postfix.
8078	if (getLangOpts().CUID.empty()) {
8079	SourceManager &SM = getContext().getSourceManager();
8080	PresumedLoc PLoc = SM.getPresumedLoc(Loc: D->getLocation());
8081	assert(PLoc.isValid() && "Source location is expected to be valid.");
8082
8083	// Get the hash of the user defined macros.
8084	llvm::MD5 Hash;
8085	llvm::MD5::MD5Result Result;
8086	for (const auto &Arg : PreprocessorOpts.Macros)
8087	Hash.update(Str: Arg.first);
8088	Hash.final(Result);
8089
8090	// Get the UniqueID for the file containing the decl.
8091	llvm::sys::fs::UniqueID ID;
8092	if (llvm::sys::fs::getUniqueID(Path: PLoc.getFilename(), Result&: ID)) {
8093	PLoc = SM.getPresumedLoc(Loc: D->getLocation(), /*UseLineDirectives=*/false);
8094	assert(PLoc.isValid() && "Source location is expected to be valid.");
8095	if (auto EC = llvm::sys::fs::getUniqueID(Path: PLoc.getFilename(), Result&: ID))
8096	SM.getDiagnostics().Report(DiagID: diag::err_cannot_open_file)
8097	<< PLoc.getFilename() << EC.message();
8098	}
8099	OS << llvm::format(Fmt: "%x", Vals: ID.getFile()) << llvm::format(Fmt: "%x", Vals: ID.getDevice())
8100	<< "_" << llvm::utohexstr(X: Result.low(), /*LowerCase=*/true, /*Width=*/8);
8101	} else {
8102	OS << getContext().getCUIDHash();
8103	}
8104	}
8105
8106	void CodeGenModule::moveLazyEmissionStates(CodeGenModule *NewBuilder) {
8107	assert(DeferredDeclsToEmit.empty() &&
8108	"Should have emitted all decls deferred to emit.");
8109	assert(NewBuilder->DeferredDecls.empty() &&
8110	"Newly created module should not have deferred decls");
8111	NewBuilder->DeferredDecls = std::move(DeferredDecls);
8112	assert(EmittedDeferredDecls.empty() &&
8113	"Still have (unmerged) EmittedDeferredDecls deferred decls");
8114
8115	assert(NewBuilder->DeferredVTables.empty() &&
8116	"Newly created module should not have deferred vtables");
8117	NewBuilder->DeferredVTables = std::move(DeferredVTables);
8118
8119	assert(NewBuilder->MangledDeclNames.empty() &&
8120	"Newly created module should not have mangled decl names");
8121	assert(NewBuilder->Manglings.empty() &&
8122	"Newly created module should not have manglings");
8123	NewBuilder->Manglings = std::move(Manglings);
8124
8125	NewBuilder->WeakRefReferences = std::move(WeakRefReferences);
8126
8127	NewBuilder->ABI->MangleCtx = std::move(ABI->MangleCtx);
8128	}
8129