1	//===--- CodeGenModule.cpp - Emit LLVM Code from ASTs for a Module --------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This coordinates the per-module state used while generating code.
10	//
11	//===----------------------------------------------------------------------===//
12
13	#include "CodeGenModule.h"
14	#include "ABIInfo.h"
15	#include "CGBlocks.h"
16	#include "CGCUDARuntime.h"
17	#include "CGCXXABI.h"
18	#include "CGCall.h"
19	#include "CGDebugInfo.h"
20	#include "CGHLSLRuntime.h"
21	#include "CGObjCRuntime.h"
22	#include "CGOpenCLRuntime.h"
23	#include "CGOpenMPRuntime.h"
24	#include "CGOpenMPRuntimeGPU.h"
25	#include "CodeGenFunction.h"
26	#include "CodeGenPGO.h"
27	#include "ConstantEmitter.h"
28	#include "CoverageMappingGen.h"
29	#include "TargetInfo.h"
30	#include "clang/AST/ASTContext.h"
31	#include "clang/AST/ASTLambda.h"
32	#include "clang/AST/CharUnits.h"
33	#include "clang/AST/Decl.h"
34	#include "clang/AST/DeclCXX.h"
35	#include "clang/AST/DeclObjC.h"
36	#include "clang/AST/DeclTemplate.h"
37	#include "clang/AST/Mangle.h"
38	#include "clang/AST/RecursiveASTVisitor.h"
39	#include "clang/AST/StmtVisitor.h"
40	#include "clang/Basic/Builtins.h"
41	#include "clang/Basic/CharInfo.h"
42	#include "clang/Basic/CodeGenOptions.h"
43	#include "clang/Basic/Diagnostic.h"
44	#include "clang/Basic/FileManager.h"
45	#include "clang/Basic/Module.h"
46	#include "clang/Basic/SourceManager.h"
47	#include "clang/Basic/TargetInfo.h"
48	#include "clang/Basic/Version.h"
49	#include "clang/CodeGen/BackendUtil.h"
50	#include "clang/CodeGen/ConstantInitBuilder.h"
51	#include "clang/Frontend/FrontendDiagnostic.h"
52	#include "llvm/ADT/STLExtras.h"
53	#include "llvm/ADT/StringExtras.h"
54	#include "llvm/ADT/StringSwitch.h"
55	#include "llvm/Analysis/TargetLibraryInfo.h"
56	#include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
57	#include "llvm/IR/AttributeMask.h"
58	#include "llvm/IR/CallingConv.h"
59	#include "llvm/IR/DataLayout.h"
60	#include "llvm/IR/Intrinsics.h"
61	#include "llvm/IR/LLVMContext.h"
62	#include "llvm/IR/Module.h"
63	#include "llvm/IR/ProfileSummary.h"
64	#include "llvm/ProfileData/InstrProfReader.h"
65	#include "llvm/ProfileData/SampleProf.h"
66	#include "llvm/Support/CRC.h"
67	#include "llvm/Support/CodeGen.h"
68	#include "llvm/Support/CommandLine.h"
69	#include "llvm/Support/ConvertUTF.h"
70	#include "llvm/Support/ErrorHandling.h"
71	#include "llvm/Support/TimeProfiler.h"
72	#include "llvm/Support/xxhash.h"
73	#include "llvm/TargetParser/RISCVISAInfo.h"
74	#include "llvm/TargetParser/Triple.h"
75	#include "llvm/TargetParser/X86TargetParser.h"
76	#include <optional>
77
78	using namespace clang;
79	using namespace CodeGen;
80
81	static llvm::cl::opt<bool> LimitedCoverage(
82	"limited-coverage-experimental", llvm::cl::Hidden,
83	llvm::cl::desc("Emit limited coverage mapping information (experimental)"));
84
85	static const char AnnotationSection[] = "llvm.metadata";
86
87	static CGCXXABI *createCXXABI(CodeGenModule &CGM) {
88	switch (CGM.getContext().getCXXABIKind()) {
89	case TargetCXXABI::AppleARM64:
90	case TargetCXXABI::Fuchsia:
91	case TargetCXXABI::GenericAArch64:
92	case TargetCXXABI::GenericARM:
93	case TargetCXXABI::iOS:
94	case TargetCXXABI::WatchOS:
95	case TargetCXXABI::GenericMIPS:
96	case TargetCXXABI::GenericItanium:
97	case TargetCXXABI::WebAssembly:
98	case TargetCXXABI::XL:
99	return CreateItaniumCXXABI(CGM);
100	case TargetCXXABI::Microsoft:
101	return CreateMicrosoftCXXABI(CGM);
102	}
103
104	llvm_unreachable("invalid C++ ABI kind");
105	}
106
107	static std::unique_ptr<TargetCodeGenInfo>
108	createTargetCodeGenInfo(CodeGenModule &CGM) {
109	const TargetInfo &Target = CGM.getTarget();
110	const llvm::Triple &Triple = Target.getTriple();
111	const CodeGenOptions &CodeGenOpts = CGM.getCodeGenOpts();
112
113	switch (Triple.getArch()) {
114	default:
115	return createDefaultTargetCodeGenInfo(CGM);
116
117	case llvm::Triple::le32:
118	return createPNaClTargetCodeGenInfo(CGM);
119	case llvm::Triple::m68k:
120	return createM68kTargetCodeGenInfo(CGM);
121	case llvm::Triple::mips:
122	case llvm::Triple::mipsel:
123	if (Triple.getOS() == llvm::Triple::NaCl)
124	return createPNaClTargetCodeGenInfo(CGM);
125	return createMIPSTargetCodeGenInfo(CGM, /*IsOS32=*/true);
126
127	case llvm::Triple::mips64:
128	case llvm::Triple::mips64el:
129	return createMIPSTargetCodeGenInfo(CGM, /*IsOS32=*/false);
130
131	case llvm::Triple::avr: {
132	// For passing parameters, R8~R25 are used on avr, and R18~R25 are used
133	// on avrtiny. For passing return value, R18~R25 are used on avr, and
134	// R22~R25 are used on avrtiny.
135	unsigned NPR = Target.getABI() == "avrtiny" ? 6 : 18;
136	unsigned NRR = Target.getABI() == "avrtiny" ? 4 : 8;
137	return createAVRTargetCodeGenInfo(CGM, NPR, NRR);
138	}
139
140	case llvm::Triple::aarch64:
141	case llvm::Triple::aarch64_32:
142	case llvm::Triple::aarch64_be: {
143	AArch64ABIKind Kind = AArch64ABIKind::AAPCS;
144	if (Target.getABI() == "darwinpcs")
145	Kind = AArch64ABIKind::DarwinPCS;
146	else if (Triple.isOSWindows())
147	return createWindowsAArch64TargetCodeGenInfo(CGM, K: AArch64ABIKind::Win64);
148	else if (Target.getABI() == "aapcs-soft")
149	Kind = AArch64ABIKind::AAPCSSoft;
150
151	return createAArch64TargetCodeGenInfo(CGM, Kind);
152	}
153
154	case llvm::Triple::wasm32:
155	case llvm::Triple::wasm64: {
156	WebAssemblyABIKind Kind = WebAssemblyABIKind::MVP;
157	if (Target.getABI() == "experimental-mv")
158	Kind = WebAssemblyABIKind::ExperimentalMV;
159	return createWebAssemblyTargetCodeGenInfo(CGM, K: Kind);
160	}
161
162	case llvm::Triple::arm:
163	case llvm::Triple::armeb:
164	case llvm::Triple::thumb:
165	case llvm::Triple::thumbeb: {
166	if (Triple.getOS() == llvm::Triple::Win32)
167	return createWindowsARMTargetCodeGenInfo(CGM, K: ARMABIKind::AAPCS_VFP);
168
169	ARMABIKind Kind = ARMABIKind::AAPCS;
170	StringRef ABIStr = Target.getABI();
171	if (ABIStr == "apcs-gnu")
172	Kind = ARMABIKind::APCS;
173	else if (ABIStr == "aapcs16")
174	Kind = ARMABIKind::AAPCS16_VFP;
175	else if (CodeGenOpts.FloatABI == "hard" ||
176	(CodeGenOpts.FloatABI != "soft" &&
177	(Triple.getEnvironment() == llvm::Triple::GNUEABIHF ||
178	Triple.getEnvironment() == llvm::Triple::MuslEABIHF ||
179	Triple.getEnvironment() == llvm::Triple::EABIHF)))
180	Kind = ARMABIKind::AAPCS_VFP;
181
182	return createARMTargetCodeGenInfo(CGM, Kind);
183	}
184
185	case llvm::Triple::ppc: {
186	if (Triple.isOSAIX())
187	return createAIXTargetCodeGenInfo(CGM, /*Is64Bit=*/false);
188
189	bool IsSoftFloat =
190	CodeGenOpts.FloatABI == "soft" || Target.hasFeature(Feature: "spe");
191	return createPPC32TargetCodeGenInfo(CGM, SoftFloatABI: IsSoftFloat);
192	}
193	case llvm::Triple::ppcle: {
194	bool IsSoftFloat = CodeGenOpts.FloatABI == "soft";
195	return createPPC32TargetCodeGenInfo(CGM, SoftFloatABI: IsSoftFloat);
196	}
197	case llvm::Triple::ppc64:
198	if (Triple.isOSAIX())
199	return createAIXTargetCodeGenInfo(CGM, /*Is64Bit=*/true);
200
201	if (Triple.isOSBinFormatELF()) {
202	PPC64_SVR4_ABIKind Kind = PPC64_SVR4_ABIKind::ELFv1;
203	if (Target.getABI() == "elfv2")
204	Kind = PPC64_SVR4_ABIKind::ELFv2;
205	bool IsSoftFloat = CodeGenOpts.FloatABI == "soft";
206
207	return createPPC64_SVR4_TargetCodeGenInfo(CGM, Kind, SoftFloatABI: IsSoftFloat);
208	}
209	return createPPC64TargetCodeGenInfo(CGM);
210	case llvm::Triple::ppc64le: {
211	assert(Triple.isOSBinFormatELF() && "PPC64 LE non-ELF not supported!");
212	PPC64_SVR4_ABIKind Kind = PPC64_SVR4_ABIKind::ELFv2;
213	if (Target.getABI() == "elfv1")
214	Kind = PPC64_SVR4_ABIKind::ELFv1;
215	bool IsSoftFloat = CodeGenOpts.FloatABI == "soft";
216
217	return createPPC64_SVR4_TargetCodeGenInfo(CGM, Kind, SoftFloatABI: IsSoftFloat);
218	}
219
220	case llvm::Triple::nvptx:
221	case llvm::Triple::nvptx64:
222	return createNVPTXTargetCodeGenInfo(CGM);
223
224	case llvm::Triple::msp430:
225	return createMSP430TargetCodeGenInfo(CGM);
226
227	case llvm::Triple::riscv32:
228	case llvm::Triple::riscv64: {
229	StringRef ABIStr = Target.getABI();
230	unsigned XLen = Target.getPointerWidth(AddrSpace: LangAS::Default);
231	unsigned ABIFLen = 0;
232	if (ABIStr.ends_with(Suffix: "f"))
233	ABIFLen = 32;
234	else if (ABIStr.ends_with(Suffix: "d"))
235	ABIFLen = 64;
236	bool EABI = ABIStr.ends_with(Suffix: "e");
237	return createRISCVTargetCodeGenInfo(CGM, XLen, FLen: ABIFLen, EABI);
238	}
239
240	case llvm::Triple::systemz: {
241	bool SoftFloat = CodeGenOpts.FloatABI == "soft";
242	bool HasVector = !SoftFloat && Target.getABI() == "vector";
243	return createSystemZTargetCodeGenInfo(CGM, HasVector, SoftFloatABI: SoftFloat);
244	}
245
246	case llvm::Triple::tce:
247	case llvm::Triple::tcele:
248	return createTCETargetCodeGenInfo(CGM);
249
250	case llvm::Triple::x86: {
251	bool IsDarwinVectorABI = Triple.isOSDarwin();
252	bool IsWin32FloatStructABI = Triple.isOSWindows() && !Triple.isOSCygMing();
253
254	if (Triple.getOS() == llvm::Triple::Win32) {
255	return createWinX86_32TargetCodeGenInfo(
256	CGM, DarwinVectorABI: IsDarwinVectorABI, Win32StructABI: IsWin32FloatStructABI,
257	NumRegisterParameters: CodeGenOpts.NumRegisterParameters);
258	}
259	return createX86_32TargetCodeGenInfo(
260	CGM, DarwinVectorABI: IsDarwinVectorABI, Win32StructABI: IsWin32FloatStructABI,
261	NumRegisterParameters: CodeGenOpts.NumRegisterParameters, SoftFloatABI: CodeGenOpts.FloatABI == "soft");
262	}
263
264	case llvm::Triple::x86_64: {
265	StringRef ABI = Target.getABI();
266	X86AVXABILevel AVXLevel = (ABI == "avx512" ? X86AVXABILevel::AVX512
267	: ABI == "avx" ? X86AVXABILevel::AVX
268	: X86AVXABILevel::None);
269
270	switch (Triple.getOS()) {
271	case llvm::Triple::Win32:
272	return createWinX86_64TargetCodeGenInfo(CGM, AVXLevel);
273	default:
274	return createX86_64TargetCodeGenInfo(CGM, AVXLevel);
275	}
276	}
277	case llvm::Triple::hexagon:
278	return createHexagonTargetCodeGenInfo(CGM);
279	case llvm::Triple::lanai:
280	return createLanaiTargetCodeGenInfo(CGM);
281	case llvm::Triple::r600:
282	return createAMDGPUTargetCodeGenInfo(CGM);
283	case llvm::Triple::amdgcn:
284	return createAMDGPUTargetCodeGenInfo(CGM);
285	case llvm::Triple::sparc:
286	return createSparcV8TargetCodeGenInfo(CGM);
287	case llvm::Triple::sparcv9:
288	return createSparcV9TargetCodeGenInfo(CGM);
289	case llvm::Triple::xcore:
290	return createXCoreTargetCodeGenInfo(CGM);
291	case llvm::Triple::arc:
292	return createARCTargetCodeGenInfo(CGM);
293	case llvm::Triple::spir:
294	case llvm::Triple::spir64:
295	return createCommonSPIRTargetCodeGenInfo(CGM);
296	case llvm::Triple::spirv32:
297	case llvm::Triple::spirv64:
298	return createSPIRVTargetCodeGenInfo(CGM);
299	case llvm::Triple::ve:
300	return createVETargetCodeGenInfo(CGM);
301	case llvm::Triple::csky: {
302	bool IsSoftFloat = !Target.hasFeature(Feature: "hard-float-abi");
303	bool hasFP64 =
304	Target.hasFeature(Feature: "fpuv2_df") || Target.hasFeature(Feature: "fpuv3_df");
305	return createCSKYTargetCodeGenInfo(CGM, FLen: IsSoftFloat ? 0
306	: hasFP64 ? 64
307	: 32);
308	}
309	case llvm::Triple::bpfeb:
310	case llvm::Triple::bpfel:
311	return createBPFTargetCodeGenInfo(CGM);
312	case llvm::Triple::loongarch32:
313	case llvm::Triple::loongarch64: {
314	StringRef ABIStr = Target.getABI();
315	unsigned ABIFRLen = 0;
316	if (ABIStr.ends_with(Suffix: "f"))
317	ABIFRLen = 32;
318	else if (ABIStr.ends_with(Suffix: "d"))
319	ABIFRLen = 64;
320	return createLoongArchTargetCodeGenInfo(
321	CGM, GRLen: Target.getPointerWidth(AddrSpace: LangAS::Default), FLen: ABIFRLen);
322	}
323	}
324	}
325
326	const TargetCodeGenInfo &CodeGenModule::getTargetCodeGenInfo() {
327	if (!TheTargetCodeGenInfo)
328	TheTargetCodeGenInfo = createTargetCodeGenInfo(CGM&: *this);
329	return *TheTargetCodeGenInfo;
330	}
331
332	CodeGenModule::CodeGenModule(ASTContext &C,
333	IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS,
334	const HeaderSearchOptions &HSO,
335	const PreprocessorOptions &PPO,
336	const CodeGenOptions &CGO, llvm::Module &M,
337	DiagnosticsEngine &diags,
338	CoverageSourceInfo *CoverageInfo)
339	: Context(C), LangOpts(C.getLangOpts()), FS(FS), HeaderSearchOpts(HSO),
340	PreprocessorOpts(PPO), CodeGenOpts(CGO), TheModule(M), Diags(diags),
341	Target(C.getTargetInfo()), ABI(createCXXABI(CGM&: *this)),
342	VMContext(M.getContext()), Types(*this), VTables(*this),
343	SanitizerMD(new SanitizerMetadata(*this)) {
344
345	// Initialize the type cache.
346	llvm::LLVMContext &LLVMContext = M.getContext();
347	VoidTy = llvm::Type::getVoidTy(C&: LLVMContext);
348	Int8Ty = llvm::Type::getInt8Ty(C&: LLVMContext);
349	Int16Ty = llvm::Type::getInt16Ty(C&: LLVMContext);
350	Int32Ty = llvm::Type::getInt32Ty(C&: LLVMContext);
351	Int64Ty = llvm::Type::getInt64Ty(C&: LLVMContext);
352	HalfTy = llvm::Type::getHalfTy(C&: LLVMContext);
353	BFloatTy = llvm::Type::getBFloatTy(C&: LLVMContext);
354	FloatTy = llvm::Type::getFloatTy(C&: LLVMContext);
355	DoubleTy = llvm::Type::getDoubleTy(C&: LLVMContext);
356	PointerWidthInBits = C.getTargetInfo().getPointerWidth(AddrSpace: LangAS::Default);
357	PointerAlignInBytes =
358	C.toCharUnitsFromBits(BitSize: C.getTargetInfo().getPointerAlign(AddrSpace: LangAS::Default))
359	.getQuantity();
360	SizeSizeInBytes =
361	C.toCharUnitsFromBits(BitSize: C.getTargetInfo().getMaxPointerWidth()).getQuantity();
362	IntAlignInBytes =
363	C.toCharUnitsFromBits(BitSize: C.getTargetInfo().getIntAlign()).getQuantity();
364	CharTy =
365	llvm::IntegerType::get(C&: LLVMContext, NumBits: C.getTargetInfo().getCharWidth());
366	IntTy = llvm::IntegerType::get(C&: LLVMContext, NumBits: C.getTargetInfo().getIntWidth());
367	IntPtrTy = llvm::IntegerType::get(C&: LLVMContext,
368	NumBits: C.getTargetInfo().getMaxPointerWidth());
369	Int8PtrTy = llvm::PointerType::get(C&: LLVMContext, AddressSpace: 0);
370	const llvm::DataLayout &DL = M.getDataLayout();
371	AllocaInt8PtrTy =
372	llvm::PointerType::get(C&: LLVMContext, AddressSpace: DL.getAllocaAddrSpace());
373	GlobalsInt8PtrTy =
374	llvm::PointerType::get(C&: LLVMContext, AddressSpace: DL.getDefaultGlobalsAddressSpace());
375	ConstGlobalsPtrTy = llvm::PointerType::get(
376	C&: LLVMContext, AddressSpace: C.getTargetAddressSpace(AS: GetGlobalConstantAddressSpace()));
377	ASTAllocaAddressSpace = getTargetCodeGenInfo().getASTAllocaAddressSpace();
378
379	// Build C++20 Module initializers.
380	// TODO: Add Microsoft here once we know the mangling required for the
381	// initializers.
382	CXX20ModuleInits =
383	LangOpts.CPlusPlusModules && getCXXABI().getMangleContext().getKind() ==
384	ItaniumMangleContext::MK_Itanium;
385
386	RuntimeCC = getTargetCodeGenInfo().getABIInfo().getRuntimeCC();
387
388	if (LangOpts.ObjC)
389	createObjCRuntime();
390	if (LangOpts.OpenCL)
391	createOpenCLRuntime();
392	if (LangOpts.OpenMP)
393	createOpenMPRuntime();
394	if (LangOpts.CUDA)
395	createCUDARuntime();
396	if (LangOpts.HLSL)
397	createHLSLRuntime();
398
399	// Enable TBAA unless it's suppressed. ThreadSanitizer needs TBAA even at O0.
400	if (LangOpts.Sanitize.has(K: SanitizerKind::Thread) ||
401	(!CodeGenOpts.RelaxedAliasing && CodeGenOpts.OptimizationLevel > 0))
402	TBAA.reset(p: new CodeGenTBAA(Context, getTypes(), TheModule, CodeGenOpts,
403	getLangOpts(), getCXXABI().getMangleContext()));
404
405	// If debug info or coverage generation is enabled, create the CGDebugInfo
406	// object.
407	if (CodeGenOpts.getDebugInfo() != llvm::codegenoptions::NoDebugInfo ||
408	CodeGenOpts.CoverageNotesFile.size() ||
409	CodeGenOpts.CoverageDataFile.size())
410	DebugInfo.reset(p: new CGDebugInfo(*this));
411
412	Block.GlobalUniqueCount = 0;
413
414	if (C.getLangOpts().ObjC)
415	ObjCData.reset(p: new ObjCEntrypoints());
416
417	if (CodeGenOpts.hasProfileClangUse()) {
418	auto ReaderOrErr = llvm::IndexedInstrProfReader::create(
419	Path: CodeGenOpts.ProfileInstrumentUsePath, FS&: *FS,
420	RemappingPath: CodeGenOpts.ProfileRemappingFile);
421	// We're checking for profile read errors in CompilerInvocation, so if
422	// there was an error it should've already been caught. If it hasn't been
423	// somehow, trip an assertion.
424	assert(ReaderOrErr);
425	PGOReader = std::move(ReaderOrErr.get());
426	}
427
428	// If coverage mapping generation is enabled, create the
429	// CoverageMappingModuleGen object.
430	if (CodeGenOpts.CoverageMapping)
431	CoverageMapping.reset(p: new CoverageMappingModuleGen(*this, *CoverageInfo));
432
433	// Generate the module name hash here if needed.
434	if (CodeGenOpts.UniqueInternalLinkageNames &&
435	!getModule().getSourceFileName().empty()) {
436	std::string Path = getModule().getSourceFileName();
437	// Check if a path substitution is needed from the MacroPrefixMap.
438	for (const auto &Entry : LangOpts.MacroPrefixMap)
439	if (Path.rfind(str: Entry.first, pos: 0) != std::string::npos) {
440	Path = Entry.second + Path.substr(pos: Entry.first.size());
441	break;
442	}
443	ModuleNameHash = llvm::getUniqueInternalLinkagePostfix(FName: Path);
444	}
445	}
446
447	CodeGenModule::~CodeGenModule() {}
448
449	void CodeGenModule::createObjCRuntime() {
450	// This is just isGNUFamily(), but we want to force implementors of
451	// new ABIs to decide how best to do this.
452	switch (LangOpts.ObjCRuntime.getKind()) {
453	case ObjCRuntime::GNUstep:
454	case ObjCRuntime::GCC:
455	case ObjCRuntime::ObjFW:
456	ObjCRuntime.reset(p: CreateGNUObjCRuntime(CGM&: *this));
457	return;
458
459	case ObjCRuntime::FragileMacOSX:
460	case ObjCRuntime::MacOSX:
461	case ObjCRuntime::iOS:
462	case ObjCRuntime::WatchOS:
463	ObjCRuntime.reset(p: CreateMacObjCRuntime(CGM&: *this));
464	return;
465	}
466	llvm_unreachable("bad runtime kind");
467	}
468
469	void CodeGenModule::createOpenCLRuntime() {
470	OpenCLRuntime.reset(p: new CGOpenCLRuntime(*this));
471	}
472
473	void CodeGenModule::createOpenMPRuntime() {
474	// Select a specialized code generation class based on the target, if any.
475	// If it does not exist use the default implementation.
476	switch (getTriple().getArch()) {
477	case llvm::Triple::nvptx:
478	case llvm::Triple::nvptx64:
479	case llvm::Triple::amdgcn:
480	assert(getLangOpts().OpenMPIsTargetDevice &&
481	"OpenMP AMDGPU/NVPTX is only prepared to deal with device code.");
482	OpenMPRuntime.reset(new CGOpenMPRuntimeGPU(*this));
483	break;
484	default:
485	if (LangOpts.OpenMPSimd)
486	OpenMPRuntime.reset(new CGOpenMPSIMDRuntime(*this));
487	else
488	OpenMPRuntime.reset(p: new CGOpenMPRuntime(*this));
489	break;
490	}
491	}
492
493	void CodeGenModule::createCUDARuntime() {
494	CUDARuntime.reset(p: CreateNVCUDARuntime(CGM&: *this));
495	}
496
497	void CodeGenModule::createHLSLRuntime() {
498	HLSLRuntime.reset(p: new CGHLSLRuntime(*this));
499	}
500
501	void CodeGenModule::addReplacement(StringRef Name, llvm::Constant *C) {
502	Replacements[Name] = C;
503	}
504
505	void CodeGenModule::applyReplacements() {
506	for (auto &I : Replacements) {
507	StringRef MangledName = I.first;
508	llvm::Constant *Replacement = I.second;
509	llvm::GlobalValue *Entry = GetGlobalValue(Ref: MangledName);
510	if (!Entry)
511	continue;
512	auto *OldF = cast<llvm::Function>(Val: Entry);
513	auto *NewF = dyn_cast<llvm::Function>(Val: Replacement);
514	if (!NewF) {
515	if (auto *Alias = dyn_cast<llvm::GlobalAlias>(Val: Replacement)) {
516	NewF = dyn_cast<llvm::Function>(Val: Alias->getAliasee());
517	} else {
518	auto *CE = cast<llvm::ConstantExpr>(Val: Replacement);
519	assert(CE->getOpcode() == llvm::Instruction::BitCast ||
520	CE->getOpcode() == llvm::Instruction::GetElementPtr);
521	NewF = dyn_cast<llvm::Function>(Val: CE->getOperand(i_nocapture: 0));
522	}
523	}
524
525	// Replace old with new, but keep the old order.
526	OldF->replaceAllUsesWith(V: Replacement);
527	if (NewF) {
528	NewF->removeFromParent();
529	OldF->getParent()->getFunctionList().insertAfter(where: OldF->getIterator(),
530	New: NewF);
531	}
532	OldF->eraseFromParent();
533	}
534	}
535
536	void CodeGenModule::addGlobalValReplacement(llvm::GlobalValue *GV, llvm::Constant *C) {
537	GlobalValReplacements.push_back(Elt: std::make_pair(x&: GV, y&: C));
538	}
539
540	void CodeGenModule::applyGlobalValReplacements() {
541	for (auto &I : GlobalValReplacements) {
542	llvm::GlobalValue *GV = I.first;
543	llvm::Constant *C = I.second;
544
545	GV->replaceAllUsesWith(V: C);
546	GV->eraseFromParent();
547	}
548	}
549
550	// This is only used in aliases that we created and we know they have a
551	// linear structure.
552	static const llvm::GlobalValue *getAliasedGlobal(const llvm::GlobalValue *GV) {
553	const llvm::Constant *C;
554	if (auto *GA = dyn_cast<llvm::GlobalAlias>(Val: GV))
555	C = GA->getAliasee();
556	else if (auto *GI = dyn_cast<llvm::GlobalIFunc>(Val: GV))
557	C = GI->getResolver();
558	else
559	return GV;
560
561	const auto *AliaseeGV = dyn_cast<llvm::GlobalValue>(Val: C->stripPointerCasts());
562	if (!AliaseeGV)
563	return nullptr;
564
565	const llvm::GlobalValue *FinalGV = AliaseeGV->getAliaseeObject();
566	if (FinalGV == GV)
567	return nullptr;
568
569	return FinalGV;
570	}
571
572	static bool checkAliasedGlobal(
573	const ASTContext &Context, DiagnosticsEngine &Diags, SourceLocation Location,
574	bool IsIFunc, const llvm::GlobalValue *Alias, const llvm::GlobalValue *&GV,
575	const llvm::MapVector<GlobalDecl, StringRef> &MangledDeclNames,
576	SourceRange AliasRange) {
577	GV = getAliasedGlobal(GV: Alias);
578	if (!GV) {
579	Diags.Report(Location, diag::err_cyclic_alias) << IsIFunc;
580	return false;
581	}
582
583	if (GV->hasCommonLinkage()) {
584	const llvm::Triple &Triple = Context.getTargetInfo().getTriple();
585	if (Triple.getObjectFormat() == llvm::Triple::XCOFF) {
586	Diags.Report(Location, diag::err_alias_to_common);
587	return false;
588	}
589	}
590
591	if (GV->isDeclaration()) {
592	Diags.Report(Location, diag::err_alias_to_undefined) << IsIFunc << IsIFunc;
593	Diags.Report(Location, diag::note_alias_requires_mangled_name)
594	<< IsIFunc << IsIFunc;
595	// Provide a note if the given function is not found and exists as a
596	// mangled name.
597	for (const auto &[Decl, Name] : MangledDeclNames) {
598	if (const auto *ND = dyn_cast<NamedDecl>(Val: Decl.getDecl())) {
599	if (ND->getName() == GV->getName()) {
600	Diags.Report(Location, diag::note_alias_mangled_name_alternative)
601	<< Name
602	<< FixItHint::CreateReplacement(
603	AliasRange,
604	(Twine(IsIFunc ? "ifunc" : "alias") + "(\"" + Name + "\")")
605	.str());
606	}
607	}
608	}
609	return false;
610	}
611
612	if (IsIFunc) {
613	// Check resolver function type.
614	const auto *F = dyn_cast<llvm::Function>(Val: GV);
615	if (!F) {
616	Diags.Report(Location, diag::err_alias_to_undefined)
617	<< IsIFunc << IsIFunc;
618	return false;
619	}
620
621	llvm::FunctionType *FTy = F->getFunctionType();
622	if (!FTy->getReturnType()->isPointerTy()) {
623	Diags.Report(Location, diag::err_ifunc_resolver_return);
624	return false;
625	}
626	}
627
628	return true;
629	}
630
631	// Emit a warning if toc-data attribute is requested for global variables that
632	// have aliases and remove the toc-data attribute.
633	static void checkAliasForTocData(llvm::GlobalVariable *GVar,
634	const CodeGenOptions &CodeGenOpts,
635	DiagnosticsEngine &Diags,
636	SourceLocation Location) {
637	if (GVar->hasAttribute(Kind: "toc-data")) {
638	auto GVId = GVar->getName();
639	// Is this a global variable specified by the user as local?
640	if ((llvm::binary_search(Range: CodeGenOpts.TocDataVarsUserSpecified, Value&: GVId))) {
641	Diags.Report(Location, diag::warn_toc_unsupported_type)
642	<< GVId << "the variable has an alias";
643	}
644	llvm::AttributeSet CurrAttributes = GVar->getAttributes();
645	llvm::AttributeSet NewAttributes =
646	CurrAttributes.removeAttribute(C&: GVar->getContext(), Kind: "toc-data");
647	GVar->setAttributes(NewAttributes);
648	}
649	}
650
651	void CodeGenModule::checkAliases() {
652	// Check if the constructed aliases are well formed. It is really unfortunate
653	// that we have to do this in CodeGen, but we only construct mangled names
654	// and aliases during codegen.
655	bool Error = false;
656	DiagnosticsEngine &Diags = getDiags();
657	for (const GlobalDecl &GD : Aliases) {
658	const auto *D = cast<ValueDecl>(Val: GD.getDecl());
659	SourceLocation Location;
660	SourceRange Range;
661	bool IsIFunc = D->hasAttr<IFuncAttr>();
662	if (const Attr *A = D->getDefiningAttr()) {
663	Location = A->getLocation();
664	Range = A->getRange();
665	} else
666	llvm_unreachable("Not an alias or ifunc?");
667
668	StringRef MangledName = getMangledName(GD);
669	llvm::GlobalValue *Alias = GetGlobalValue(Ref: MangledName);
670	const llvm::GlobalValue *GV = nullptr;
671	if (!checkAliasedGlobal(Context: getContext(), Diags, Location, IsIFunc, Alias, GV,
672	MangledDeclNames, AliasRange: Range)) {
673	Error = true;
674	continue;
675	}
676
677	if (getContext().getTargetInfo().getTriple().isOSAIX())
678	if (const llvm::GlobalVariable *GVar =
679	dyn_cast<const llvm::GlobalVariable>(Val: GV))
680	checkAliasForTocData(GVar: const_cast<llvm::GlobalVariable *>(GVar),
681	CodeGenOpts: getCodeGenOpts(), Diags, Location);
682
683	llvm::Constant *Aliasee =
684	IsIFunc ? cast<llvm::GlobalIFunc>(Val: Alias)->getResolver()
685	: cast<llvm::GlobalAlias>(Val: Alias)->getAliasee();
686
687	llvm::GlobalValue *AliaseeGV;
688	if (auto CE = dyn_cast<llvm::ConstantExpr>(Val: Aliasee))
689	AliaseeGV = cast<llvm::GlobalValue>(Val: CE->getOperand(i_nocapture: 0));
690	else
691	AliaseeGV = cast<llvm::GlobalValue>(Val: Aliasee);
692
693	if (const SectionAttr *SA = D->getAttr<SectionAttr>()) {
694	StringRef AliasSection = SA->getName();
695	if (AliasSection != AliaseeGV->getSection())
696	Diags.Report(SA->getLocation(), diag::warn_alias_with_section)
697	<< AliasSection << IsIFunc << IsIFunc;
698	}
699
700	// We have to handle alias to weak aliases in here. LLVM itself disallows
701	// this since the object semantics would not match the IL one. For
702	// compatibility with gcc we implement it by just pointing the alias
703	// to its aliasee's aliasee. We also warn, since the user is probably
704	// expecting the link to be weak.
705	if (auto *GA = dyn_cast<llvm::GlobalAlias>(Val: AliaseeGV)) {
706	if (GA->isInterposable()) {
707	Diags.Report(Location, diag::warn_alias_to_weak_alias)
708	<< GV->getName() << GA->getName() << IsIFunc;
709	Aliasee = llvm::ConstantExpr::getPointerBitCastOrAddrSpaceCast(
710	C: GA->getAliasee(), Ty: Alias->getType());
711
712	if (IsIFunc)
713	cast<llvm::GlobalIFunc>(Val: Alias)->setResolver(Aliasee);
714	else
715	cast<llvm::GlobalAlias>(Val: Alias)->setAliasee(Aliasee);
716	}
717	}
718	}
719	if (!Error)
720	return;
721
722	for (const GlobalDecl &GD : Aliases) {
723	StringRef MangledName = getMangledName(GD);
724	llvm::GlobalValue *Alias = GetGlobalValue(Ref: MangledName);
725	Alias->replaceAllUsesWith(V: llvm::UndefValue::get(T: Alias->getType()));
726	Alias->eraseFromParent();
727	}
728	}
729
730	void CodeGenModule::clear() {
731	DeferredDeclsToEmit.clear();
732	EmittedDeferredDecls.clear();
733	DeferredAnnotations.clear();
734	if (OpenMPRuntime)
735	OpenMPRuntime->clear();
736	}
737
738	void InstrProfStats::reportDiagnostics(DiagnosticsEngine &Diags,
739	StringRef MainFile) {
740	if (!hasDiagnostics())
741	return;
742	if (VisitedInMainFile > 0 && VisitedInMainFile == MissingInMainFile) {
743	if (MainFile.empty())
744	MainFile = "<stdin>";
745	Diags.Report(diag::warn_profile_data_unprofiled) << MainFile;
746	} else {
747	if (Mismatched > 0)
748	Diags.Report(diag::warn_profile_data_out_of_date) << Visited << Mismatched;
749
750	if (Missing > 0)
751	Diags.Report(diag::warn_profile_data_missing) << Visited << Missing;
752	}
753	}
754
755	static std::optional<llvm::GlobalValue::VisibilityTypes>
756	getLLVMVisibility(clang::LangOptions::VisibilityFromDLLStorageClassKinds K) {
757	// Map to LLVM visibility.
758	switch (K) {
759	case clang::LangOptions::VisibilityFromDLLStorageClassKinds::Keep:
760	return std::nullopt;
761	case clang::LangOptions::VisibilityFromDLLStorageClassKinds::Default:
762	return llvm::GlobalValue::DefaultVisibility;
763	case clang::LangOptions::VisibilityFromDLLStorageClassKinds::Hidden:
764	return llvm::GlobalValue::HiddenVisibility;
765	case clang::LangOptions::VisibilityFromDLLStorageClassKinds::Protected:
766	return llvm::GlobalValue::ProtectedVisibility;
767	}
768	llvm_unreachable("unknown option value!");
769	}
770
771	void setLLVMVisibility(llvm::GlobalValue &GV,
772	std::optional<llvm::GlobalValue::VisibilityTypes> V) {
773	if (!V)
774	return;
775
776	// Reset DSO locality before setting the visibility. This removes
777	// any effects that visibility options and annotations may have
778	// had on the DSO locality. Setting the visibility will implicitly set
779	// appropriate globals to DSO Local; however, this will be pessimistic
780	// w.r.t. to the normal compiler IRGen.
781	GV.setDSOLocal(false);
782	GV.setVisibility(*V);
783	}
784
785	static void setVisibilityFromDLLStorageClass(const clang::LangOptions &LO,
786	llvm::Module &M) {
787	if (!LO.VisibilityFromDLLStorageClass)
788	return;
789
790	std::optional<llvm::GlobalValue::VisibilityTypes> DLLExportVisibility =
791	getLLVMVisibility(K: LO.getDLLExportVisibility());
792
793	std::optional<llvm::GlobalValue::VisibilityTypes>
794	NoDLLStorageClassVisibility =
795	getLLVMVisibility(K: LO.getNoDLLStorageClassVisibility());
796
797	std::optional<llvm::GlobalValue::VisibilityTypes>
798	ExternDeclDLLImportVisibility =
799	getLLVMVisibility(K: LO.getExternDeclDLLImportVisibility());
800
801	std::optional<llvm::GlobalValue::VisibilityTypes>
802	ExternDeclNoDLLStorageClassVisibility =
803	getLLVMVisibility(K: LO.getExternDeclNoDLLStorageClassVisibility());
804
805	for (llvm::GlobalValue &GV : M.global_values()) {
806	if (GV.hasAppendingLinkage() || GV.hasLocalLinkage())
807	continue;
808
809	if (GV.isDeclarationForLinker())
810	setLLVMVisibility(GV, V: GV.getDLLStorageClass() ==
811	llvm::GlobalValue::DLLImportStorageClass
812	? ExternDeclDLLImportVisibility
813	: ExternDeclNoDLLStorageClassVisibility);
814	else
815	setLLVMVisibility(GV, V: GV.getDLLStorageClass() ==
816	llvm::GlobalValue::DLLExportStorageClass
817	? DLLExportVisibility
818	: NoDLLStorageClassVisibility);
819
820	GV.setDLLStorageClass(llvm::GlobalValue::DefaultStorageClass);
821	}
822	}
823
824	static bool isStackProtectorOn(const LangOptions &LangOpts,
825	const llvm::Triple &Triple,
826	clang::LangOptions::StackProtectorMode Mode) {
827	if (Triple.isAMDGPU() || Triple.isNVPTX())
828	return false;
829	return LangOpts.getStackProtector() == Mode;
830	}
831
832	void CodeGenModule::Release() {
833	Module *Primary = getContext().getCurrentNamedModule();
834	if (CXX20ModuleInits && Primary && !Primary->isHeaderLikeModule())
835	EmitModuleInitializers(Primary);
836	EmitDeferred();
837	DeferredDecls.insert(I: EmittedDeferredDecls.begin(),
838	E: EmittedDeferredDecls.end());
839	EmittedDeferredDecls.clear();
840	EmitVTablesOpportunistically();
841	applyGlobalValReplacements();
842	applyReplacements();
843	emitMultiVersionFunctions();
844
845	if (Context.getLangOpts().IncrementalExtensions &&
846	GlobalTopLevelStmtBlockInFlight.first) {
847	const TopLevelStmtDecl *TLSD = GlobalTopLevelStmtBlockInFlight.second;
848	GlobalTopLevelStmtBlockInFlight.first->FinishFunction(EndLoc: TLSD->getEndLoc());
849	GlobalTopLevelStmtBlockInFlight = {nullptr, nullptr};
850	}
851
852	// Module implementations are initialized the same way as a regular TU that
853	// imports one or more modules.
854	if (CXX20ModuleInits && Primary && Primary->isInterfaceOrPartition())
855	EmitCXXModuleInitFunc(Primary);
856	else
857	EmitCXXGlobalInitFunc();
858	EmitCXXGlobalCleanUpFunc();
859	registerGlobalDtorsWithAtExit();
860	EmitCXXThreadLocalInitFunc();
861	if (ObjCRuntime)
862	if (llvm::Function *ObjCInitFunction = ObjCRuntime->ModuleInitFunction())
863	AddGlobalCtor(Ctor: ObjCInitFunction);
864	if (Context.getLangOpts().CUDA && CUDARuntime) {
865	if (llvm::Function *CudaCtorFunction = CUDARuntime->finalizeModule())
866	AddGlobalCtor(Ctor: CudaCtorFunction);
867	}
868	if (OpenMPRuntime) {
869	OpenMPRuntime->createOffloadEntriesAndInfoMetadata();
870	OpenMPRuntime->clear();
871	}
872	if (PGOReader) {
873	getModule().setProfileSummary(
874	M: PGOReader->getSummary(/* UseCS */ false).getMD(Context&: VMContext),
875	Kind: llvm::ProfileSummary::PSK_Instr);
876	if (PGOStats.hasDiagnostics())
877	PGOStats.reportDiagnostics(Diags&: getDiags(), MainFile: getCodeGenOpts().MainFileName);
878	}
879	llvm::stable_sort(Range&: GlobalCtors, C: [](const Structor &L, const Structor &R) {
880	return L.LexOrder < R.LexOrder;
881	});
882	EmitCtorList(Fns&: GlobalCtors, GlobalName: "llvm.global_ctors");
883	EmitCtorList(Fns&: GlobalDtors, GlobalName: "llvm.global_dtors");
884	EmitGlobalAnnotations();
885	EmitStaticExternCAliases();
886	checkAliases();
887	EmitDeferredUnusedCoverageMappings();
888	CodeGenPGO(*this).setValueProfilingFlag(getModule());
889	CodeGenPGO(*this).setProfileVersion(getModule());
890	if (CoverageMapping)
891	CoverageMapping->emit();
892	if (CodeGenOpts.SanitizeCfiCrossDso) {
893	CodeGenFunction(*this).EmitCfiCheckFail();
894	CodeGenFunction(*this).EmitCfiCheckStub();
895	}
896	if (LangOpts.Sanitize.has(K: SanitizerKind::KCFI))
897	finalizeKCFITypes();
898	emitAtAvailableLinkGuard();
899	if (Context.getTargetInfo().getTriple().isWasm())
900	EmitMainVoidAlias();
901
902	if (getTriple().isAMDGPU()) {
903	// Emit amdhsa_code_object_version module flag, which is code object version
904	// times 100.
905	if (getTarget().getTargetOpts().CodeObjectVersion !=
906	llvm::CodeObjectVersionKind::COV_None) {
907	getModule().addModuleFlag(Behavior: llvm::Module::Error,
908	Key: "amdhsa_code_object_version",
909	Val: getTarget().getTargetOpts().CodeObjectVersion);
910	}
911
912	// Currently, "-mprintf-kind" option is only supported for HIP
913	if (LangOpts.HIP) {
914	auto *MDStr = llvm::MDString::get(
915	Context&: getLLVMContext(), Str: (getTarget().getTargetOpts().AMDGPUPrintfKindVal ==
916	TargetOptions::AMDGPUPrintfKind::Hostcall)
917	? "hostcall"
918	: "buffered");
919	getModule().addModuleFlag(Behavior: llvm::Module::Error, Key: "amdgpu_printf_kind",
920	Val: MDStr);
921	}
922	}
923
924	// Emit a global array containing all external kernels or device variables
925	// used by host functions and mark it as used for CUDA/HIP. This is necessary
926	// to get kernels or device variables in archives linked in even if these
927	// kernels or device variables are only used in host functions.
928	if (!Context.CUDAExternalDeviceDeclODRUsedByHost.empty()) {
929	SmallVector<llvm::Constant *, 8> UsedArray;
930	for (auto D : Context.CUDAExternalDeviceDeclODRUsedByHost) {
931	GlobalDecl GD;
932	if (auto *FD = dyn_cast<FunctionDecl>(Val: D))
933	GD = GlobalDecl(FD, KernelReferenceKind::Kernel);
934	else
935	GD = GlobalDecl(D);
936	UsedArray.push_back(Elt: llvm::ConstantExpr::getPointerBitCastOrAddrSpaceCast(
937	C: GetAddrOfGlobal(GD), Ty: Int8PtrTy));
938	}
939
940	llvm::ArrayType *ATy = llvm::ArrayType::get(ElementType: Int8PtrTy, NumElements: UsedArray.size());
941
942	auto *GV = new llvm::GlobalVariable(
943	getModule(), ATy, false, llvm::GlobalValue::InternalLinkage,
944	llvm::ConstantArray::get(T: ATy, V: UsedArray), "__clang_gpu_used_external");
945	addCompilerUsedGlobal(GV);
946	}
947	if (LangOpts.HIP && !getLangOpts().OffloadingNewDriver) {
948	// Emit a unique ID so that host and device binaries from the same
949	// compilation unit can be associated.
950	auto *GV = new llvm::GlobalVariable(
951	getModule(), Int8Ty, false, llvm::GlobalValue::ExternalLinkage,
952	llvm::Constant::getNullValue(Ty: Int8Ty),
953	"__hip_cuid_" + getContext().getCUIDHash());
954	addCompilerUsedGlobal(GV);
955	}
956	emitLLVMUsed();
957	if (SanStats)
958	SanStats->finish();
959
960	if (CodeGenOpts.Autolink &&
961	(Context.getLangOpts().Modules || !LinkerOptionsMetadata.empty())) {
962	EmitModuleLinkOptions();
963	}
964
965	// On ELF we pass the dependent library specifiers directly to the linker
966	// without manipulating them. This is in contrast to other platforms where
967	// they are mapped to a specific linker option by the compiler. This
968	// difference is a result of the greater variety of ELF linkers and the fact
969	// that ELF linkers tend to handle libraries in a more complicated fashion
970	// than on other platforms. This forces us to defer handling the dependent
971	// libs to the linker.
972	//
973	// CUDA/HIP device and host libraries are different. Currently there is no
974	// way to differentiate dependent libraries for host or device. Existing
975	// usage of #pragma comment(lib, *) is intended for host libraries on
976	// Windows. Therefore emit llvm.dependent-libraries only for host.
977	if (!ELFDependentLibraries.empty() && !Context.getLangOpts().CUDAIsDevice) {
978	auto *NMD = getModule().getOrInsertNamedMetadata(Name: "llvm.dependent-libraries");
979	for (auto *MD : ELFDependentLibraries)
980	NMD->addOperand(M: MD);
981	}
982
983	// Record mregparm value now so it is visible through rest of codegen.
984	if (Context.getTargetInfo().getTriple().getArch() == llvm::Triple::x86)
985	getModule().addModuleFlag(Behavior: llvm::Module::Error, Key: "NumRegisterParameters",
986	Val: CodeGenOpts.NumRegisterParameters);
987
988	if (CodeGenOpts.DwarfVersion) {
989	getModule().addModuleFlag(Behavior: llvm::Module::Max, Key: "Dwarf Version",
990	Val: CodeGenOpts.DwarfVersion);
991	}
992
993	if (CodeGenOpts.Dwarf64)
994	getModule().addModuleFlag(Behavior: llvm::Module::Max, Key: "DWARF64", Val: 1);
995
996	if (Context.getLangOpts().SemanticInterposition)
997	// Require various optimization to respect semantic interposition.
998	getModule().setSemanticInterposition(true);
999
1000	if (CodeGenOpts.EmitCodeView) {
1001	// Indicate that we want CodeView in the metadata.
1002	getModule().addModuleFlag(Behavior: llvm::Module::Warning, Key: "CodeView", Val: 1);
1003	}
1004	if (CodeGenOpts.CodeViewGHash) {
1005	getModule().addModuleFlag(Behavior: llvm::Module::Warning, Key: "CodeViewGHash", Val: 1);
1006	}
1007	if (CodeGenOpts.ControlFlowGuard) {
1008	// Function ID tables and checks for Control Flow Guard (cfguard=2).
1009	getModule().addModuleFlag(Behavior: llvm::Module::Warning, Key: "cfguard", Val: 2);
1010	} else if (CodeGenOpts.ControlFlowGuardNoChecks) {
1011	// Function ID tables for Control Flow Guard (cfguard=1).
1012	getModule().addModuleFlag(Behavior: llvm::Module::Warning, Key: "cfguard", Val: 1);
1013	}
1014	if (CodeGenOpts.EHContGuard) {
1015	// Function ID tables for EH Continuation Guard.
1016	getModule().addModuleFlag(Behavior: llvm::Module::Warning, Key: "ehcontguard", Val: 1);
1017	}
1018	if (Context.getLangOpts().Kernel) {
1019	// Note if we are compiling with /kernel.
1020	getModule().addModuleFlag(Behavior: llvm::Module::Warning, Key: "ms-kernel", Val: 1);
1021	}
1022	if (CodeGenOpts.OptimizationLevel > 0 && CodeGenOpts.StrictVTablePointers) {
1023	// We don't support LTO with 2 with different StrictVTablePointers
1024	// FIXME: we could support it by stripping all the information introduced
1025	// by StrictVTablePointers.
1026
1027	getModule().addModuleFlag(Behavior: llvm::Module::Error, Key: "StrictVTablePointers",Val: 1);
1028
1029	llvm::Metadata *Ops[2] = {
1030	llvm::MDString::get(Context&: VMContext, Str: "StrictVTablePointers"),
1031	llvm::ConstantAsMetadata::get(C: llvm::ConstantInt::get(
1032	Ty: llvm::Type::getInt32Ty(C&: VMContext), V: 1))};
1033
1034	getModule().addModuleFlag(Behavior: llvm::Module::Require,
1035	Key: "StrictVTablePointersRequirement",
1036	Val: llvm::MDNode::get(Context&: VMContext, MDs: Ops));
1037	}
1038	if (getModuleDebugInfo())
1039	// We support a single version in the linked module. The LLVM
1040	// parser will drop debug info with a different version number
1041	// (and warn about it, too).
1042	getModule().addModuleFlag(Behavior: llvm::Module::Warning, Key: "Debug Info Version",
1043	Val: llvm::DEBUG_METADATA_VERSION);
1044
1045	// We need to record the widths of enums and wchar_t, so that we can generate
1046	// the correct build attributes in the ARM backend. wchar_size is also used by
1047	// TargetLibraryInfo.
1048	uint64_t WCharWidth =
1049	Context.getTypeSizeInChars(T: Context.getWideCharType()).getQuantity();
1050	getModule().addModuleFlag(Behavior: llvm::Module::Error, Key: "wchar_size", Val: WCharWidth);
1051
1052	if (getTriple().isOSzOS()) {
1053	getModule().addModuleFlag(Behavior: llvm::Module::Warning,
1054	Key: "zos_product_major_version",
1055	Val: uint32_t(CLANG_VERSION_MAJOR));
1056	getModule().addModuleFlag(Behavior: llvm::Module::Warning,
1057	Key: "zos_product_minor_version",
1058	Val: uint32_t(CLANG_VERSION_MINOR));
1059	getModule().addModuleFlag(Behavior: llvm::Module::Warning, Key: "zos_product_patchlevel",
1060	Val: uint32_t(CLANG_VERSION_PATCHLEVEL));
1061	std::string ProductId = getClangVendor() + "clang";
1062	getModule().addModuleFlag(Behavior: llvm::Module::Error, Key: "zos_product_id",
1063	Val: llvm::MDString::get(Context&: VMContext, Str: ProductId));
1064
1065	// Record the language because we need it for the PPA2.
1066	StringRef lang_str = languageToString(
1067	L: LangStandard::getLangStandardForKind(K: LangOpts.LangStd).Language);
1068	getModule().addModuleFlag(Behavior: llvm::Module::Error, Key: "zos_cu_language",
1069	Val: llvm::MDString::get(Context&: VMContext, Str: lang_str));
1070
1071	time_t TT = PreprocessorOpts.SourceDateEpoch
1072	? *PreprocessorOpts.SourceDateEpoch
1073	: std::time(timer: nullptr);
1074	getModule().addModuleFlag(Behavior: llvm::Module::Max, Key: "zos_translation_time",
1075	Val: static_cast<uint64_t>(TT));
1076
1077	// Multiple modes will be supported here.
1078	getModule().addModuleFlag(Behavior: llvm::Module::Error, Key: "zos_le_char_mode",
1079	Val: llvm::MDString::get(Context&: VMContext, Str: "ascii"));
1080	}
1081
1082	llvm::Triple T = Context.getTargetInfo().getTriple();
1083	if (T.isARM() || T.isThumb()) {
1084	// The minimum width of an enum in bytes
1085	uint64_t EnumWidth = Context.getLangOpts().ShortEnums ? 1 : 4;
1086	getModule().addModuleFlag(Behavior: llvm::Module::Error, Key: "min_enum_size", Val: EnumWidth);
1087	}
1088
1089	if (T.isRISCV()) {
1090	StringRef ABIStr = Target.getABI();
1091	llvm::LLVMContext &Ctx = TheModule.getContext();
1092	getModule().addModuleFlag(Behavior: llvm::Module::Error, Key: "target-abi",
1093	Val: llvm::MDString::get(Context&: Ctx, Str: ABIStr));
1094
1095	// Add the canonical ISA string as metadata so the backend can set the ELF
1096	// attributes correctly. We use AppendUnique so LTO will keep all of the
1097	// unique ISA strings that were linked together.
1098	const std::vector<std::string> &Features =
1099	getTarget().getTargetOpts().Features;
1100	auto ParseResult =
1101	llvm::RISCVISAInfo::parseFeatures(XLen: T.isRISCV64() ? 64 : 32, Features);
1102	if (!errorToBool(Err: ParseResult.takeError()))
1103	getModule().addModuleFlag(
1104	Behavior: llvm::Module::AppendUnique, Key: "riscv-isa",
1105	Val: llvm::MDNode::get(
1106	Context&: Ctx, MDs: llvm::MDString::get(Context&: Ctx, Str: (*ParseResult)->toString())));
1107	}
1108
1109	if (CodeGenOpts.SanitizeCfiCrossDso) {
1110	// Indicate that we want cross-DSO control flow integrity checks.
1111	getModule().addModuleFlag(Behavior: llvm::Module::Override, Key: "Cross-DSO CFI", Val: 1);
1112	}
1113
1114	if (CodeGenOpts.WholeProgramVTables) {
1115	// Indicate whether VFE was enabled for this module, so that the
1116	// vcall_visibility metadata added under whole program vtables is handled
1117	// appropriately in the optimizer.
1118	getModule().addModuleFlag(Behavior: llvm::Module::Error, Key: "Virtual Function Elim",
1119	Val: CodeGenOpts.VirtualFunctionElimination);
1120	}
1121
1122	if (LangOpts.Sanitize.has(K: SanitizerKind::CFIICall)) {
1123	getModule().addModuleFlag(Behavior: llvm::Module::Override,
1124	Key: "CFI Canonical Jump Tables",
1125	Val: CodeGenOpts.SanitizeCfiCanonicalJumpTables);
1126	}
1127
1128	if (LangOpts.Sanitize.has(K: SanitizerKind::KCFI)) {
1129	getModule().addModuleFlag(Behavior: llvm::Module::Override, Key: "kcfi", Val: 1);
1130	// KCFI assumes patchable-function-prefix is the same for all indirectly
1131	// called functions. Store the expected offset for code generation.
1132	if (CodeGenOpts.PatchableFunctionEntryOffset)
1133	getModule().addModuleFlag(Behavior: llvm::Module::Override, Key: "kcfi-offset",
1134	Val: CodeGenOpts.PatchableFunctionEntryOffset);
1135	}
1136
1137	if (CodeGenOpts.CFProtectionReturn &&
1138	Target.checkCFProtectionReturnSupported(Diags&: getDiags())) {
1139	// Indicate that we want to instrument return control flow protection.
1140	getModule().addModuleFlag(Behavior: llvm::Module::Min, Key: "cf-protection-return",
1141	Val: 1);
1142	}
1143
1144	if (CodeGenOpts.CFProtectionBranch &&
1145	Target.checkCFProtectionBranchSupported(Diags&: getDiags())) {
1146	// Indicate that we want to instrument branch control flow protection.
1147	getModule().addModuleFlag(Behavior: llvm::Module::Min, Key: "cf-protection-branch",
1148	Val: 1);
1149	}
1150
1151	if (CodeGenOpts.FunctionReturnThunks)
1152	getModule().addModuleFlag(Behavior: llvm::Module::Override, Key: "function_return_thunk_extern", Val: 1);
1153
1154	if (CodeGenOpts.IndirectBranchCSPrefix)
1155	getModule().addModuleFlag(Behavior: llvm::Module::Override, Key: "indirect_branch_cs_prefix", Val: 1);
1156
1157	// Add module metadata for return address signing (ignoring
1158	// non-leaf/all) and stack tagging. These are actually turned on by function
1159	// attributes, but we use module metadata to emit build attributes. This is
1160	// needed for LTO, where the function attributes are inside bitcode
1161	// serialised into a global variable by the time build attributes are
1162	// emitted, so we can't access them. LTO objects could be compiled with
1163	// different flags therefore module flags are set to "Min" behavior to achieve
1164	// the same end result of the normal build where e.g BTI is off if any object
1165	// doesn't support it.
1166	if (Context.getTargetInfo().hasFeature(Feature: "ptrauth") &&
1167	LangOpts.getSignReturnAddressScope() !=
1168	LangOptions::SignReturnAddressScopeKind::None)
1169	getModule().addModuleFlag(Behavior: llvm::Module::Override,
1170	Key: "sign-return-address-buildattr", Val: 1);
1171	if (LangOpts.Sanitize.has(K: SanitizerKind::MemtagStack))
1172	getModule().addModuleFlag(Behavior: llvm::Module::Override,
1173	Key: "tag-stack-memory-buildattr", Val: 1);
1174
1175	if (T.isARM() || T.isThumb() || T.isAArch64()) {
1176	if (LangOpts.BranchTargetEnforcement)
1177	getModule().addModuleFlag(Behavior: llvm::Module::Min, Key: "branch-target-enforcement",
1178	Val: 1);
1179	if (LangOpts.BranchProtectionPAuthLR)
1180	getModule().addModuleFlag(Behavior: llvm::Module::Min, Key: "branch-protection-pauth-lr",
1181	Val: 1);
1182	if (LangOpts.GuardedControlStack)
1183	getModule().addModuleFlag(Behavior: llvm::Module::Min, Key: "guarded-control-stack", Val: 1);
1184	if (LangOpts.hasSignReturnAddress())
1185	getModule().addModuleFlag(Behavior: llvm::Module::Min, Key: "sign-return-address", Val: 1);
1186	if (LangOpts.isSignReturnAddressScopeAll())
1187	getModule().addModuleFlag(Behavior: llvm::Module::Min, Key: "sign-return-address-all",
1188	Val: 1);
1189	if (!LangOpts.isSignReturnAddressWithAKey())
1190	getModule().addModuleFlag(Behavior: llvm::Module::Min,
1191	Key: "sign-return-address-with-bkey", Val: 1);
1192	}
1193
1194	if (CodeGenOpts.StackClashProtector)
1195	getModule().addModuleFlag(
1196	Behavior: llvm::Module::Override, Key: "probe-stack",
1197	Val: llvm::MDString::get(Context&: TheModule.getContext(), Str: "inline-asm"));
1198
1199	if (CodeGenOpts.StackProbeSize && CodeGenOpts.StackProbeSize != 4096)
1200	getModule().addModuleFlag(Behavior: llvm::Module::Min, Key: "stack-probe-size",
1201	Val: CodeGenOpts.StackProbeSize);
1202
1203	if (!CodeGenOpts.MemoryProfileOutput.empty()) {
1204	llvm::LLVMContext &Ctx = TheModule.getContext();
1205	getModule().addModuleFlag(
1206	Behavior: llvm::Module::Error, Key: "MemProfProfileFilename",
1207	Val: llvm::MDString::get(Context&: Ctx, Str: CodeGenOpts.MemoryProfileOutput));
1208	}
1209
1210	if (LangOpts.CUDAIsDevice && getTriple().isNVPTX()) {
1211	// Indicate whether __nvvm_reflect should be configured to flush denormal
1212	// floating point values to 0. (This corresponds to its "__CUDA_FTZ"
1213	// property.)
1214	getModule().addModuleFlag(Behavior: llvm::Module::Override, Key: "nvvm-reflect-ftz",
1215	Val: CodeGenOpts.FP32DenormalMode.Output !=
1216	llvm::DenormalMode::IEEE);
1217	}
1218
1219	if (LangOpts.EHAsynch)
1220	getModule().addModuleFlag(Behavior: llvm::Module::Warning, Key: "eh-asynch", Val: 1);
1221
1222	// Indicate whether this Module was compiled with -fopenmp
1223	if (getLangOpts().OpenMP && !getLangOpts().OpenMPSimd)
1224	getModule().addModuleFlag(Behavior: llvm::Module::Max, Key: "openmp", Val: LangOpts.OpenMP);
1225	if (getLangOpts().OpenMPIsTargetDevice)
1226	getModule().addModuleFlag(Behavior: llvm::Module::Max, Key: "openmp-device",
1227	Val: LangOpts.OpenMP);
1228
1229	// Emit OpenCL specific module metadata: OpenCL/SPIR version.
1230	if (LangOpts.OpenCL || (LangOpts.CUDAIsDevice && getTriple().isSPIRV())) {
1231	EmitOpenCLMetadata();
1232	// Emit SPIR version.
1233	if (getTriple().isSPIR()) {
1234	// SPIR v2.0 s2.12 - The SPIR version used by the module is stored in the
1235	// opencl.spir.version named metadata.
1236	// C++ for OpenCL has a distinct mapping for version compatibility with
1237	// OpenCL.
1238	auto Version = LangOpts.getOpenCLCompatibleVersion();
1239	llvm::Metadata *SPIRVerElts[] = {
1240	llvm::ConstantAsMetadata::get(C: llvm::ConstantInt::get(
1241	Ty: Int32Ty, V: Version / 100)),
1242	llvm::ConstantAsMetadata::get(C: llvm::ConstantInt::get(
1243	Ty: Int32Ty, V: (Version / 100 > 1) ? 0 : 2))};
1244	llvm::NamedMDNode *SPIRVerMD =
1245	TheModule.getOrInsertNamedMetadata(Name: "opencl.spir.version");
1246	llvm::LLVMContext &Ctx = TheModule.getContext();
1247	SPIRVerMD->addOperand(M: llvm::MDNode::get(Context&: Ctx, MDs: SPIRVerElts));
1248	}
1249	}
1250
1251	// HLSL related end of code gen work items.
1252	if (LangOpts.HLSL)
1253	getHLSLRuntime().finishCodeGen();
1254
1255	if (uint32_t PLevel = Context.getLangOpts().PICLevel) {
1256	assert(PLevel < 3 && "Invalid PIC Level");
1257	getModule().setPICLevel(static_cast<llvm::PICLevel::Level>(PLevel));
1258	if (Context.getLangOpts().PIE)
1259	getModule().setPIELevel(static_cast<llvm::PIELevel::Level>(PLevel));
1260	}
1261
1262	if (getCodeGenOpts().CodeModel.size() > 0) {
1263	unsigned CM = llvm::StringSwitch<unsigned>(getCodeGenOpts().CodeModel)
1264	.Case(S: "tiny", Value: llvm::CodeModel::Tiny)
1265	.Case(S: "small", Value: llvm::CodeModel::Small)
1266	.Case(S: "kernel", Value: llvm::CodeModel::Kernel)
1267	.Case(S: "medium", Value: llvm::CodeModel::Medium)
1268	.Case(S: "large", Value: llvm::CodeModel::Large)
1269	.Default(Value: ~0u);
1270	if (CM != ~0u) {
1271	llvm::CodeModel::Model codeModel = static_cast<llvm::CodeModel::Model>(CM);
1272	getModule().setCodeModel(codeModel);
1273
1274	if ((CM == llvm::CodeModel::Medium || CM == llvm::CodeModel::Large) &&
1275	Context.getTargetInfo().getTriple().getArch() ==
1276	llvm::Triple::x86_64) {
1277	getModule().setLargeDataThreshold(getCodeGenOpts().LargeDataThreshold);
1278	}
1279	}
1280	}
1281
1282	if (CodeGenOpts.NoPLT)
1283	getModule().setRtLibUseGOT();
1284	if (getTriple().isOSBinFormatELF() &&
1285	CodeGenOpts.DirectAccessExternalData !=
1286	getModule().getDirectAccessExternalData()) {
1287	getModule().setDirectAccessExternalData(
1288	CodeGenOpts.DirectAccessExternalData);
1289	}
1290	if (CodeGenOpts.UnwindTables)
1291	getModule().setUwtable(llvm::UWTableKind(CodeGenOpts.UnwindTables));
1292
1293	switch (CodeGenOpts.getFramePointer()) {
1294	case CodeGenOptions::FramePointerKind::None:
1295	// 0 ("none") is the default.
1296	break;
1297	case CodeGenOptions::FramePointerKind::NonLeaf:
1298	getModule().setFramePointer(llvm::FramePointerKind::NonLeaf);
1299	break;
1300	case CodeGenOptions::FramePointerKind::All:
1301	getModule().setFramePointer(llvm::FramePointerKind::All);
1302	break;
1303	}
1304
1305	SimplifyPersonality();
1306
1307	if (getCodeGenOpts().EmitDeclMetadata)
1308	EmitDeclMetadata();
1309
1310	if (getCodeGenOpts().CoverageNotesFile.size() ||
1311	getCodeGenOpts().CoverageDataFile.size())
1312	EmitCoverageFile();
1313
1314	if (CGDebugInfo *DI = getModuleDebugInfo())
1315	DI->finalize();
1316
1317	if (getCodeGenOpts().EmitVersionIdentMetadata)
1318	EmitVersionIdentMetadata();
1319
1320	if (!getCodeGenOpts().RecordCommandLine.empty())
1321	EmitCommandLineMetadata();
1322
1323	if (!getCodeGenOpts().StackProtectorGuard.empty())
1324	getModule().setStackProtectorGuard(getCodeGenOpts().StackProtectorGuard);
1325	if (!getCodeGenOpts().StackProtectorGuardReg.empty())
1326	getModule().setStackProtectorGuardReg(
1327	getCodeGenOpts().StackProtectorGuardReg);
1328	if (!getCodeGenOpts().StackProtectorGuardSymbol.empty())
1329	getModule().setStackProtectorGuardSymbol(
1330	getCodeGenOpts().StackProtectorGuardSymbol);
1331	if (getCodeGenOpts().StackProtectorGuardOffset != INT_MAX)
1332	getModule().setStackProtectorGuardOffset(
1333	getCodeGenOpts().StackProtectorGuardOffset);
1334	if (getCodeGenOpts().StackAlignment)
1335	getModule().setOverrideStackAlignment(getCodeGenOpts().StackAlignment);
1336	if (getCodeGenOpts().SkipRaxSetup)
1337	getModule().addModuleFlag(Behavior: llvm::Module::Override, Key: "SkipRaxSetup", Val: 1);
1338	if (getLangOpts().RegCall4)
1339	getModule().addModuleFlag(Behavior: llvm::Module::Override, Key: "RegCallv4", Val: 1);
1340
1341	if (getContext().getTargetInfo().getMaxTLSAlign())
1342	getModule().addModuleFlag(Behavior: llvm::Module::Error, Key: "MaxTLSAlign",
1343	Val: getContext().getTargetInfo().getMaxTLSAlign());
1344
1345	getTargetCodeGenInfo().emitTargetGlobals(CGM&: *this);
1346
1347	getTargetCodeGenInfo().emitTargetMetadata(CGM&: *this, MangledDeclNames);
1348
1349	EmitBackendOptionsMetadata(CodeGenOpts: getCodeGenOpts());
1350
1351	// If there is device offloading code embed it in the host now.
1352	EmbedObject(M: &getModule(), CGOpts: CodeGenOpts, Diags&: getDiags());
1353
1354	// Set visibility from DLL storage class
1355	// We do this at the end of LLVM IR generation; after any operation
1356	// that might affect the DLL storage class or the visibility, and
1357	// before anything that might act on these.
1358	setVisibilityFromDLLStorageClass(LO: LangOpts, M&: getModule());
1359	}
1360
1361	void CodeGenModule::EmitOpenCLMetadata() {
1362	// SPIR v2.0 s2.13 - The OpenCL version used by the module is stored in the
1363	// opencl.ocl.version named metadata node.
1364	// C++ for OpenCL has a distinct mapping for versions compatibile with OpenCL.
1365	auto Version = LangOpts.getOpenCLCompatibleVersion();
1366	llvm::Metadata *OCLVerElts[] = {
1367	llvm::ConstantAsMetadata::get(C: llvm::ConstantInt::get(
1368	Ty: Int32Ty, V: Version / 100)),
1369	llvm::ConstantAsMetadata::get(C: llvm::ConstantInt::get(
1370	Ty: Int32Ty, V: (Version % 100) / 10))};
1371	llvm::NamedMDNode *OCLVerMD =
1372	TheModule.getOrInsertNamedMetadata(Name: "opencl.ocl.version");
1373	llvm::LLVMContext &Ctx = TheModule.getContext();
1374	OCLVerMD->addOperand(M: llvm::MDNode::get(Context&: Ctx, MDs: OCLVerElts));
1375	}
1376
1377	void CodeGenModule::EmitBackendOptionsMetadata(
1378	const CodeGenOptions &CodeGenOpts) {
1379	if (getTriple().isRISCV()) {
1380	getModule().addModuleFlag(Behavior: llvm::Module::Min, Key: "SmallDataLimit",
1381	Val: CodeGenOpts.SmallDataLimit);
1382	}
1383	}
1384
1385	void CodeGenModule::UpdateCompletedType(const TagDecl *TD) {
1386	// Make sure that this type is translated.
1387	Types.UpdateCompletedType(TD);
1388	}
1389
1390	void CodeGenModule::RefreshTypeCacheForClass(const CXXRecordDecl *RD) {
1391	// Make sure that this type is translated.
1392	Types.RefreshTypeCacheForClass(RD);
1393	}
1394
1395	llvm::MDNode *CodeGenModule::getTBAATypeInfo(QualType QTy) {
1396	if (!TBAA)
1397	return nullptr;
1398	return TBAA->getTypeInfo(QTy);
1399	}
1400
1401	TBAAAccessInfo CodeGenModule::getTBAAAccessInfo(QualType AccessType) {
1402	if (!TBAA)
1403	return TBAAAccessInfo();
1404	if (getLangOpts().CUDAIsDevice) {
1405	// As CUDA builtin surface/texture types are replaced, skip generating TBAA
1406	// access info.
1407	if (AccessType->isCUDADeviceBuiltinSurfaceType()) {
1408	if (getTargetCodeGenInfo().getCUDADeviceBuiltinSurfaceDeviceType() !=
1409	nullptr)
1410	return TBAAAccessInfo();
1411	} else if (AccessType->isCUDADeviceBuiltinTextureType()) {
1412	if (getTargetCodeGenInfo().getCUDADeviceBuiltinTextureDeviceType() !=
1413	nullptr)
1414	return TBAAAccessInfo();
1415	}
1416	}
1417	return TBAA->getAccessInfo(AccessType);
1418	}
1419
1420	TBAAAccessInfo
1421	CodeGenModule::getTBAAVTablePtrAccessInfo(llvm::Type *VTablePtrType) {
1422	if (!TBAA)
1423	return TBAAAccessInfo();
1424	return TBAA->getVTablePtrAccessInfo(VTablePtrType);
1425	}
1426
1427	llvm::MDNode *CodeGenModule::getTBAAStructInfo(QualType QTy) {
1428	if (!TBAA)
1429	return nullptr;
1430	return TBAA->getTBAAStructInfo(QTy);
1431	}
1432
1433	llvm::MDNode *CodeGenModule::getTBAABaseTypeInfo(QualType QTy) {
1434	if (!TBAA)
1435	return nullptr;
1436	return TBAA->getBaseTypeInfo(QTy);
1437	}
1438
1439	llvm::MDNode *CodeGenModule::getTBAAAccessTagInfo(TBAAAccessInfo Info) {
1440	if (!TBAA)
1441	return nullptr;
1442	return TBAA->getAccessTagInfo(Info);
1443	}
1444
1445	TBAAAccessInfo CodeGenModule::mergeTBAAInfoForCast(TBAAAccessInfo SourceInfo,
1446	TBAAAccessInfo TargetInfo) {
1447	if (!TBAA)
1448	return TBAAAccessInfo();
1449	return TBAA->mergeTBAAInfoForCast(SourceInfo, TargetInfo);
1450	}
1451
1452	TBAAAccessInfo
1453	CodeGenModule::mergeTBAAInfoForConditionalOperator(TBAAAccessInfo InfoA,
1454	TBAAAccessInfo InfoB) {
1455	if (!TBAA)
1456	return TBAAAccessInfo();
1457	return TBAA->mergeTBAAInfoForConditionalOperator(InfoA, InfoB);
1458	}
1459
1460	TBAAAccessInfo
1461	CodeGenModule::mergeTBAAInfoForMemoryTransfer(TBAAAccessInfo DestInfo,
1462	TBAAAccessInfo SrcInfo) {
1463	if (!TBAA)
1464	return TBAAAccessInfo();
1465	return TBAA->mergeTBAAInfoForConditionalOperator(InfoA: DestInfo, InfoB: SrcInfo);
1466	}
1467
1468	void CodeGenModule::DecorateInstructionWithTBAA(llvm::Instruction *Inst,
1469	TBAAAccessInfo TBAAInfo) {
1470	if (llvm::MDNode *Tag = getTBAAAccessTagInfo(Info: TBAAInfo))
1471	Inst->setMetadata(KindID: llvm::LLVMContext::MD_tbaa, Node: Tag);
1472	}
1473
1474	void CodeGenModule::DecorateInstructionWithInvariantGroup(
1475	llvm::Instruction *I, const CXXRecordDecl *RD) {
1476	I->setMetadata(KindID: llvm::LLVMContext::MD_invariant_group,
1477	Node: llvm::MDNode::get(Context&: getLLVMContext(), MDs: {}));
1478	}
1479
1480	void CodeGenModule::Error(SourceLocation loc, StringRef message) {
1481	unsigned diagID = getDiags().getCustomDiagID(L: DiagnosticsEngine::Error, FormatString: "%0");
1482	getDiags().Report(Loc: Context.getFullLoc(Loc: loc), DiagID: diagID) << message;
1483	}
1484
1485	/// ErrorUnsupported - Print out an error that codegen doesn't support the
1486	/// specified stmt yet.
1487	void CodeGenModule::ErrorUnsupported(const Stmt *S, const char *Type) {
1488	unsigned DiagID = getDiags().getCustomDiagID(L: DiagnosticsEngine::Error,
1489	FormatString: "cannot compile this %0 yet");
1490	std::string Msg = Type;
1491	getDiags().Report(Loc: Context.getFullLoc(Loc: S->getBeginLoc()), DiagID)
1492	<< Msg << S->getSourceRange();
1493	}
1494
1495	/// ErrorUnsupported - Print out an error that codegen doesn't support the
1496	/// specified decl yet.
1497	void CodeGenModule::ErrorUnsupported(const Decl *D, const char *Type) {
1498	unsigned DiagID = getDiags().getCustomDiagID(L: DiagnosticsEngine::Error,
1499	FormatString: "cannot compile this %0 yet");
1500	std::string Msg = Type;
1501	getDiags().Report(Loc: Context.getFullLoc(Loc: D->getLocation()), DiagID) << Msg;
1502	}
1503
1504	llvm::ConstantInt *CodeGenModule::getSize(CharUnits size) {
1505	return llvm::ConstantInt::get(Ty: SizeTy, V: size.getQuantity());
1506	}
1507
1508	void CodeGenModule::setGlobalVisibility(llvm::GlobalValue *GV,
1509	const NamedDecl *D) const {
1510	// Internal definitions always have default visibility.
1511	if (GV->hasLocalLinkage()) {
1512	GV->setVisibility(llvm::GlobalValue::DefaultVisibility);
1513	return;
1514	}
1515	if (!D)
1516	return;
1517
1518	// Set visibility for definitions, and for declarations if requested globally
1519	// or set explicitly.
1520	LinkageInfo LV = D->getLinkageAndVisibility();
1521
1522	// OpenMP declare target variables must be visible to the host so they can
1523	// be registered. We require protected visibility unless the variable has
1524	// the DT_nohost modifier and does not need to be registered.
1525	if (Context.getLangOpts().OpenMP &&
1526	Context.getLangOpts().OpenMPIsTargetDevice && isa<VarDecl>(D) &&
1527	D->hasAttr<OMPDeclareTargetDeclAttr>() &&
1528	D->getAttr<OMPDeclareTargetDeclAttr>()->getDevType() !=
1529	OMPDeclareTargetDeclAttr::DT_NoHost &&
1530	LV.getVisibility() == HiddenVisibility) {
1531	GV->setVisibility(llvm::GlobalValue::ProtectedVisibility);
1532	return;
1533	}
1534
1535	if (GV->hasDLLExportStorageClass() || GV->hasDLLImportStorageClass()) {
1536	// Reject incompatible dlllstorage and visibility annotations.
1537	if (!LV.isVisibilityExplicit())
1538	return;
1539	if (GV->hasDLLExportStorageClass()) {
1540	if (LV.getVisibility() == HiddenVisibility)
1541	getDiags().Report(D->getLocation(),
1542	diag::err_hidden_visibility_dllexport);
1543	} else if (LV.getVisibility() != DefaultVisibility) {
1544	getDiags().Report(D->getLocation(),
1545	diag::err_non_default_visibility_dllimport);
1546	}
1547	return;
1548	}
1549
1550	if (LV.isVisibilityExplicit() || getLangOpts().SetVisibilityForExternDecls ||
1551	!GV->isDeclarationForLinker())
1552	GV->setVisibility(GetLLVMVisibility(V: LV.getVisibility()));
1553	}
1554
1555	static bool shouldAssumeDSOLocal(const CodeGenModule &CGM,
1556	llvm::GlobalValue *GV) {
1557	if (GV->hasLocalLinkage())
1558	return true;
1559
1560	if (!GV->hasDefaultVisibility() && !GV->hasExternalWeakLinkage())
1561	return true;
1562
1563	// DLLImport explicitly marks the GV as external.
1564	if (GV->hasDLLImportStorageClass())
1565	return false;
1566
1567	const llvm::Triple &TT = CGM.getTriple();
1568	const auto &CGOpts = CGM.getCodeGenOpts();
1569	if (TT.isWindowsGNUEnvironment()) {
1570	// In MinGW, variables without DLLImport can still be automatically
1571	// imported from a DLL by the linker; don't mark variables that
1572	// potentially could come from another DLL as DSO local.
1573
1574	// With EmulatedTLS, TLS variables can be autoimported from other DLLs
1575	// (and this actually happens in the public interface of libstdc++), so
1576	// such variables can't be marked as DSO local. (Native TLS variables
1577	// can't be dllimported at all, though.)
1578	if (GV->isDeclarationForLinker() && isa<llvm::GlobalVariable>(Val: GV) &&
1579	(!GV->isThreadLocal() || CGM.getCodeGenOpts().EmulatedTLS) &&
1580	CGOpts.AutoImport)
1581	return false;
1582	}
1583
1584	// On COFF, don't mark 'extern_weak' symbols as DSO local. If these symbols
1585	// remain unresolved in the link, they can be resolved to zero, which is
1586	// outside the current DSO.
1587	if (TT.isOSBinFormatCOFF() && GV->hasExternalWeakLinkage())
1588	return false;
1589
1590	// Every other GV is local on COFF.
1591	// Make an exception for windows OS in the triple: Some firmware builds use
1592	// *-win32-macho triples. This (accidentally?) produced windows relocations
1593	// without GOT tables in older clang versions; Keep this behaviour.
1594	// FIXME: even thread local variables?
1595	if (TT.isOSBinFormatCOFF() || (TT.isOSWindows() && TT.isOSBinFormatMachO()))
1596	return true;
1597
1598	// Only handle COFF and ELF for now.
1599	if (!TT.isOSBinFormatELF())
1600	return false;
1601
1602	// If this is not an executable, don't assume anything is local.
1603	llvm::Reloc::Model RM = CGOpts.RelocationModel;
1604	const auto &LOpts = CGM.getLangOpts();
1605	if (RM != llvm::Reloc::Static && !LOpts.PIE) {
1606	// On ELF, if -fno-semantic-interposition is specified and the target
1607	// supports local aliases, there will be neither CC1
1608	// -fsemantic-interposition nor -fhalf-no-semantic-interposition. Set
1609	// dso_local on the function if using a local alias is preferable (can avoid
1610	// PLT indirection).
1611	if (!(isa<llvm::Function>(Val: GV) && GV->canBenefitFromLocalAlias()))
1612	return false;
1613	return !(CGM.getLangOpts().SemanticInterposition ||
1614	CGM.getLangOpts().HalfNoSemanticInterposition);
1615	}
1616
1617	// A definition cannot be preempted from an executable.
1618	if (!GV->isDeclarationForLinker())
1619	return true;
1620
1621	// Most PIC code sequences that assume that a symbol is local cannot produce a
1622	// 0 if it turns out the symbol is undefined. While this is ABI and relocation
1623	// depended, it seems worth it to handle it here.
1624	if (RM == llvm::Reloc::PIC_ && GV->hasExternalWeakLinkage())
1625	return false;
1626
1627	// PowerPC64 prefers TOC indirection to avoid copy relocations.
1628	if (TT.isPPC64())
1629	return false;
1630
1631	if (CGOpts.DirectAccessExternalData) {
1632	// If -fdirect-access-external-data (default for -fno-pic), set dso_local
1633	// for non-thread-local variables. If the symbol is not defined in the
1634	// executable, a copy relocation will be needed at link time. dso_local is
1635	// excluded for thread-local variables because they generally don't support
1636	// copy relocations.
1637	if (auto *Var = dyn_cast<llvm::GlobalVariable>(Val: GV))
1638	if (!Var->isThreadLocal())
1639	return true;
1640
1641	// -fno-pic sets dso_local on a function declaration to allow direct
1642	// accesses when taking its address (similar to a data symbol). If the
1643	// function is not defined in the executable, a canonical PLT entry will be
1644	// needed at link time. -fno-direct-access-external-data can avoid the
1645	// canonical PLT entry. We don't generalize this condition to -fpie/-fpic as
1646	// it could just cause trouble without providing perceptible benefits.
1647	if (isa<llvm::Function>(Val: GV) && !CGOpts.NoPLT && RM == llvm::Reloc::Static)
1648	return true;
1649	}
1650
1651	// If we can use copy relocations we can assume it is local.
1652
1653	// Otherwise don't assume it is local.
1654	return false;
1655	}
1656
1657	void CodeGenModule::setDSOLocal(llvm::GlobalValue *GV) const {
1658	GV->setDSOLocal(shouldAssumeDSOLocal(CGM: *this, GV));
1659	}
1660
1661	void CodeGenModule::setDLLImportDLLExport(llvm::GlobalValue *GV,
1662	GlobalDecl GD) const {
1663	const auto *D = dyn_cast<NamedDecl>(Val: GD.getDecl());
1664	// C++ destructors have a few C++ ABI specific special cases.
1665	if (const auto *Dtor = dyn_cast_or_null<CXXDestructorDecl>(Val: D)) {
1666	getCXXABI().setCXXDestructorDLLStorage(GV, Dtor, DT: GD.getDtorType());
1667	return;
1668	}
1669	setDLLImportDLLExport(GV, D);
1670	}
1671
1672	void CodeGenModule::setDLLImportDLLExport(llvm::GlobalValue *GV,
1673	const NamedDecl *D) const {
1674	if (D && D->isExternallyVisible()) {
1675	if (D->hasAttr<DLLImportAttr>())
1676	GV->setDLLStorageClass(llvm::GlobalVariable::DLLImportStorageClass);
1677	else if ((D->hasAttr<DLLExportAttr>() ||
1678	shouldMapVisibilityToDLLExport(D)) &&
1679	!GV->isDeclarationForLinker())
1680	GV->setDLLStorageClass(llvm::GlobalVariable::DLLExportStorageClass);
1681	}
1682	}
1683
1684	void CodeGenModule::setGVProperties(llvm::GlobalValue *GV,
1685	GlobalDecl GD) const {
1686	setDLLImportDLLExport(GV, GD);
1687	setGVPropertiesAux(GV, D: dyn_cast<NamedDecl>(Val: GD.getDecl()));
1688	}
1689
1690	void CodeGenModule::setGVProperties(llvm::GlobalValue *GV,
1691	const NamedDecl *D) const {
1692	setDLLImportDLLExport(GV, D);
1693	setGVPropertiesAux(GV, D);
1694	}
1695
1696	void CodeGenModule::setGVPropertiesAux(llvm::GlobalValue *GV,
1697	const NamedDecl *D) const {
1698	setGlobalVisibility(GV, D);
1699	setDSOLocal(GV);
1700	GV->setPartition(CodeGenOpts.SymbolPartition);
1701	}
1702
1703	static llvm::GlobalVariable::ThreadLocalMode GetLLVMTLSModel(StringRef S) {
1704	return llvm::StringSwitch<llvm::GlobalVariable::ThreadLocalMode>(S)
1705	.Case(S: "global-dynamic", Value: llvm::GlobalVariable::GeneralDynamicTLSModel)
1706	.Case(S: "local-dynamic", Value: llvm::GlobalVariable::LocalDynamicTLSModel)
1707	.Case(S: "initial-exec", Value: llvm::GlobalVariable::InitialExecTLSModel)
1708	.Case(S: "local-exec", Value: llvm::GlobalVariable::LocalExecTLSModel);
1709	}
1710
1711	llvm::GlobalVariable::ThreadLocalMode
1712	CodeGenModule::GetDefaultLLVMTLSModel() const {
1713	switch (CodeGenOpts.getDefaultTLSModel()) {
1714	case CodeGenOptions::GeneralDynamicTLSModel:
1715	return llvm::GlobalVariable::GeneralDynamicTLSModel;
1716	case CodeGenOptions::LocalDynamicTLSModel:
1717	return llvm::GlobalVariable::LocalDynamicTLSModel;
1718	case CodeGenOptions::InitialExecTLSModel:
1719	return llvm::GlobalVariable::InitialExecTLSModel;
1720	case CodeGenOptions::LocalExecTLSModel:
1721	return llvm::GlobalVariable::LocalExecTLSModel;
1722	}
1723	llvm_unreachable("Invalid TLS model!");
1724	}
1725
1726	void CodeGenModule::setTLSMode(llvm::GlobalValue *GV, const VarDecl &D) const {
1727	assert(D.getTLSKind() && "setting TLS mode on non-TLS var!");
1728
1729	llvm::GlobalValue::ThreadLocalMode TLM;
1730	TLM = GetDefaultLLVMTLSModel();
1731
1732	// Override the TLS model if it is explicitly specified.
1733	if (const TLSModelAttr *Attr = D.getAttr<TLSModelAttr>()) {
1734	TLM = GetLLVMTLSModel(Attr->getModel());
1735	}
1736
1737	GV->setThreadLocalMode(TLM);
1738	}
1739
1740	static std::string getCPUSpecificMangling(const CodeGenModule &CGM,
1741	StringRef Name) {
1742	const TargetInfo &Target = CGM.getTarget();
1743	return (Twine('.') + Twine(Target.CPUSpecificManglingCharacter(Name))).str();
1744	}
1745
1746	static void AppendCPUSpecificCPUDispatchMangling(const CodeGenModule &CGM,
1747	const CPUSpecificAttr *Attr,
1748	unsigned CPUIndex,
1749	raw_ostream &Out) {
1750	// cpu_specific gets the current name, dispatch gets the resolver if IFunc is
1751	// supported.
1752	if (Attr)
1753	Out << getCPUSpecificMangling(CGM, Attr->getCPUName(CPUIndex)->getName());
1754	else if (CGM.getTarget().supportsIFunc())
1755	Out << ".resolver";
1756	}
1757
1758	// Returns true if GD is a function decl with internal linkage and
1759	// needs a unique suffix after the mangled name.
1760	static bool isUniqueInternalLinkageDecl(GlobalDecl GD,
1761	CodeGenModule &CGM) {
1762	const Decl *D = GD.getDecl();
1763	return !CGM.getModuleNameHash().empty() && isa<FunctionDecl>(Val: D) &&
1764	(CGM.getFunctionLinkage(GD) == llvm::GlobalValue::InternalLinkage);
1765	}
1766
1767	static std::string getMangledNameImpl(CodeGenModule &CGM, GlobalDecl GD,
1768	const NamedDecl *ND,
1769	bool OmitMultiVersionMangling = false) {
1770	SmallString<256> Buffer;
1771	llvm::raw_svector_ostream Out(Buffer);
1772	MangleContext &MC = CGM.getCXXABI().getMangleContext();
1773	if (!CGM.getModuleNameHash().empty())
1774	MC.needsUniqueInternalLinkageNames();
1775	bool ShouldMangle = MC.shouldMangleDeclName(D: ND);
1776	if (ShouldMangle)
1777	MC.mangleName(GD: GD.getWithDecl(ND), Out);
1778	else {
1779	IdentifierInfo *II = ND->getIdentifier();
1780	assert(II && "Attempt to mangle unnamed decl.");
1781	const auto *FD = dyn_cast<FunctionDecl>(Val: ND);
1782
1783	if (FD &&
1784	FD->getType()->castAs<FunctionType>()->getCallConv() == CC_X86RegCall) {
1785	if (CGM.getLangOpts().RegCall4)
1786	Out << "__regcall4__" << II->getName();
1787	else
1788	Out << "__regcall3__" << II->getName();
1789	} else if (FD && FD->hasAttr<CUDAGlobalAttr>() &&
1790	GD.getKernelReferenceKind() == KernelReferenceKind::Stub) {
1791	Out << "__device_stub__" << II->getName();
1792	} else {
1793	Out << II->getName();
1794	}
1795	}
1796
1797	// Check if the module name hash should be appended for internal linkage
1798	// symbols. This should come before multi-version target suffixes are
1799	// appended. This is to keep the name and module hash suffix of the
1800	// internal linkage function together. The unique suffix should only be
1801	// added when name mangling is done to make sure that the final name can
1802	// be properly demangled. For example, for C functions without prototypes,
1803	// name mangling is not done and the unique suffix should not be appeneded
1804	// then.
1805	if (ShouldMangle && isUniqueInternalLinkageDecl(GD, CGM)) {
1806	assert(CGM.getCodeGenOpts().UniqueInternalLinkageNames &&
1807	"Hash computed when not explicitly requested");
1808	Out << CGM.getModuleNameHash();
1809	}
1810
1811	if (const auto *FD = dyn_cast<FunctionDecl>(Val: ND))
1812	if (FD->isMultiVersion() && !OmitMultiVersionMangling) {
1813	switch (FD->getMultiVersionKind()) {
1814	case MultiVersionKind::CPUDispatch:
1815	case MultiVersionKind::CPUSpecific:
1816	AppendCPUSpecificCPUDispatchMangling(CGM,
1817	FD->getAttr<CPUSpecificAttr>(),
1818	GD.getMultiVersionIndex(), Out);
1819	break;
1820	case MultiVersionKind::Target: {
1821	auto *Attr = FD->getAttr<TargetAttr>();
1822	const ABIInfo &Info = CGM.getTargetCodeGenInfo().getABIInfo();
1823	Info.appendAttributeMangling(Attr, Out);
1824	break;
1825	}
1826	case MultiVersionKind::TargetVersion: {
1827	auto *Attr = FD->getAttr<TargetVersionAttr>();
1828	const ABIInfo &Info = CGM.getTargetCodeGenInfo().getABIInfo();
1829	Info.appendAttributeMangling(Attr, Out);
1830	break;
1831	}
1832	case MultiVersionKind::TargetClones: {
1833	auto *Attr = FD->getAttr<TargetClonesAttr>();
1834	unsigned Index = GD.getMultiVersionIndex();
1835	const ABIInfo &Info = CGM.getTargetCodeGenInfo().getABIInfo();
1836	Info.appendAttributeMangling(Attr, Index, Out);
1837	break;
1838	}
1839	case MultiVersionKind::None:
1840	llvm_unreachable("None multiversion type isn't valid here");
1841	}
1842	}
1843
1844	// Make unique name for device side static file-scope variable for HIP.
1845	if (CGM.getContext().shouldExternalize(ND) &&
1846	CGM.getLangOpts().GPURelocatableDeviceCode &&
1847	CGM.getLangOpts().CUDAIsDevice)
1848	CGM.printPostfixForExternalizedDecl(Out, ND);
1849
1850	return std::string(Out.str());
1851	}
1852
1853	void CodeGenModule::UpdateMultiVersionNames(GlobalDecl GD,
1854	const FunctionDecl *FD,
1855	StringRef &CurName) {
1856	if (!FD->isMultiVersion())
1857	return;
1858
1859	// Get the name of what this would be without the 'target' attribute. This
1860	// allows us to lookup the version that was emitted when this wasn't a
1861	// multiversion function.
1862	std::string NonTargetName =
1863	getMangledNameImpl(*this, GD, FD, /*OmitMultiVersionMangling=*/true);
1864	GlobalDecl OtherGD;
1865	if (lookupRepresentativeDecl(MangledName: NonTargetName, Result&: OtherGD)) {
1866	assert(OtherGD.getCanonicalDecl()
1867	.getDecl()
1868	->getAsFunction()
1869	->isMultiVersion() &&
1870	"Other GD should now be a multiversioned function");
1871	// OtherFD is the version of this function that was mangled BEFORE
1872	// becoming a MultiVersion function. It potentially needs to be updated.
1873	const FunctionDecl *OtherFD = OtherGD.getCanonicalDecl()
1874	.getDecl()
1875	->getAsFunction()
1876	->getMostRecentDecl();
1877	std::string OtherName = getMangledNameImpl(*this, OtherGD, OtherFD);
1878	// This is so that if the initial version was already the 'default'
1879	// version, we don't try to update it.
1880	if (OtherName != NonTargetName) {
1881	// Remove instead of erase, since others may have stored the StringRef
1882	// to this.
1883	const auto ExistingRecord = Manglings.find(Key: NonTargetName);
1884	if (ExistingRecord != std::end(cont&: Manglings))
1885	Manglings.remove(KeyValue: &(*ExistingRecord));
1886	auto Result = Manglings.insert(KV: std::make_pair(x&: OtherName, y&: OtherGD));
1887	StringRef OtherNameRef = MangledDeclNames[OtherGD.getCanonicalDecl()] =
1888	Result.first->first();
1889	// If this is the current decl is being created, make sure we update the name.
1890	if (GD.getCanonicalDecl() == OtherGD.getCanonicalDecl())
1891	CurName = OtherNameRef;
1892	if (llvm::GlobalValue *Entry = GetGlobalValue(Ref: NonTargetName))
1893	Entry->setName(OtherName);
1894	}
1895	}
1896	}
1897
1898	StringRef CodeGenModule::getMangledName(GlobalDecl GD) {
1899	GlobalDecl CanonicalGD = GD.getCanonicalDecl();
1900
1901	// Some ABIs don't have constructor variants. Make sure that base and
1902	// complete constructors get mangled the same.
1903	if (const auto *CD = dyn_cast<CXXConstructorDecl>(Val: CanonicalGD.getDecl())) {
1904	if (!getTarget().getCXXABI().hasConstructorVariants()) {
1905	CXXCtorType OrigCtorType = GD.getCtorType();
1906	assert(OrigCtorType == Ctor_Base || OrigCtorType == Ctor_Complete);
1907	if (OrigCtorType == Ctor_Base)
1908	CanonicalGD = GlobalDecl(CD, Ctor_Complete);
1909	}
1910	}
1911
1912	// In CUDA/HIP device compilation with -fgpu-rdc, the mangled name of a
1913	// static device variable depends on whether the variable is referenced by
1914	// a host or device host function. Therefore the mangled name cannot be
1915	// cached.
1916	if (!LangOpts.CUDAIsDevice || !getContext().mayExternalize(D: GD.getDecl())) {
1917	auto FoundName = MangledDeclNames.find(Key: CanonicalGD);
1918	if (FoundName != MangledDeclNames.end())
1919	return FoundName->second;
1920	}
1921
1922	// Keep the first result in the case of a mangling collision.
1923	const auto *ND = cast<NamedDecl>(Val: GD.getDecl());
1924	std::string MangledName = getMangledNameImpl(CGM&: *this, GD, ND);
1925
1926	// Ensure either we have different ABIs between host and device compilations,
1927	// says host compilation following MSVC ABI but device compilation follows
1928	// Itanium C++ ABI or, if they follow the same ABI, kernel names after
1929	// mangling should be the same after name stubbing. The later checking is
1930	// very important as the device kernel name being mangled in host-compilation
1931	// is used to resolve the device binaries to be executed. Inconsistent naming
1932	// result in undefined behavior. Even though we cannot check that naming
1933	// directly between host- and device-compilations, the host- and
1934	// device-mangling in host compilation could help catching certain ones.
1935	assert(!isa<FunctionDecl>(ND) || !ND->hasAttr<CUDAGlobalAttr>() ||
1936	getContext().shouldExternalize(ND) || getLangOpts().CUDAIsDevice ||
1937	(getContext().getAuxTargetInfo() &&
1938	(getContext().getAuxTargetInfo()->getCXXABI() !=
1939	getContext().getTargetInfo().getCXXABI())) ||
1940	getCUDARuntime().getDeviceSideName(ND) ==
1941	getMangledNameImpl(
1942	*this,
1943	GD.getWithKernelReferenceKind(KernelReferenceKind::Kernel),
1944	ND));
1945
1946	auto Result = Manglings.insert(KV: std::make_pair(x&: MangledName, y&: GD));
1947	return MangledDeclNames[CanonicalGD] = Result.first->first();
1948	}
1949
1950	StringRef CodeGenModule::getBlockMangledName(GlobalDecl GD,
1951	const BlockDecl *BD) {
1952	MangleContext &MangleCtx = getCXXABI().getMangleContext();
1953	const Decl *D = GD.getDecl();
1954
1955	SmallString<256> Buffer;
1956	llvm::raw_svector_ostream Out(Buffer);
1957	if (!D)
1958	MangleCtx.mangleGlobalBlock(BD,
1959	dyn_cast_or_null<VarDecl>(Val: initializedGlobalDecl.getDecl()), Out);
1960	else if (const auto *CD = dyn_cast<CXXConstructorDecl>(Val: D))
1961	MangleCtx.mangleCtorBlock(CD, CT: GD.getCtorType(), BD, Out);
1962	else if (const auto *DD = dyn_cast<CXXDestructorDecl>(Val: D))
1963	MangleCtx.mangleDtorBlock(CD: DD, DT: GD.getDtorType(), BD, Out);
1964	else
1965	MangleCtx.mangleBlock(DC: cast<DeclContext>(Val: D), BD, Out);
1966
1967	auto Result = Manglings.insert(KV: std::make_pair(x: Out.str(), y&: BD));
1968	return Result.first->first();
1969	}
1970
1971	const GlobalDecl CodeGenModule::getMangledNameDecl(StringRef Name) {
1972	auto it = MangledDeclNames.begin();
1973	while (it != MangledDeclNames.end()) {
1974	if (it->second == Name)
1975	return it->first;
1976	it++;
1977	}
1978	return GlobalDecl();
1979	}
1980
1981	llvm::GlobalValue *CodeGenModule::GetGlobalValue(StringRef Name) {
1982	return getModule().getNamedValue(Name);
1983	}
1984
1985	/// AddGlobalCtor - Add a function to the list that will be called before
1986	/// main() runs.
1987	void CodeGenModule::AddGlobalCtor(llvm::Function *Ctor, int Priority,
1988	unsigned LexOrder,
1989	llvm::Constant *AssociatedData) {
1990	// FIXME: Type coercion of void()* types.
1991	GlobalCtors.push_back(x: Structor(Priority, LexOrder, Ctor, AssociatedData));
1992	}
1993
1994	/// AddGlobalDtor - Add a function to the list that will be called
1995	/// when the module is unloaded.
1996	void CodeGenModule::AddGlobalDtor(llvm::Function *Dtor, int Priority,
1997	bool IsDtorAttrFunc) {
1998	if (CodeGenOpts.RegisterGlobalDtorsWithAtExit &&
1999	(!getContext().getTargetInfo().getTriple().isOSAIX() || IsDtorAttrFunc)) {
2000	DtorsUsingAtExit[Priority].push_back(NewVal: Dtor);
2001	return;
2002	}
2003
2004	// FIXME: Type coercion of void()* types.
2005	GlobalDtors.push_back(x: Structor(Priority, ~0U, Dtor, nullptr));
2006	}
2007
2008	void CodeGenModule::EmitCtorList(CtorList &Fns, const char *GlobalName) {
2009	if (Fns.empty()) return;
2010
2011	// Ctor function type is void()*.
2012	llvm::FunctionType* CtorFTy = llvm::FunctionType::get(Result: VoidTy, isVarArg: false);
2013	llvm::Type *CtorPFTy = llvm::PointerType::get(ElementType: CtorFTy,
2014	AddressSpace: TheModule.getDataLayout().getProgramAddressSpace());
2015
2016	// Get the type of a ctor entry, { i32, void ()*, i8* }.
2017	llvm::StructType *CtorStructTy = llvm::StructType::get(
2018	elt1: Int32Ty, elts: CtorPFTy, elts: VoidPtrTy);
2019
2020	// Construct the constructor and destructor arrays.
2021	ConstantInitBuilder builder(*this);
2022	auto ctors = builder.beginArray(eltTy: CtorStructTy);
2023	for (const auto &I : Fns) {
2024	auto ctor = ctors.beginStruct(ty: CtorStructTy);
2025	ctor.addInt(intTy: Int32Ty, value: I.Priority);
2026	ctor.add(value: I.Initializer);
2027	if (I.AssociatedData)
2028	ctor.add(value: I.AssociatedData);
2029	else
2030	ctor.addNullPointer(ptrTy: VoidPtrTy);
2031	ctor.finishAndAddTo(parent&: ctors);
2032	}
2033
2034	auto list =
2035	ctors.finishAndCreateGlobal(GlobalName, getPointerAlign(),
2036	/*constant*/ false,
2037	llvm::GlobalValue::AppendingLinkage);
2038
2039	// The LTO linker doesn't seem to like it when we set an alignment
2040	// on appending variables. Take it off as a workaround.
2041	list->setAlignment(std::nullopt);
2042
2043	Fns.clear();
2044	}
2045
2046	llvm::GlobalValue::LinkageTypes
2047	CodeGenModule::getFunctionLinkage(GlobalDecl GD) {
2048	const auto *D = cast<FunctionDecl>(Val: GD.getDecl());
2049
2050	GVALinkage Linkage = getContext().GetGVALinkageForFunction(FD: D);
2051
2052	if (const auto *Dtor = dyn_cast<CXXDestructorDecl>(Val: D))
2053	return getCXXABI().getCXXDestructorLinkage(Linkage, Dtor, DT: GD.getDtorType());
2054
2055	return getLLVMLinkageForDeclarator(D, Linkage);
2056	}
2057
2058	llvm::ConstantInt *CodeGenModule::CreateCrossDsoCfiTypeId(llvm::Metadata *MD) {
2059	llvm::MDString *MDS = dyn_cast<llvm::MDString>(Val: MD);
2060	if (!MDS) return nullptr;
2061
2062	return llvm::ConstantInt::get(Ty: Int64Ty, V: llvm::MD5Hash(Str: MDS->getString()));
2063	}
2064
2065	llvm::ConstantInt *CodeGenModule::CreateKCFITypeId(QualType T) {
2066	if (auto *FnType = T->getAs<FunctionProtoType>())
2067	T = getContext().getFunctionType(
2068	ResultTy: FnType->getReturnType(), Args: FnType->getParamTypes(),
2069	EPI: FnType->getExtProtoInfo().withExceptionSpec(ESI: EST_None));
2070
2071	std::string OutName;
2072	llvm::raw_string_ostream Out(OutName);
2073	getCXXABI().getMangleContext().mangleCanonicalTypeName(
2074	T, Out, NormalizeIntegers: getCodeGenOpts().SanitizeCfiICallNormalizeIntegers);
2075
2076	if (getCodeGenOpts().SanitizeCfiICallNormalizeIntegers)
2077	Out << ".normalized";
2078
2079	return llvm::ConstantInt::get(Ty: Int32Ty,
2080	V: static_cast<uint32_t>(llvm::xxHash64(Data: OutName)));
2081	}
2082
2083	void CodeGenModule::SetLLVMFunctionAttributes(GlobalDecl GD,
2084	const CGFunctionInfo &Info,
2085	llvm::Function *F, bool IsThunk) {
2086	unsigned CallingConv;
2087	llvm::AttributeList PAL;
2088	ConstructAttributeList(Name: F->getName(), Info, CalleeInfo: GD, Attrs&: PAL, CallingConv,
2089	/*AttrOnCallSite=*/false, IsThunk);
2090	if (CallingConv == llvm::CallingConv::X86_VectorCall &&
2091	getTarget().getTriple().isWindowsArm64EC()) {
2092	SourceLocation Loc;
2093	if (const Decl *D = GD.getDecl())
2094	Loc = D->getLocation();
2095
2096	Error(loc: Loc, message: "__vectorcall calling convention is not currently supported");
2097	}
2098	F->setAttributes(PAL);
2099	F->setCallingConv(static_cast<llvm::CallingConv::ID>(CallingConv));
2100	}
2101
2102	static void removeImageAccessQualifier(std::string& TyName) {
2103	std::string ReadOnlyQual("__read_only");
2104	std::string::size_type ReadOnlyPos = TyName.find(str: ReadOnlyQual);
2105	if (ReadOnlyPos != std::string::npos)
2106	// "+ 1" for the space after access qualifier.
2107	TyName.erase(pos: ReadOnlyPos, n: ReadOnlyQual.size() + 1);
2108	else {
2109	std::string WriteOnlyQual("__write_only");
2110	std::string::size_type WriteOnlyPos = TyName.find(str: WriteOnlyQual);
2111	if (WriteOnlyPos != std::string::npos)
2112	TyName.erase(pos: WriteOnlyPos, n: WriteOnlyQual.size() + 1);
2113	else {
2114	std::string ReadWriteQual("__read_write");
2115	std::string::size_type ReadWritePos = TyName.find(str: ReadWriteQual);
2116	if (ReadWritePos != std::string::npos)
2117	TyName.erase(pos: ReadWritePos, n: ReadWriteQual.size() + 1);
2118	}
2119	}
2120	}
2121
2122	// Returns the address space id that should be produced to the
2123	// kernel_arg_addr_space metadata. This is always fixed to the ids
2124	// as specified in the SPIR 2.0 specification in order to differentiate
2125	// for example in clGetKernelArgInfo() implementation between the address
2126	// spaces with targets without unique mapping to the OpenCL address spaces
2127	// (basically all single AS CPUs).
2128	static unsigned ArgInfoAddressSpace(LangAS AS) {
2129	switch (AS) {
2130	case LangAS::opencl_global:
2131	return 1;
2132	case LangAS::opencl_constant:
2133	return 2;
2134	case LangAS::opencl_local:
2135	return 3;
2136	case LangAS::opencl_generic:
2137	return 4; // Not in SPIR 2.0 specs.
2138	case LangAS::opencl_global_device:
2139	return 5;
2140	case LangAS::opencl_global_host:
2141	return 6;
2142	default:
2143	return 0; // Assume private.
2144	}
2145	}
2146
2147	void CodeGenModule::GenKernelArgMetadata(llvm::Function *Fn,
2148	const FunctionDecl *FD,
2149	CodeGenFunction *CGF) {
2150	assert(((FD && CGF) || (!FD && !CGF)) &&
2151	"Incorrect use - FD and CGF should either be both null or not!");
2152	// Create MDNodes that represent the kernel arg metadata.
2153	// Each MDNode is a list in the form of "key", N number of values which is
2154	// the same number of values as their are kernel arguments.
2155
2156	const PrintingPolicy &Policy = Context.getPrintingPolicy();
2157
2158	// MDNode for the kernel argument address space qualifiers.
2159	SmallVector<llvm::Metadata *, 8> addressQuals;
2160
2161	// MDNode for the kernel argument access qualifiers (images only).
2162	SmallVector<llvm::Metadata *, 8> accessQuals;
2163
2164	// MDNode for the kernel argument type names.
2165	SmallVector<llvm::Metadata *, 8> argTypeNames;
2166
2167	// MDNode for the kernel argument base type names.
2168	SmallVector<llvm::Metadata *, 8> argBaseTypeNames;
2169
2170	// MDNode for the kernel argument type qualifiers.
2171	SmallVector<llvm::Metadata *, 8> argTypeQuals;
2172
2173	// MDNode for the kernel argument names.
2174	SmallVector<llvm::Metadata *, 8> argNames;
2175
2176	if (FD && CGF)
2177	for (unsigned i = 0, e = FD->getNumParams(); i != e; ++i) {
2178	const ParmVarDecl *parm = FD->getParamDecl(i);
2179	// Get argument name.
2180	argNames.push_back(Elt: llvm::MDString::get(VMContext, parm->getName()));
2181
2182	if (!getLangOpts().OpenCL)
2183	continue;
2184	QualType ty = parm->getType();
2185	std::string typeQuals;
2186
2187	// Get image and pipe access qualifier:
2188	if (ty->isImageType() || ty->isPipeType()) {
2189	const Decl *PDecl = parm;
2190	if (const auto *TD = ty->getAs<TypedefType>())
2191	PDecl = TD->getDecl();
2192	const OpenCLAccessAttr *A = PDecl->getAttr<OpenCLAccessAttr>();
2193	if (A && A->isWriteOnly())
2194	accessQuals.push_back(Elt: llvm::MDString::get(Context&: VMContext, Str: "write_only"));
2195	else if (A && A->isReadWrite())
2196	accessQuals.push_back(Elt: llvm::MDString::get(Context&: VMContext, Str: "read_write"));
2197	else
2198	accessQuals.push_back(Elt: llvm::MDString::get(Context&: VMContext, Str: "read_only"));
2199	} else
2200	accessQuals.push_back(Elt: llvm::MDString::get(Context&: VMContext, Str: "none"));
2201
2202	auto getTypeSpelling = [&](QualType Ty) {
2203	auto typeName = Ty.getUnqualifiedType().getAsString(Policy);
2204
2205	if (Ty.isCanonical()) {
2206	StringRef typeNameRef = typeName;
2207	// Turn "unsigned type" to "utype"
2208	if (typeNameRef.consume_front(Prefix: "unsigned "))
2209	return std::string("u") + typeNameRef.str();
2210	if (typeNameRef.consume_front(Prefix: "signed "))
2211	return typeNameRef.str();
2212	}
2213
2214	return typeName;
2215	};
2216
2217	if (ty->isPointerType()) {
2218	QualType pointeeTy = ty->getPointeeType();
2219
2220	// Get address qualifier.
2221	addressQuals.push_back(
2222	Elt: llvm::ConstantAsMetadata::get(C: CGF->Builder.getInt32(
2223	C: ArgInfoAddressSpace(AS: pointeeTy.getAddressSpace()))));
2224
2225	// Get argument type name.
2226	std::string typeName = getTypeSpelling(pointeeTy) + "*";
2227	std::string baseTypeName =
2228	getTypeSpelling(pointeeTy.getCanonicalType()) + "*";
2229	argTypeNames.push_back(Elt: llvm::MDString::get(Context&: VMContext, Str: typeName));
2230	argBaseTypeNames.push_back(
2231	Elt: llvm::MDString::get(Context&: VMContext, Str: baseTypeName));
2232
2233	// Get argument type qualifiers:
2234	if (ty.isRestrictQualified())
2235	typeQuals = "restrict";
2236	if (pointeeTy.isConstQualified() ||
2237	(pointeeTy.getAddressSpace() == LangAS::opencl_constant))
2238	typeQuals += typeQuals.empty() ? "const" : " const";
2239	if (pointeeTy.isVolatileQualified())
2240	typeQuals += typeQuals.empty() ? "volatile" : " volatile";
2241	} else {
2242	uint32_t AddrSpc = 0;
2243	bool isPipe = ty->isPipeType();
2244	if (ty->isImageType() || isPipe)
2245	AddrSpc = ArgInfoAddressSpace(AS: LangAS::opencl_global);
2246
2247	addressQuals.push_back(
2248	Elt: llvm::ConstantAsMetadata::get(C: CGF->Builder.getInt32(C: AddrSpc)));
2249
2250	// Get argument type name.
2251	ty = isPipe ? ty->castAs<PipeType>()->getElementType() : ty;
2252	std::string typeName = getTypeSpelling(ty);
2253	std::string baseTypeName = getTypeSpelling(ty.getCanonicalType());
2254
2255	// Remove access qualifiers on images
2256	// (as they are inseparable from type in clang implementation,
2257	// but OpenCL spec provides a special query to get access qualifier
2258	// via clGetKernelArgInfo with CL_KERNEL_ARG_ACCESS_QUALIFIER):
2259	if (ty->isImageType()) {
2260	removeImageAccessQualifier(TyName&: typeName);
2261	removeImageAccessQualifier(TyName&: baseTypeName);
2262	}
2263
2264	argTypeNames.push_back(Elt: llvm::MDString::get(Context&: VMContext, Str: typeName));
2265	argBaseTypeNames.push_back(
2266	Elt: llvm::MDString::get(Context&: VMContext, Str: baseTypeName));
2267
2268	if (isPipe)
2269	typeQuals = "pipe";
2270	}
2271	argTypeQuals.push_back(Elt: llvm::MDString::get(Context&: VMContext, Str: typeQuals));
2272	}
2273
2274	if (getLangOpts().OpenCL) {
2275	Fn->setMetadata(Kind: "kernel_arg_addr_space",
2276	Node: llvm::MDNode::get(Context&: VMContext, MDs: addressQuals));
2277	Fn->setMetadata(Kind: "kernel_arg_access_qual",
2278	Node: llvm::MDNode::get(Context&: VMContext, MDs: accessQuals));
2279	Fn->setMetadata(Kind: "kernel_arg_type",
2280	Node: llvm::MDNode::get(Context&: VMContext, MDs: argTypeNames));
2281	Fn->setMetadata(Kind: "kernel_arg_base_type",
2282	Node: llvm::MDNode::get(Context&: VMContext, MDs: argBaseTypeNames));
2283	Fn->setMetadata(Kind: "kernel_arg_type_qual",
2284	Node: llvm::MDNode::get(Context&: VMContext, MDs: argTypeQuals));
2285	}
2286	if (getCodeGenOpts().EmitOpenCLArgMetadata ||
2287	getCodeGenOpts().HIPSaveKernelArgName)
2288	Fn->setMetadata(Kind: "kernel_arg_name",
2289	Node: llvm::MDNode::get(Context&: VMContext, MDs: argNames));
2290	}
2291
2292	/// Determines whether the language options require us to model
2293	/// unwind exceptions. We treat -fexceptions as mandating this
2294	/// except under the fragile ObjC ABI with only ObjC exceptions
2295	/// enabled. This means, for example, that C with -fexceptions
2296	/// enables this.
2297	static bool hasUnwindExceptions(const LangOptions &LangOpts) {
2298	// If exceptions are completely disabled, obviously this is false.
2299	if (!LangOpts.Exceptions) return false;
2300
2301	// If C++ exceptions are enabled, this is true.
2302	if (LangOpts.CXXExceptions) return true;
2303
2304	// If ObjC exceptions are enabled, this depends on the ABI.
2305	if (LangOpts.ObjCExceptions) {
2306	return LangOpts.ObjCRuntime.hasUnwindExceptions();
2307	}
2308
2309	return true;
2310	}
2311
2312	static bool requiresMemberFunctionPointerTypeMetadata(CodeGenModule &CGM,
2313	const CXXMethodDecl *MD) {
2314	// Check that the type metadata can ever actually be used by a call.
2315	if (!CGM.getCodeGenOpts().LTOUnit ||
2316	!CGM.HasHiddenLTOVisibility(RD: MD->getParent()))
2317	return false;
2318
2319	// Only functions whose address can be taken with a member function pointer
2320	// need this sort of type metadata.
2321	return MD->isImplicitObjectMemberFunction() && !MD->isVirtual() &&
2322	!isa<CXXConstructorDecl, CXXDestructorDecl>(Val: MD);
2323	}
2324
2325	SmallVector<const CXXRecordDecl *, 0>
2326	CodeGenModule::getMostBaseClasses(const CXXRecordDecl *RD) {
2327	llvm::SetVector<const CXXRecordDecl *> MostBases;
2328
2329	std::function<void (const CXXRecordDecl *)> CollectMostBases;
2330	CollectMostBases = [&](const CXXRecordDecl *RD) {
2331	if (RD->getNumBases() == 0)
2332	MostBases.insert(X: RD);
2333	for (const CXXBaseSpecifier &B : RD->bases())
2334	CollectMostBases(B.getType()->getAsCXXRecordDecl());
2335	};
2336	CollectMostBases(RD);
2337	return MostBases.takeVector();
2338	}
2339
2340	void CodeGenModule::SetLLVMFunctionAttributesForDefinition(const Decl *D,
2341	llvm::Function *F) {
2342	llvm::AttrBuilder B(F->getContext());
2343
2344	if ((!D || !D->hasAttr<NoUwtableAttr>()) && CodeGenOpts.UnwindTables)
2345	B.addUWTableAttr(Kind: llvm::UWTableKind(CodeGenOpts.UnwindTables));
2346
2347	if (CodeGenOpts.StackClashProtector)
2348	B.addAttribute(A: "probe-stack", V: "inline-asm");
2349
2350	if (CodeGenOpts.StackProbeSize && CodeGenOpts.StackProbeSize != 4096)
2351	B.addAttribute(A: "stack-probe-size",
2352	V: std::to_string(val: CodeGenOpts.StackProbeSize));
2353
2354	if (!hasUnwindExceptions(LangOpts))
2355	B.addAttribute(llvm::Attribute::NoUnwind);
2356
2357	if (D && D->hasAttr<NoStackProtectorAttr>())
2358	; // Do nothing.
2359	else if (D && D->hasAttr<StrictGuardStackCheckAttr>() &&
2360	isStackProtectorOn(LangOpts, getTriple(), LangOptions::SSPOn))
2361	B.addAttribute(llvm::Attribute::StackProtectStrong);
2362	else if (isStackProtectorOn(LangOpts, getTriple(), LangOptions::SSPOn))
2363	B.addAttribute(llvm::Attribute::StackProtect);
2364	else if (isStackProtectorOn(LangOpts, getTriple(), LangOptions::SSPStrong))
2365	B.addAttribute(llvm::Attribute::StackProtectStrong);
2366	else if (isStackProtectorOn(LangOpts, getTriple(), LangOptions::SSPReq))
2367	B.addAttribute(llvm::Attribute::StackProtectReq);
2368
2369	if (!D) {
2370	// If we don't have a declaration to control inlining, the function isn't
2371	// explicitly marked as alwaysinline for semantic reasons, and inlining is
2372	// disabled, mark the function as noinline.
2373	if (!F->hasFnAttribute(llvm::Attribute::AlwaysInline) &&
2374	CodeGenOpts.getInlining() == CodeGenOptions::OnlyAlwaysInlining)
2375	B.addAttribute(llvm::Attribute::NoInline);
2376
2377	F->addFnAttrs(Attrs: B);
2378	return;
2379	}
2380
2381	// Handle SME attributes that apply to function definitions,
2382	// rather than to function prototypes.
2383	if (D->hasAttr<ArmLocallyStreamingAttr>())
2384	B.addAttribute(A: "aarch64_pstate_sm_body");
2385
2386	if (auto *Attr = D->getAttr<ArmNewAttr>()) {
2387	if (Attr->isNewZA())
2388	B.addAttribute(A: "aarch64_new_za");
2389	if (Attr->isNewZT0())
2390	B.addAttribute(A: "aarch64_new_zt0");
2391	}
2392
2393	// Track whether we need to add the optnone LLVM attribute,
2394	// starting with the default for this optimization level.
2395	bool ShouldAddOptNone =
2396	!CodeGenOpts.DisableO0ImplyOptNone && CodeGenOpts.OptimizationLevel == 0;
2397	// We can't add optnone in the following cases, it won't pass the verifier.
2398	ShouldAddOptNone &= !D->hasAttr<MinSizeAttr>();
2399	ShouldAddOptNone &= !D->hasAttr<AlwaysInlineAttr>();
2400
2401	// Add optnone, but do so only if the function isn't always_inline.
2402	if ((ShouldAddOptNone || D->hasAttr<OptimizeNoneAttr>()) &&
2403	!F->hasFnAttribute(llvm::Attribute::AlwaysInline)) {
2404	B.addAttribute(llvm::Attribute::OptimizeNone);
2405
2406	// OptimizeNone implies noinline; we should not be inlining such functions.
2407	B.addAttribute(llvm::Attribute::NoInline);
2408
2409	// We still need to handle naked functions even though optnone subsumes
2410	// much of their semantics.
2411	if (D->hasAttr<NakedAttr>())
2412	B.addAttribute(llvm::Attribute::Naked);
2413
2414	// OptimizeNone wins over OptimizeForSize and MinSize.
2415	F->removeFnAttr(llvm::Attribute::OptimizeForSize);
2416	F->removeFnAttr(llvm::Attribute::MinSize);
2417	} else if (D->hasAttr<NakedAttr>()) {
2418	// Naked implies noinline: we should not be inlining such functions.
2419	B.addAttribute(llvm::Attribute::Naked);
2420	B.addAttribute(llvm::Attribute::NoInline);
2421	} else if (D->hasAttr<NoDuplicateAttr>()) {
2422	B.addAttribute(llvm::Attribute::NoDuplicate);
2423	} else if (D->hasAttr<NoInlineAttr>() && !F->hasFnAttribute(llvm::Attribute::AlwaysInline)) {
2424	// Add noinline if the function isn't always_inline.
2425	B.addAttribute(llvm::Attribute::NoInline);
2426	} else if (D->hasAttr<AlwaysInlineAttr>() &&
2427	!F->hasFnAttribute(llvm::Attribute::NoInline)) {
2428	// (noinline wins over always_inline, and we can't specify both in IR)
2429	B.addAttribute(llvm::Attribute::AlwaysInline);
2430	} else if (CodeGenOpts.getInlining() == CodeGenOptions::OnlyAlwaysInlining) {
2431	// If we're not inlining, then force everything that isn't always_inline to
2432	// carry an explicit noinline attribute.
2433	if (!F->hasFnAttribute(llvm::Attribute::AlwaysInline))
2434	B.addAttribute(llvm::Attribute::NoInline);
2435	} else {
2436	// Otherwise, propagate the inline hint attribute and potentially use its
2437	// absence to mark things as noinline.
2438	if (auto *FD = dyn_cast<FunctionDecl>(Val: D)) {
2439	// Search function and template pattern redeclarations for inline.
2440	auto CheckForInline = [](const FunctionDecl *FD) {
2441	auto CheckRedeclForInline = [](const FunctionDecl *Redecl) {
2442	return Redecl->isInlineSpecified();
2443	};
2444	if (any_of(FD->redecls(), CheckRedeclForInline))
2445	return true;
2446	const FunctionDecl *Pattern = FD->getTemplateInstantiationPattern();
2447	if (!Pattern)
2448	return false;
2449	return any_of(Pattern->redecls(), CheckRedeclForInline);
2450	};
2451	if (CheckForInline(FD)) {
2452	B.addAttribute(llvm::Attribute::InlineHint);
2453	} else if (CodeGenOpts.getInlining() ==
2454	CodeGenOptions::OnlyHintInlining &&
2455	!FD->isInlined() &&
2456	!F->hasFnAttribute(llvm::Attribute::AlwaysInline)) {
2457	B.addAttribute(llvm::Attribute::NoInline);
2458	}
2459	}
2460	}
2461
2462	// Add other optimization related attributes if we are optimizing this
2463	// function.
2464	if (!D->hasAttr<OptimizeNoneAttr>()) {
2465	if (D->hasAttr<ColdAttr>()) {
2466	if (!ShouldAddOptNone)
2467	B.addAttribute(llvm::Attribute::OptimizeForSize);
2468	B.addAttribute(llvm::Attribute::Cold);
2469	}
2470	if (D->hasAttr<HotAttr>())
2471	B.addAttribute(llvm::Attribute::Hot);
2472	if (D->hasAttr<MinSizeAttr>())
2473	B.addAttribute(llvm::Attribute::MinSize);
2474	}
2475
2476	F->addFnAttrs(Attrs: B);
2477
2478	unsigned alignment = D->getMaxAlignment() / Context.getCharWidth();
2479	if (alignment)
2480	F->setAlignment(llvm::Align(alignment));
2481
2482	if (!D->hasAttr<AlignedAttr>())
2483	if (LangOpts.FunctionAlignment)
2484	F->setAlignment(llvm::Align(1ull << LangOpts.FunctionAlignment));
2485
2486	// Some C++ ABIs require 2-byte alignment for member functions, in order to
2487	// reserve a bit for differentiating between virtual and non-virtual member
2488	// functions. If the current target's C++ ABI requires this and this is a
2489	// member function, set its alignment accordingly.
2490	if (getTarget().getCXXABI().areMemberFunctionsAligned()) {
2491	if (isa<CXXMethodDecl>(Val: D) && F->getPointerAlignment(DL: getDataLayout()) < 2)
2492	F->setAlignment(std::max(a: llvm::Align(2), b: F->getAlign().valueOrOne()));
2493	}
2494
2495	// In the cross-dso CFI mode with canonical jump tables, we want !type
2496	// attributes on definitions only.
2497	if (CodeGenOpts.SanitizeCfiCrossDso &&
2498	CodeGenOpts.SanitizeCfiCanonicalJumpTables) {
2499	if (auto *FD = dyn_cast<FunctionDecl>(Val: D)) {
2500	// Skip available_externally functions. They won't be codegen'ed in the
2501	// current module anyway.
2502	if (getContext().GetGVALinkageForFunction(FD) != GVA_AvailableExternally)
2503	CreateFunctionTypeMetadataForIcall(FD, F);
2504	}
2505	}
2506
2507	// Emit type metadata on member functions for member function pointer checks.
2508	// These are only ever necessary on definitions; we're guaranteed that the
2509	// definition will be present in the LTO unit as a result of LTO visibility.
2510	auto *MD = dyn_cast<CXXMethodDecl>(Val: D);
2511	if (MD && requiresMemberFunctionPointerTypeMetadata(CGM&: *this, MD)) {
2512	for (const CXXRecordDecl *Base : getMostBaseClasses(RD: MD->getParent())) {
2513	llvm::Metadata *Id =
2514	CreateMetadataIdentifierForType(T: Context.getMemberPointerType(
2515	T: MD->getType(), Cls: Context.getRecordType(Base).getTypePtr()));
2516	F->addTypeMetadata(Offset: 0, TypeID: Id);
2517	}
2518	}
2519	}
2520
2521	void CodeGenModule::SetCommonAttributes(GlobalDecl GD, llvm::GlobalValue *GV) {
2522	const Decl *D = GD.getDecl();
2523	if (isa_and_nonnull<NamedDecl>(Val: D))
2524	setGVProperties(GV, GD);
2525	else
2526	GV->setVisibility(llvm::GlobalValue::DefaultVisibility);
2527
2528	if (D && D->hasAttr<UsedAttr>())
2529	addUsedOrCompilerUsedGlobal(GV);
2530
2531	if (const auto *VD = dyn_cast_if_present<VarDecl>(Val: D);
2532	VD &&
2533	((CodeGenOpts.KeepPersistentStorageVariables &&
2534	(VD->getStorageDuration() == SD_Static ||
2535	VD->getStorageDuration() == SD_Thread)) ||
2536	(CodeGenOpts.KeepStaticConsts && VD->getStorageDuration() == SD_Static &&
2537	VD->getType().isConstQualified())))
2538	addUsedOrCompilerUsedGlobal(GV);
2539	}
2540
2541	bool CodeGenModule::GetCPUAndFeaturesAttributes(GlobalDecl GD,
2542	llvm::AttrBuilder &Attrs,
2543	bool SetTargetFeatures) {
2544	// Add target-cpu and target-features attributes to functions. If
2545	// we have a decl for the function and it has a target attribute then
2546	// parse that and add it to the feature set.
2547	StringRef TargetCPU = getTarget().getTargetOpts().CPU;
2548	StringRef TuneCPU = getTarget().getTargetOpts().TuneCPU;
2549	std::vector<std::string> Features;
2550	const auto *FD = dyn_cast_or_null<FunctionDecl>(Val: GD.getDecl());
2551	FD = FD ? FD->getMostRecentDecl() : FD;
2552	const auto *TD = FD ? FD->getAttr<TargetAttr>() : nullptr;
2553	const auto *TV = FD ? FD->getAttr<TargetVersionAttr>() : nullptr;
2554	assert((!TD || !TV) && "both target_version and target specified");
2555	const auto *SD = FD ? FD->getAttr<CPUSpecificAttr>() : nullptr;
2556	const auto *TC = FD ? FD->getAttr<TargetClonesAttr>() : nullptr;
2557	bool AddedAttr = false;
2558	if (TD || TV || SD || TC) {
2559	llvm::StringMap<bool> FeatureMap;
2560	getContext().getFunctionFeatureMap(FeatureMap, GD);
2561
2562	// Produce the canonical string for this set of features.
2563	for (const llvm::StringMap<bool>::value_type &Entry : FeatureMap)
2564	Features.push_back(x: (Entry.getValue() ? "+" : "-") + Entry.getKey().str());
2565
2566	// Now add the target-cpu and target-features to the function.
2567	// While we populated the feature map above, we still need to
2568	// get and parse the target attribute so we can get the cpu for
2569	// the function.
2570	if (TD) {
2571	ParsedTargetAttr ParsedAttr =
2572	Target.parseTargetAttr(Str: TD->getFeaturesStr());
2573	if (!ParsedAttr.CPU.empty() &&
2574	getTarget().isValidCPUName(Name: ParsedAttr.CPU)) {
2575	TargetCPU = ParsedAttr.CPU;
2576	TuneCPU = ""; // Clear the tune CPU.
2577	}
2578	if (!ParsedAttr.Tune.empty() &&
2579	getTarget().isValidCPUName(Name: ParsedAttr.Tune))
2580	TuneCPU = ParsedAttr.Tune;
2581	}
2582
2583	if (SD) {
2584	// Apply the given CPU name as the 'tune-cpu' so that the optimizer can
2585	// favor this processor.
2586	TuneCPU = SD->getCPUName(GD.getMultiVersionIndex())->getName();
2587	}
2588	} else {
2589	// Otherwise just add the existing target cpu and target features to the
2590	// function.
2591	Features = getTarget().getTargetOpts().Features;
2592	}
2593
2594	if (!TargetCPU.empty()) {
2595	Attrs.addAttribute(A: "target-cpu", V: TargetCPU);
2596	AddedAttr = true;
2597	}
2598	if (!TuneCPU.empty()) {
2599	Attrs.addAttribute(A: "tune-cpu", V: TuneCPU);
2600	AddedAttr = true;
2601	}
2602	if (!Features.empty() && SetTargetFeatures) {
2603	llvm::erase_if(C&: Features, P: [&](const std::string& F) {
2604	return getTarget().isReadOnlyFeature(Feature: F.substr(pos: 1));
2605	});
2606	llvm::sort(C&: Features);
2607	Attrs.addAttribute(A: "target-features", V: llvm::join(R&: Features, Separator: ","));
2608	AddedAttr = true;
2609	}
2610
2611	return AddedAttr;
2612	}
2613
2614	void CodeGenModule::setNonAliasAttributes(GlobalDecl GD,
2615	llvm::GlobalObject *GO) {
2616	const Decl *D = GD.getDecl();
2617	SetCommonAttributes(GD, GV: GO);
2618
2619	if (D) {
2620	if (auto *GV = dyn_cast<llvm::GlobalVariable>(Val: GO)) {
2621	if (D->hasAttr<RetainAttr>())
2622	addUsedGlobal(GV);
2623	if (auto *SA = D->getAttr<PragmaClangBSSSectionAttr>())
2624	GV->addAttribute("bss-section", SA->getName());
2625	if (auto *SA = D->getAttr<PragmaClangDataSectionAttr>())
2626	GV->addAttribute("data-section", SA->getName());
2627	if (auto *SA = D->getAttr<PragmaClangRodataSectionAttr>())
2628	GV->addAttribute("rodata-section", SA->getName());
2629	if (auto *SA = D->getAttr<PragmaClangRelroSectionAttr>())
2630	GV->addAttribute("relro-section", SA->getName());
2631	}
2632
2633	if (auto *F = dyn_cast<llvm::Function>(Val: GO)) {
2634	if (D->hasAttr<RetainAttr>())
2635	addUsedGlobal(GV: F);
2636	if (auto *SA = D->getAttr<PragmaClangTextSectionAttr>())
2637	if (!D->getAttr<SectionAttr>())
2638	F->setSection(SA->getName());
2639
2640	llvm::AttrBuilder Attrs(F->getContext());
2641	if (GetCPUAndFeaturesAttributes(GD, Attrs)) {
2642	// We know that GetCPUAndFeaturesAttributes will always have the
2643	// newest set, since it has the newest possible FunctionDecl, so the
2644	// new ones should replace the old.
2645	llvm::AttributeMask RemoveAttrs;
2646	RemoveAttrs.addAttribute(A: "target-cpu");
2647	RemoveAttrs.addAttribute(A: "target-features");
2648	RemoveAttrs.addAttribute(A: "tune-cpu");
2649	F->removeFnAttrs(Attrs: RemoveAttrs);
2650	F->addFnAttrs(Attrs);
2651	}
2652	}
2653
2654	if (const auto *CSA = D->getAttr<CodeSegAttr>())
2655	GO->setSection(CSA->getName());
2656	else if (const auto *SA = D->getAttr<SectionAttr>())
2657	GO->setSection(SA->getName());
2658	}
2659
2660	getTargetCodeGenInfo().setTargetAttributes(D, GV: GO, M&: *this);
2661	}
2662
2663	void CodeGenModule::SetInternalFunctionAttributes(GlobalDecl GD,
2664	llvm::Function *F,
2665	const CGFunctionInfo &FI) {
2666	const Decl *D = GD.getDecl();
2667	SetLLVMFunctionAttributes(GD, Info: FI, F, /*IsThunk=*/false);
2668	SetLLVMFunctionAttributesForDefinition(D, F);
2669
2670	F->setLinkage(llvm::Function::InternalLinkage);
2671
2672	setNonAliasAttributes(GD, GO: F);
2673	}
2674
2675	static void setLinkageForGV(llvm::GlobalValue *GV, const NamedDecl *ND) {
2676	// Set linkage and visibility in case we never see a definition.
2677	LinkageInfo LV = ND->getLinkageAndVisibility();
2678	// Don't set internal linkage on declarations.
2679	// "extern_weak" is overloaded in LLVM; we probably should have
2680	// separate linkage types for this.
2681	if (isExternallyVisible(LV.getLinkage()) &&
2682	(ND->hasAttr<WeakAttr>() || ND->isWeakImported()))
2683	GV->setLinkage(llvm::GlobalValue::ExternalWeakLinkage);
2684	}
2685
2686	void CodeGenModule::CreateFunctionTypeMetadataForIcall(const FunctionDecl *FD,
2687	llvm::Function *F) {
2688	// Only if we are checking indirect calls.
2689	if (!LangOpts.Sanitize.has(K: SanitizerKind::CFIICall))
2690	return;
2691
2692	// Non-static class methods are handled via vtable or member function pointer
2693	// checks elsewhere.
2694	if (isa<CXXMethodDecl>(Val: FD) && !cast<CXXMethodDecl>(Val: FD)->isStatic())
2695	return;
2696
2697	llvm::Metadata *MD = CreateMetadataIdentifierForType(T: FD->getType());
2698	F->addTypeMetadata(Offset: 0, TypeID: MD);
2699	F->addTypeMetadata(Offset: 0, TypeID: CreateMetadataIdentifierGeneralized(T: FD->getType()));
2700
2701	// Emit a hash-based bit set entry for cross-DSO calls.
2702	if (CodeGenOpts.SanitizeCfiCrossDso)
2703	if (auto CrossDsoTypeId = CreateCrossDsoCfiTypeId(MD))
2704	F->addTypeMetadata(Offset: 0, TypeID: llvm::ConstantAsMetadata::get(C: CrossDsoTypeId));
2705	}
2706
2707	void CodeGenModule::setKCFIType(const FunctionDecl *FD, llvm::Function *F) {
2708	llvm::LLVMContext &Ctx = F->getContext();
2709	llvm::MDBuilder MDB(Ctx);
2710	F->setMetadata(llvm::LLVMContext::MD_kcfi_type,
2711	llvm::MDNode::get(
2712	Context&: Ctx, MDs: MDB.createConstant(C: CreateKCFITypeId(T: FD->getType()))));
2713	}
2714
2715	static bool allowKCFIIdentifier(StringRef Name) {
2716	// KCFI type identifier constants are only necessary for external assembly
2717	// functions, which means it's safe to skip unusual names. Subset of
2718	// MCAsmInfo::isAcceptableChar() and MCAsmInfoXCOFF::isAcceptableChar().
2719	return llvm::all_of(Range&: Name, P: [](const char &C) {
2720	return llvm::isAlnum(C) || C == '_' || C == '.';
2721	});
2722	}
2723
2724	void CodeGenModule::finalizeKCFITypes() {
2725	llvm::Module &M = getModule();
2726	for (auto &F : M.functions()) {
2727	// Remove KCFI type metadata from non-address-taken local functions.
2728	bool AddressTaken = F.hasAddressTaken();
2729	if (!AddressTaken && F.hasLocalLinkage())
2730	F.eraseMetadata(KindID: llvm::LLVMContext::MD_kcfi_type);
2731
2732	// Generate a constant with the expected KCFI type identifier for all
2733	// address-taken function declarations to support annotating indirectly
2734	// called assembly functions.
2735	if (!AddressTaken || !F.isDeclaration())
2736	continue;
2737
2738	const llvm::ConstantInt *Type;
2739	if (const llvm::MDNode *MD = F.getMetadata(KindID: llvm::LLVMContext::MD_kcfi_type))
2740	Type = llvm::mdconst::extract<llvm::ConstantInt>(MD: MD->getOperand(I: 0));
2741	else
2742	continue;
2743
2744	StringRef Name = F.getName();
2745	if (!allowKCFIIdentifier(Name))
2746	continue;
2747
2748	std::string Asm = (".weak __kcfi_typeid_" + Name + "\n.set __kcfi_typeid_" +
2749	Name + ", " + Twine(Type->getZExtValue()) + "\n")
2750	.str();
2751	M.appendModuleInlineAsm(Asm);
2752	}
2753	}
2754
2755	void CodeGenModule::SetFunctionAttributes(GlobalDecl GD, llvm::Function *F,
2756	bool IsIncompleteFunction,
2757	bool IsThunk) {
2758
2759	if (llvm::Intrinsic::ID IID = F->getIntrinsicID()) {
2760	// If this is an intrinsic function, set the function's attributes
2761	// to the intrinsic's attributes.
2762	F->setAttributes(llvm::Intrinsic::getAttributes(C&: getLLVMContext(), id: IID));
2763	return;
2764	}
2765
2766	const auto *FD = cast<FunctionDecl>(Val: GD.getDecl());
2767
2768	if (!IsIncompleteFunction)
2769	SetLLVMFunctionAttributes(GD, Info: getTypes().arrangeGlobalDeclaration(GD), F,
2770	IsThunk);
2771
2772	// Add the Returned attribute for "this", except for iOS 5 and earlier
2773	// where substantial code, including the libstdc++ dylib, was compiled with
2774	// GCC and does not actually return "this".
2775	if (!IsThunk && getCXXABI().HasThisReturn(GD) &&
2776	!(getTriple().isiOS() && getTriple().isOSVersionLT(Major: 6))) {
2777	assert(!F->arg_empty() &&
2778	F->arg_begin()->getType()
2779	->canLosslesslyBitCastTo(F->getReturnType()) &&
2780	"unexpected this return");
2781	F->addParamAttr(0, llvm::Attribute::Returned);
2782	}
2783
2784	// Only a few attributes are set on declarations; these may later be
2785	// overridden by a definition.
2786
2787	setLinkageForGV(F, FD);
2788	setGVProperties(GV: F, GD: FD);
2789
2790	// Setup target-specific attributes.
2791	if (!IsIncompleteFunction && F->isDeclaration())
2792	getTargetCodeGenInfo().setTargetAttributes(FD, F, *this);
2793
2794	if (const auto *CSA = FD->getAttr<CodeSegAttr>())
2795	F->setSection(CSA->getName());
2796	else if (const auto *SA = FD->getAttr<SectionAttr>())
2797	F->setSection(SA->getName());
2798
2799	if (const auto *EA = FD->getAttr<ErrorAttr>()) {
2800	if (EA->isError())
2801	F->addFnAttr("dontcall-error", EA->getUserDiagnostic());
2802	else if (EA->isWarning())
2803	F->addFnAttr("dontcall-warn", EA->getUserDiagnostic());
2804	}
2805
2806	// If we plan on emitting this inline builtin, we can't treat it as a builtin.
2807	if (FD->isInlineBuiltinDeclaration()) {
2808	const FunctionDecl *FDBody;
2809	bool HasBody = FD->hasBody(Definition&: FDBody);
2810	(void)HasBody;
2811	assert(HasBody && "Inline builtin declarations should always have an "
2812	"available body!");
2813	if (shouldEmitFunction(FDBody))
2814	F->addFnAttr(llvm::Attribute::NoBuiltin);
2815	}
2816
2817	if (FD->isReplaceableGlobalAllocationFunction()) {
2818	// A replaceable global allocation function does not act like a builtin by
2819	// default, only if it is invoked by a new-expression or delete-expression.
2820	F->addFnAttr(llvm::Attribute::NoBuiltin);
2821	}
2822
2823	if (isa<CXXConstructorDecl>(Val: FD) || isa<CXXDestructorDecl>(Val: FD))
2824	F->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
2825	else if (const auto *MD = dyn_cast<CXXMethodDecl>(Val: FD))
2826	if (MD->isVirtual())
2827	F->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
2828
2829	// Don't emit entries for function declarations in the cross-DSO mode. This
2830	// is handled with better precision by the receiving DSO. But if jump tables
2831	// are non-canonical then we need type metadata in order to produce the local
2832	// jump table.
2833	if (!CodeGenOpts.SanitizeCfiCrossDso ||
2834	!CodeGenOpts.SanitizeCfiCanonicalJumpTables)
2835	CreateFunctionTypeMetadataForIcall(FD, F);
2836
2837	if (LangOpts.Sanitize.has(K: SanitizerKind::KCFI))
2838	setKCFIType(FD, F);
2839
2840	if (getLangOpts().OpenMP && FD->hasAttr<OMPDeclareSimdDeclAttr>())
2841	getOpenMPRuntime().emitDeclareSimdFunction(FD, Fn: F);
2842
2843	if (CodeGenOpts.InlineMaxStackSize != UINT_MAX)
2844	F->addFnAttr(Kind: "inline-max-stacksize", Val: llvm::utostr(X: CodeGenOpts.InlineMaxStackSize));
2845
2846	if (const auto *CB = FD->getAttr<CallbackAttr>()) {
2847	// Annotate the callback behavior as metadata:
2848	// - The callback callee (as argument number).
2849	// - The callback payloads (as argument numbers).
2850	llvm::LLVMContext &Ctx = F->getContext();
2851	llvm::MDBuilder MDB(Ctx);
2852
2853	// The payload indices are all but the first one in the encoding. The first
2854	// identifies the callback callee.
2855	int CalleeIdx = *CB->encoding_begin();
2856	ArrayRef<int> PayloadIndices(CB->encoding_begin() + 1, CB->encoding_end());
2857	F->addMetadata(KindID: llvm::LLVMContext::MD_callback,
2858	MD&: *llvm::MDNode::get(Context&: Ctx, MDs: {MDB.createCallbackEncoding(
2859	CalleeArgNo: CalleeIdx, Arguments: PayloadIndices,
2860	/* VarArgsArePassed */ false)}));
2861	}
2862	}
2863
2864	void CodeGenModule::addUsedGlobal(llvm::GlobalValue *GV) {
2865	assert((isa<llvm::Function>(GV) || !GV->isDeclaration()) &&
2866	"Only globals with definition can force usage.");
2867	LLVMUsed.emplace_back(args&: GV);
2868	}
2869
2870	void CodeGenModule::addCompilerUsedGlobal(llvm::GlobalValue *GV) {
2871	assert(!GV->isDeclaration() &&
2872	"Only globals with definition can force usage.");
2873	LLVMCompilerUsed.emplace_back(args&: GV);
2874	}
2875
2876	void CodeGenModule::addUsedOrCompilerUsedGlobal(llvm::GlobalValue *GV) {
2877	assert((isa<llvm::Function>(GV) || !GV->isDeclaration()) &&
2878	"Only globals with definition can force usage.");
2879	if (getTriple().isOSBinFormatELF())
2880	LLVMCompilerUsed.emplace_back(args&: GV);
2881	else
2882	LLVMUsed.emplace_back(args&: GV);
2883	}
2884
2885	static void emitUsed(CodeGenModule &CGM, StringRef Name,
2886	std::vector<llvm::WeakTrackingVH> &List) {
2887	// Don't create llvm.used if there is no need.
2888	if (List.empty())
2889	return;
2890
2891	// Convert List to what ConstantArray needs.
2892	SmallVector<llvm::Constant*, 8> UsedArray;
2893	UsedArray.resize(N: List.size());
2894	for (unsigned i = 0, e = List.size(); i != e; ++i) {
2895	UsedArray[i] =
2896	llvm::ConstantExpr::getPointerBitCastOrAddrSpaceCast(
2897	C: cast<llvm::Constant>(Val: &*List[i]), Ty: CGM.Int8PtrTy);
2898	}
2899
2900	if (UsedArray.empty())
2901	return;
2902	llvm::ArrayType *ATy = llvm::ArrayType::get(ElementType: CGM.Int8PtrTy, NumElements: UsedArray.size());
2903
2904	auto *GV = new llvm::GlobalVariable(
2905	CGM.getModule(), ATy, false, llvm::GlobalValue::AppendingLinkage,
2906	llvm::ConstantArray::get(T: ATy, V: UsedArray), Name);
2907
2908	GV->setSection("llvm.metadata");
2909	}
2910
2911	void CodeGenModule::emitLLVMUsed() {
2912	emitUsed(CGM&: *this, Name: "llvm.used", List&: LLVMUsed);
2913	emitUsed(CGM&: *this, Name: "llvm.compiler.used", List&: LLVMCompilerUsed);
2914	}
2915
2916	void CodeGenModule::AppendLinkerOptions(StringRef Opts) {
2917	auto *MDOpts = llvm::MDString::get(Context&: getLLVMContext(), Str: Opts);
2918	LinkerOptionsMetadata.push_back(Elt: llvm::MDNode::get(Context&: getLLVMContext(), MDs: MDOpts));
2919	}
2920
2921	void CodeGenModule::AddDetectMismatch(StringRef Name, StringRef Value) {
2922	llvm::SmallString<32> Opt;
2923	getTargetCodeGenInfo().getDetectMismatchOption(Name, Value, Opt);
2924	if (Opt.empty())
2925	return;
2926	auto *MDOpts = llvm::MDString::get(Context&: getLLVMContext(), Str: Opt);
2927	LinkerOptionsMetadata.push_back(Elt: llvm::MDNode::get(Context&: getLLVMContext(), MDs: MDOpts));
2928	}
2929
2930	void CodeGenModule::AddDependentLib(StringRef Lib) {
2931	auto &C = getLLVMContext();
2932	if (getTarget().getTriple().isOSBinFormatELF()) {
2933	ELFDependentLibraries.push_back(
2934	Elt: llvm::MDNode::get(Context&: C, MDs: llvm::MDString::get(Context&: C, Str: Lib)));
2935	return;
2936	}
2937
2938	llvm::SmallString<24> Opt;
2939	getTargetCodeGenInfo().getDependentLibraryOption(Lib, Opt);
2940	auto *MDOpts = llvm::MDString::get(Context&: getLLVMContext(), Str: Opt);
2941	LinkerOptionsMetadata.push_back(Elt: llvm::MDNode::get(Context&: C, MDs: MDOpts));
2942	}
2943
2944	/// Add link options implied by the given module, including modules
2945	/// it depends on, using a postorder walk.
2946	static void addLinkOptionsPostorder(CodeGenModule &CGM, Module *Mod,
2947	SmallVectorImpl<llvm::MDNode *> &Metadata,
2948	llvm::SmallPtrSet<Module *, 16> &Visited) {
2949	// Import this module's parent.
2950	if (Mod->Parent && Visited.insert(Ptr: Mod->Parent).second) {
2951	addLinkOptionsPostorder(CGM, Mod: Mod->Parent, Metadata, Visited);
2952	}
2953
2954	// Import this module's dependencies.
2955	for (Module *Import : llvm::reverse(C&: Mod->Imports)) {
2956	if (Visited.insert(Ptr: Import).second)
2957	addLinkOptionsPostorder(CGM, Mod: Import, Metadata, Visited);
2958	}
2959
2960	// Add linker options to link against the libraries/frameworks
2961	// described by this module.
2962	llvm::LLVMContext &Context = CGM.getLLVMContext();
2963	bool IsELF = CGM.getTarget().getTriple().isOSBinFormatELF();
2964
2965	// For modules that use export_as for linking, use that module
2966	// name instead.
2967	if (Mod->UseExportAsModuleLinkName)
2968	return;
2969
2970	for (const Module::LinkLibrary &LL : llvm::reverse(C&: Mod->LinkLibraries)) {
2971	// Link against a framework. Frameworks are currently Darwin only, so we
2972	// don't to ask TargetCodeGenInfo for the spelling of the linker option.
2973	if (LL.IsFramework) {
2974	llvm::Metadata *Args[2] = {llvm::MDString::get(Context, Str: "-framework"),
2975	llvm::MDString::get(Context, Str: LL.Library)};
2976
2977	Metadata.push_back(Elt: llvm::MDNode::get(Context, MDs: Args));
2978	continue;
2979	}
2980
2981	// Link against a library.
2982	if (IsELF) {
2983	llvm::Metadata *Args[2] = {
2984	llvm::MDString::get(Context, Str: "lib"),
2985	llvm::MDString::get(Context, Str: LL.Library),
2986	};
2987	Metadata.push_back(Elt: llvm::MDNode::get(Context, MDs: Args));
2988	} else {
2989	llvm::SmallString<24> Opt;
2990	CGM.getTargetCodeGenInfo().getDependentLibraryOption(Lib: LL.Library, Opt);
2991	auto *OptString = llvm::MDString::get(Context, Str: Opt);
2992	Metadata.push_back(Elt: llvm::MDNode::get(Context, MDs: OptString));
2993	}
2994	}
2995	}
2996
2997	void CodeGenModule::EmitModuleInitializers(clang::Module *Primary) {
2998	assert(Primary->isNamedModuleUnit() &&
2999	"We should only emit module initializers for named modules.");
3000
3001	// Emit the initializers in the order that sub-modules appear in the
3002	// source, first Global Module Fragments, if present.
3003	if (auto GMF = Primary->getGlobalModuleFragment()) {
3004	for (Decl *D : getContext().getModuleInitializers(M: GMF)) {
3005	if (isa<ImportDecl>(Val: D))
3006	continue;
3007	assert(isa<VarDecl>(D) && "GMF initializer decl is not a var?");
3008	EmitTopLevelDecl(D);
3009	}
3010	}
3011	// Second any associated with the module, itself.
3012	for (Decl *D : getContext().getModuleInitializers(M: Primary)) {
3013	// Skip import decls, the inits for those are called explicitly.
3014	if (isa<ImportDecl>(Val: D))
3015	continue;
3016	EmitTopLevelDecl(D);
3017	}
3018	// Third any associated with the Privat eMOdule Fragment, if present.
3019	if (auto PMF = Primary->getPrivateModuleFragment()) {
3020	for (Decl *D : getContext().getModuleInitializers(M: PMF)) {
3021	// Skip import decls, the inits for those are called explicitly.
3022	if (isa<ImportDecl>(Val: D))
3023	continue;
3024	assert(isa<VarDecl>(D) && "PMF initializer decl is not a var?");
3025	EmitTopLevelDecl(D);
3026	}
3027	}
3028	}
3029
3030	void CodeGenModule::EmitModuleLinkOptions() {
3031	// Collect the set of all of the modules we want to visit to emit link
3032	// options, which is essentially the imported modules and all of their
3033	// non-explicit child modules.
3034	llvm::SetVector<clang::Module *> LinkModules;
3035	llvm::SmallPtrSet<clang::Module *, 16> Visited;
3036	SmallVector<clang::Module *, 16> Stack;
3037
3038	// Seed the stack with imported modules.
3039	for (Module *M : ImportedModules) {
3040	// Do not add any link flags when an implementation TU of a module imports
3041	// a header of that same module.
3042	if (M->getTopLevelModuleName() == getLangOpts().CurrentModule &&
3043	!getLangOpts().isCompilingModule())
3044	continue;
3045	if (Visited.insert(Ptr: M).second)
3046	Stack.push_back(Elt: M);
3047	}
3048
3049	// Find all of the modules to import, making a little effort to prune
3050	// non-leaf modules.
3051	while (!Stack.empty()) {
3052	clang::Module *Mod = Stack.pop_back_val();
3053
3054	bool AnyChildren = false;
3055
3056	// Visit the submodules of this module.
3057	for (const auto &SM : Mod->submodules()) {
3058	// Skip explicit children; they need to be explicitly imported to be
3059	// linked against.
3060	if (SM->IsExplicit)
3061	continue;
3062
3063	if (Visited.insert(Ptr: SM).second) {
3064	Stack.push_back(Elt: SM);
3065	AnyChildren = true;
3066	}
3067	}
3068
3069	// We didn't find any children, so add this module to the list of
3070	// modules to link against.
3071	if (!AnyChildren) {
3072	LinkModules.insert(X: Mod);
3073	}
3074	}
3075
3076	// Add link options for all of the imported modules in reverse topological
3077	// order. We don't do anything to try to order import link flags with respect
3078	// to linker options inserted by things like #pragma comment().
3079	SmallVector<llvm::MDNode *, 16> MetadataArgs;
3080	Visited.clear();
3081	for (Module *M : LinkModules)
3082	if (Visited.insert(Ptr: M).second)
3083	addLinkOptionsPostorder(CGM&: *this, Mod: M, Metadata&: MetadataArgs, Visited);
3084	std::reverse(first: MetadataArgs.begin(), last: MetadataArgs.end());
3085	LinkerOptionsMetadata.append(in_start: MetadataArgs.begin(), in_end: MetadataArgs.end());
3086
3087	// Add the linker options metadata flag.
3088	auto *NMD = getModule().getOrInsertNamedMetadata(Name: "llvm.linker.options");
3089	for (auto *MD : LinkerOptionsMetadata)
3090	NMD->addOperand(M: MD);
3091	}
3092
3093	void CodeGenModule::EmitDeferred() {
3094	// Emit deferred declare target declarations.
3095	if (getLangOpts().OpenMP && !getLangOpts().OpenMPSimd)
3096	getOpenMPRuntime().emitDeferredTargetDecls();
3097
3098	// Emit code for any potentially referenced deferred decls. Since a
3099	// previously unused static decl may become used during the generation of code
3100	// for a static function, iterate until no changes are made.
3101
3102	if (!DeferredVTables.empty()) {
3103	EmitDeferredVTables();
3104
3105	// Emitting a vtable doesn't directly cause more vtables to
3106	// become deferred, although it can cause functions to be
3107	// emitted that then need those vtables.
3108	assert(DeferredVTables.empty());
3109	}
3110
3111	// Emit CUDA/HIP static device variables referenced by host code only.
3112	// Note we should not clear CUDADeviceVarODRUsedByHost since it is still
3113	// needed for further handling.
3114	if (getLangOpts().CUDA && getLangOpts().CUDAIsDevice)
3115	llvm::append_range(C&: DeferredDeclsToEmit,
3116	R&: getContext().CUDADeviceVarODRUsedByHost);
3117
3118	// Stop if we're out of both deferred vtables and deferred declarations.
3119	if (DeferredDeclsToEmit.empty())
3120	return;
3121
3122	// Grab the list of decls to emit. If EmitGlobalDefinition schedules more
3123	// work, it will not interfere with this.
3124	std::vector<GlobalDecl> CurDeclsToEmit;
3125	CurDeclsToEmit.swap(x&: DeferredDeclsToEmit);
3126
3127	for (GlobalDecl &D : CurDeclsToEmit) {
3128	// We should call GetAddrOfGlobal with IsForDefinition set to true in order
3129	// to get GlobalValue with exactly the type we need, not something that
3130	// might had been created for another decl with the same mangled name but
3131	// different type.
3132	llvm::GlobalValue *GV = dyn_cast<llvm::GlobalValue>(
3133	Val: GetAddrOfGlobal(GD: D, IsForDefinition: ForDefinition));
3134
3135	// In case of different address spaces, we may still get a cast, even with
3136	// IsForDefinition equal to true. Query mangled names table to get
3137	// GlobalValue.
3138	if (!GV)
3139	GV = GetGlobalValue(Name: getMangledName(GD: D));
3140
3141	// Make sure GetGlobalValue returned non-null.
3142	assert(GV);
3143
3144	// Check to see if we've already emitted this. This is necessary
3145	// for a couple of reasons: first, decls can end up in the
3146	// deferred-decls queue multiple times, and second, decls can end
3147	// up with definitions in unusual ways (e.g. by an extern inline
3148	// function acquiring a strong function redefinition). Just
3149	// ignore these cases.
3150	if (!GV->isDeclaration())
3151	continue;
3152
3153	// If this is OpenMP, check if it is legal to emit this global normally.
3154	if (LangOpts.OpenMP && OpenMPRuntime && OpenMPRuntime->emitTargetGlobal(GD: D))
3155	continue;
3156
3157	// Otherwise, emit the definition and move on to the next one.
3158	EmitGlobalDefinition(D, GV);
3159
3160	// If we found out that we need to emit more decls, do that recursively.
3161	// This has the advantage that the decls are emitted in a DFS and related
3162	// ones are close together, which is convenient for testing.
3163	if (!DeferredVTables.empty() || !DeferredDeclsToEmit.empty()) {
3164	EmitDeferred();
3165	assert(DeferredVTables.empty() && DeferredDeclsToEmit.empty());
3166	}
3167	}
3168	}
3169
3170	void CodeGenModule::EmitVTablesOpportunistically() {
3171	// Try to emit external vtables as available_externally if they have emitted
3172	// all inlined virtual functions. It runs after EmitDeferred() and therefore
3173	// is not allowed to create new references to things that need to be emitted
3174	// lazily. Note that it also uses fact that we eagerly emitting RTTI.
3175
3176	assert((OpportunisticVTables.empty() || shouldOpportunisticallyEmitVTables())
3177	&& "Only emit opportunistic vtables with optimizations");
3178
3179	for (const CXXRecordDecl *RD : OpportunisticVTables) {
3180	assert(getVTables().isVTableExternal(RD) &&
3181	"This queue should only contain external vtables");
3182	if (getCXXABI().canSpeculativelyEmitVTable(RD))
3183	VTables.GenerateClassData(RD);
3184	}
3185	OpportunisticVTables.clear();
3186	}
3187
3188	void CodeGenModule::EmitGlobalAnnotations() {
3189	for (const auto& [MangledName, VD] : DeferredAnnotations) {
3190	llvm::GlobalValue *GV = GetGlobalValue(Name: MangledName);
3191	if (GV)
3192	AddGlobalAnnotations(D: VD, GV);
3193	}
3194	DeferredAnnotations.clear();
3195
3196	if (Annotations.empty())
3197	return;
3198
3199	// Create a new global variable for the ConstantStruct in the Module.
3200	llvm::Constant *Array = llvm::ConstantArray::get(T: llvm::ArrayType::get(
3201	ElementType: Annotations[0]->getType(), NumElements: Annotations.size()), V: Annotations);
3202	auto *gv = new llvm::GlobalVariable(getModule(), Array->getType(), false,
3203	llvm::GlobalValue::AppendingLinkage,
3204	Array, "llvm.global.annotations");
3205	gv->setSection(AnnotationSection);
3206	}
3207
3208	llvm::Constant *CodeGenModule::EmitAnnotationString(StringRef Str) {
3209	llvm::Constant *&AStr = AnnotationStrings[Str];
3210	if (AStr)
3211	return AStr;
3212
3213	// Not found yet, create a new global.
3214	llvm::Constant *s = llvm::ConstantDataArray::getString(Context&: getLLVMContext(), Initializer: Str);
3215	auto *gv = new llvm::GlobalVariable(
3216	getModule(), s->getType(), true, llvm::GlobalValue::PrivateLinkage, s,
3217	".str", nullptr, llvm::GlobalValue::NotThreadLocal,
3218	ConstGlobalsPtrTy->getAddressSpace());
3219	gv->setSection(AnnotationSection);
3220	gv->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
3221	AStr = gv;
3222	return gv;
3223	}
3224
3225	llvm::Constant *CodeGenModule::EmitAnnotationUnit(SourceLocation Loc) {
3226	SourceManager &SM = getContext().getSourceManager();
3227	PresumedLoc PLoc = SM.getPresumedLoc(Loc);
3228	if (PLoc.isValid())
3229	return EmitAnnotationString(Str: PLoc.getFilename());
3230	return EmitAnnotationString(Str: SM.getBufferName(Loc));
3231	}
3232
3233	llvm::Constant *CodeGenModule::EmitAnnotationLineNo(SourceLocation L) {
3234	SourceManager &SM = getContext().getSourceManager();
3235	PresumedLoc PLoc = SM.getPresumedLoc(Loc: L);
3236	unsigned LineNo = PLoc.isValid() ? PLoc.getLine() :
3237	SM.getExpansionLineNumber(Loc: L);
3238	return llvm::ConstantInt::get(Ty: Int32Ty, V: LineNo);
3239	}
3240
3241	llvm::Constant *CodeGenModule::EmitAnnotationArgs(const AnnotateAttr *Attr) {
3242	ArrayRef<Expr *> Exprs = {Attr->args_begin(), Attr->args_size()};
3243	if (Exprs.empty())
3244	return llvm::ConstantPointerNull::get(T: ConstGlobalsPtrTy);
3245
3246	llvm::FoldingSetNodeID ID;
3247	for (Expr *E : Exprs) {
3248	ID.Add(cast<clang::ConstantExpr>(E)->getAPValueResult());
3249	}
3250	llvm::Constant *&Lookup = AnnotationArgs[ID.ComputeHash()];
3251	if (Lookup)
3252	return Lookup;
3253
3254	llvm::SmallVector<llvm::Constant *, 4> LLVMArgs;
3255	LLVMArgs.reserve(N: Exprs.size());
3256	ConstantEmitter ConstEmiter(*this);
3257	llvm::transform(Range&: Exprs, d_first: std::back_inserter(x&: LLVMArgs), F: [&](const Expr *E) {
3258	const auto *CE = cast<clang::ConstantExpr>(Val: E);
3259	return ConstEmiter.emitAbstract(CE->getBeginLoc(), CE->getAPValueResult(),
3260	CE->getType());
3261	});
3262	auto *Struct = llvm::ConstantStruct::getAnon(V: LLVMArgs);
3263	auto *GV = new llvm::GlobalVariable(getModule(), Struct->getType(), true,
3264	llvm::GlobalValue::PrivateLinkage, Struct,
3265	".args");
3266	GV->setSection(AnnotationSection);
3267	GV->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
3268
3269	Lookup = GV;
3270	return GV;
3271	}
3272
3273	llvm::Constant *CodeGenModule::EmitAnnotateAttr(llvm::GlobalValue *GV,
3274	const AnnotateAttr *AA,
3275	SourceLocation L) {
3276	// Get the globals for file name, annotation, and the line number.
3277	llvm::Constant *AnnoGV = EmitAnnotationString(Str: AA->getAnnotation()),
3278	*UnitGV = EmitAnnotationUnit(Loc: L),
3279	*LineNoCst = EmitAnnotationLineNo(L),
3280	*Args = EmitAnnotationArgs(Attr: AA);
3281
3282	llvm::Constant *GVInGlobalsAS = GV;
3283	if (GV->getAddressSpace() !=
3284	getDataLayout().getDefaultGlobalsAddressSpace()) {
3285	GVInGlobalsAS = llvm::ConstantExpr::getAddrSpaceCast(
3286	C: GV,
3287	Ty: llvm::PointerType::get(
3288	C&: GV->getContext(), AddressSpace: getDataLayout().getDefaultGlobalsAddressSpace()));
3289	}
3290
3291	// Create the ConstantStruct for the global annotation.
3292	llvm::Constant *Fields[] = {
3293	GVInGlobalsAS, AnnoGV, UnitGV, LineNoCst, Args,
3294	};
3295	return llvm::ConstantStruct::getAnon(V: Fields);
3296	}
3297
3298	void CodeGenModule::AddGlobalAnnotations(const ValueDecl *D,
3299	llvm::GlobalValue *GV) {
3300	assert(D->hasAttr<AnnotateAttr>() && "no annotate attribute");
3301	// Get the struct elements for these annotations.
3302	for (const auto *I : D->specific_attrs<AnnotateAttr>())
3303	Annotations.push_back(EmitAnnotateAttr(GV, I, D->getLocation()));
3304	}
3305
3306	bool CodeGenModule::isInNoSanitizeList(SanitizerMask Kind, llvm::Function *Fn,
3307	SourceLocation Loc) const {
3308	const auto &NoSanitizeL = getContext().getNoSanitizeList();
3309	// NoSanitize by function name.
3310	if (NoSanitizeL.containsFunction(Mask: Kind, FunctionName: Fn->getName()))
3311	return true;
3312	// NoSanitize by location. Check "mainfile" prefix.
3313	auto &SM = Context.getSourceManager();
3314	FileEntryRef MainFile = *SM.getFileEntryRefForID(FID: SM.getMainFileID());
3315	if (NoSanitizeL.containsMainFile(Mask: Kind, FileName: MainFile.getName()))
3316	return true;
3317
3318	// Check "src" prefix.
3319	if (Loc.isValid())
3320	return NoSanitizeL.containsLocation(Mask: Kind, Loc);
3321	// If location is unknown, this may be a compiler-generated function. Assume
3322	// it's located in the main file.
3323	return NoSanitizeL.containsFile(Mask: Kind, FileName: MainFile.getName());
3324	}
3325
3326	bool CodeGenModule::isInNoSanitizeList(SanitizerMask Kind,
3327	llvm::GlobalVariable *GV,
3328	SourceLocation Loc, QualType Ty,
3329	StringRef Category) const {
3330	const auto &NoSanitizeL = getContext().getNoSanitizeList();
3331	if (NoSanitizeL.containsGlobal(Mask: Kind, GlobalName: GV->getName(), Category))
3332	return true;
3333	auto &SM = Context.getSourceManager();
3334	if (NoSanitizeL.containsMainFile(
3335	Mask: Kind, FileName: SM.getFileEntryRefForID(FID: SM.getMainFileID())->getName(),
3336	Category))
3337	return true;
3338	if (NoSanitizeL.containsLocation(Mask: Kind, Loc, Category))
3339	return true;
3340
3341	// Check global type.
3342	if (!Ty.isNull()) {
3343	// Drill down the array types: if global variable of a fixed type is
3344	// not sanitized, we also don't instrument arrays of them.
3345	while (auto AT = dyn_cast<ArrayType>(Val: Ty.getTypePtr()))
3346	Ty = AT->getElementType();
3347	Ty = Ty.getCanonicalType().getUnqualifiedType();
3348	// Only record types (classes, structs etc.) are ignored.
3349	if (Ty->isRecordType()) {
3350	std::string TypeStr = Ty.getAsString(Policy: getContext().getPrintingPolicy());
3351	if (NoSanitizeL.containsType(Mask: Kind, MangledTypeName: TypeStr, Category))
3352	return true;
3353	}
3354	}
3355	return false;
3356	}
3357
3358	bool CodeGenModule::imbueXRayAttrs(llvm::Function *Fn, SourceLocation Loc,
3359	StringRef Category) const {
3360	const auto &XRayFilter = getContext().getXRayFilter();
3361	using ImbueAttr = XRayFunctionFilter::ImbueAttribute;
3362	auto Attr = ImbueAttr::NONE;
3363	if (Loc.isValid())
3364	Attr = XRayFilter.shouldImbueLocation(Loc, Category);
3365	if (Attr == ImbueAttr::NONE)
3366	Attr = XRayFilter.shouldImbueFunction(FunctionName: Fn->getName());
3367	switch (Attr) {
3368	case ImbueAttr::NONE:
3369	return false;
3370	case ImbueAttr::ALWAYS:
3371	Fn->addFnAttr(Kind: "function-instrument", Val: "xray-always");
3372	break;
3373	case ImbueAttr::ALWAYS_ARG1:
3374	Fn->addFnAttr(Kind: "function-instrument", Val: "xray-always");
3375	Fn->addFnAttr(Kind: "xray-log-args", Val: "1");
3376	break;
3377	case ImbueAttr::NEVER:
3378	Fn->addFnAttr(Kind: "function-instrument", Val: "xray-never");
3379	break;
3380	}
3381	return true;
3382	}
3383
3384	ProfileList::ExclusionType
3385	CodeGenModule::isFunctionBlockedByProfileList(llvm::Function *Fn,
3386	SourceLocation Loc) const {
3387	const auto &ProfileList = getContext().getProfileList();
3388	// If the profile list is empty, then instrument everything.
3389	if (ProfileList.isEmpty())
3390	return ProfileList::Allow;
3391	CodeGenOptions::ProfileInstrKind Kind = getCodeGenOpts().getProfileInstr();
3392	// First, check the function name.
3393	if (auto V = ProfileList.isFunctionExcluded(FunctionName: Fn->getName(), Kind))
3394	return *V;
3395	// Next, check the source location.
3396	if (Loc.isValid())
3397	if (auto V = ProfileList.isLocationExcluded(Loc, Kind))
3398	return *V;
3399	// If location is unknown, this may be a compiler-generated function. Assume
3400	// it's located in the main file.
3401	auto &SM = Context.getSourceManager();
3402	if (auto MainFile = SM.getFileEntryRefForID(FID: SM.getMainFileID()))
3403	if (auto V = ProfileList.isFileExcluded(FileName: MainFile->getName(), Kind))
3404	return *V;
3405	return ProfileList.getDefault(Kind);
3406	}
3407
3408	ProfileList::ExclusionType
3409	CodeGenModule::isFunctionBlockedFromProfileInstr(llvm::Function *Fn,
3410	SourceLocation Loc) const {
3411	auto V = isFunctionBlockedByProfileList(Fn, Loc);
3412	if (V != ProfileList::Allow)
3413	return V;
3414
3415	auto NumGroups = getCodeGenOpts().ProfileTotalFunctionGroups;
3416	if (NumGroups > 1) {
3417	auto Group = llvm::crc32(Data: arrayRefFromStringRef(Input: Fn->getName())) % NumGroups;
3418	if (Group != getCodeGenOpts().ProfileSelectedFunctionGroup)
3419	return ProfileList::Skip;
3420	}
3421	return ProfileList::Allow;
3422	}
3423
3424	bool CodeGenModule::MustBeEmitted(const ValueDecl *Global) {
3425	// Never defer when EmitAllDecls is specified.
3426	if (LangOpts.EmitAllDecls)
3427	return true;
3428
3429	const auto *VD = dyn_cast<VarDecl>(Val: Global);
3430	if (VD &&
3431	((CodeGenOpts.KeepPersistentStorageVariables &&
3432	(VD->getStorageDuration() == SD_Static ||
3433	VD->getStorageDuration() == SD_Thread)) ||
3434	(CodeGenOpts.KeepStaticConsts && VD->getStorageDuration() == SD_Static &&
3435	VD->getType().isConstQualified())))
3436	return true;
3437
3438	return getContext().DeclMustBeEmitted(Global);
3439	}
3440
3441	bool CodeGenModule::MayBeEmittedEagerly(const ValueDecl *Global) {
3442	// In OpenMP 5.0 variables and function may be marked as
3443	// device_type(host/nohost) and we should not emit them eagerly unless we sure
3444	// that they must be emitted on the host/device. To be sure we need to have
3445	// seen a declare target with an explicit mentioning of the function, we know
3446	// we have if the level of the declare target attribute is -1. Note that we
3447	// check somewhere else if we should emit this at all.
3448	if (LangOpts.OpenMP >= 50 && !LangOpts.OpenMPSimd) {
3449	std::optional<OMPDeclareTargetDeclAttr *> ActiveAttr =
3450	OMPDeclareTargetDeclAttr::getActiveAttr(Global);
3451	if (!ActiveAttr || (*ActiveAttr)->getLevel() != (unsigned)-1)
3452	return false;
3453	}
3454
3455	if (const auto *FD = dyn_cast<FunctionDecl>(Val: Global)) {
3456	if (FD->getTemplateSpecializationKind() == TSK_ImplicitInstantiation)
3457	// Implicit template instantiations may change linkage if they are later
3458	// explicitly instantiated, so they should not be emitted eagerly.
3459	return false;
3460	// Defer until all versions have been semantically checked.
3461	if (FD->hasAttr<TargetVersionAttr>() && !FD->isMultiVersion())
3462	return false;
3463	}
3464	if (const auto *VD = dyn_cast<VarDecl>(Val: Global)) {
3465	if (Context.getInlineVariableDefinitionKind(VD) ==
3466	ASTContext::InlineVariableDefinitionKind::WeakUnknown)
3467	// A definition of an inline constexpr static data member may change
3468	// linkage later if it's redeclared outside the class.
3469	return false;
3470	if (CXX20ModuleInits && VD->getOwningModule() &&
3471	!VD->getOwningModule()->isModuleMapModule()) {
3472	// For CXX20, module-owned initializers need to be deferred, since it is
3473	// not known at this point if they will be run for the current module or
3474	// as part of the initializer for an imported one.
3475	return false;
3476	}
3477	}
3478	// If OpenMP is enabled and threadprivates must be generated like TLS, delay
3479	// codegen for global variables, because they may be marked as threadprivate.
3480	if (LangOpts.OpenMP && LangOpts.OpenMPUseTLS &&
3481	getContext().getTargetInfo().isTLSSupported() && isa<VarDecl>(Global) &&
3482	!Global->getType().isConstantStorage(getContext(), false, false) &&
3483	!OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(Global))
3484	return false;
3485
3486	return true;
3487	}
3488
3489	ConstantAddress CodeGenModule::GetAddrOfMSGuidDecl(const MSGuidDecl *GD) {
3490	StringRef Name = getMangledName(GD);
3491
3492	// The UUID descriptor should be pointer aligned.
3493	CharUnits Alignment = CharUnits::fromQuantity(Quantity: PointerAlignInBytes);
3494
3495	// Look for an existing global.
3496	if (llvm::GlobalVariable *GV = getModule().getNamedGlobal(Name))
3497	return ConstantAddress(GV, GV->getValueType(), Alignment);
3498
3499	ConstantEmitter Emitter(*this);
3500	llvm::Constant *Init;
3501
3502	APValue &V = GD->getAsAPValue();
3503	if (!V.isAbsent()) {
3504	// If possible, emit the APValue version of the initializer. In particular,
3505	// this gets the type of the constant right.
3506	Init = Emitter.emitForInitializer(
3507	value: GD->getAsAPValue(), destAddrSpace: GD->getType().getAddressSpace(), destType: GD->getType());
3508	} else {
3509	// As a fallback, directly construct the constant.
3510	// FIXME: This may get padding wrong under esoteric struct layout rules.
3511	// MSVC appears to create a complete type 'struct __s_GUID' that it
3512	// presumably uses to represent these constants.
3513	MSGuidDecl::Parts Parts = GD->getParts();
3514	llvm::Constant *Fields[4] = {
3515	llvm::ConstantInt::get(Ty: Int32Ty, V: Parts.Part1),
3516	llvm::ConstantInt::get(Ty: Int16Ty, V: Parts.Part2),
3517	llvm::ConstantInt::get(Ty: Int16Ty, V: Parts.Part3),
3518	llvm::ConstantDataArray::getRaw(
3519	Data: StringRef(reinterpret_cast<char *>(Parts.Part4And5), 8), NumElements: 8,
3520	ElementTy: Int8Ty)};
3521	Init = llvm::ConstantStruct::getAnon(V: Fields);
3522	}
3523
3524	auto *GV = new llvm::GlobalVariable(
3525	getModule(), Init->getType(),
3526	/*isConstant=*/true, llvm::GlobalValue::LinkOnceODRLinkage, Init, Name);
3527	if (supportsCOMDAT())
3528	GV->setComdat(TheModule.getOrInsertComdat(Name: GV->getName()));
3529	setDSOLocal(GV);
3530
3531	if (!V.isAbsent()) {
3532	Emitter.finalize(global: GV);
3533	return ConstantAddress(GV, GV->getValueType(), Alignment);
3534	}
3535
3536	llvm::Type *Ty = getTypes().ConvertTypeForMem(T: GD->getType());
3537	return ConstantAddress(GV, Ty, Alignment);
3538	}
3539
3540	ConstantAddress CodeGenModule::GetAddrOfUnnamedGlobalConstantDecl(
3541	const UnnamedGlobalConstantDecl *GCD) {
3542	CharUnits Alignment = getContext().getTypeAlignInChars(GCD->getType());
3543
3544	llvm::GlobalVariable **Entry = nullptr;
3545	Entry = &UnnamedGlobalConstantDeclMap[GCD];
3546	if (*Entry)
3547	return ConstantAddress(*Entry, (*Entry)->getValueType(), Alignment);
3548
3549	ConstantEmitter Emitter(*this);
3550	llvm::Constant *Init;
3551
3552	const APValue &V = GCD->getValue();
3553
3554	assert(!V.isAbsent());
3555	Init = Emitter.emitForInitializer(value: V, destAddrSpace: GCD->getType().getAddressSpace(),
3556	destType: GCD->getType());
3557
3558	auto *GV = new llvm::GlobalVariable(getModule(), Init->getType(),
3559	/*isConstant=*/true,
3560	llvm::GlobalValue::PrivateLinkage, Init,
3561	".constant");
3562	GV->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
3563	GV->setAlignment(Alignment.getAsAlign());
3564
3565	Emitter.finalize(global: GV);
3566
3567	*Entry = GV;
3568	return ConstantAddress(GV, GV->getValueType(), Alignment);
3569	}
3570
3571	ConstantAddress CodeGenModule::GetAddrOfTemplateParamObject(
3572	const TemplateParamObjectDecl *TPO) {
3573	StringRef Name = getMangledName(TPO);
3574	CharUnits Alignment = getNaturalTypeAlignment(T: TPO->getType());
3575
3576	if (llvm::GlobalVariable *GV = getModule().getNamedGlobal(Name))
3577	return ConstantAddress(GV, GV->getValueType(), Alignment);
3578
3579	ConstantEmitter Emitter(*this);
3580	llvm::Constant *Init = Emitter.emitForInitializer(
3581	value: TPO->getValue(), destAddrSpace: TPO->getType().getAddressSpace(), destType: TPO->getType());
3582
3583	if (!Init) {
3584	ErrorUnsupported(TPO, "template parameter object");
3585	return ConstantAddress::invalid();
3586	}
3587
3588	llvm::GlobalValue::LinkageTypes Linkage =
3589	isExternallyVisible(TPO->getLinkageAndVisibility().getLinkage())
3590	? llvm::GlobalValue::LinkOnceODRLinkage
3591	: llvm::GlobalValue::InternalLinkage;
3592	auto *GV = new llvm::GlobalVariable(getModule(), Init->getType(),
3593	/*isConstant=*/true, Linkage, Init, Name);
3594	setGVProperties(GV, TPO);
3595	if (supportsCOMDAT())
3596	GV->setComdat(TheModule.getOrInsertComdat(Name: GV->getName()));
3597	Emitter.finalize(global: GV);
3598
3599	return ConstantAddress(GV, GV->getValueType(), Alignment);
3600	}
3601
3602	ConstantAddress CodeGenModule::GetWeakRefReference(const ValueDecl *VD) {
3603	const AliasAttr *AA = VD->getAttr<AliasAttr>();
3604	assert(AA && "No alias?");
3605
3606	CharUnits Alignment = getContext().getDeclAlign(VD);
3607	llvm::Type *DeclTy = getTypes().ConvertTypeForMem(T: VD->getType());
3608
3609	// See if there is already something with the target's name in the module.
3610	llvm::GlobalValue *Entry = GetGlobalValue(Name: AA->getAliasee());
3611	if (Entry)
3612	return ConstantAddress(Entry, DeclTy, Alignment);
3613
3614	llvm::Constant *Aliasee;
3615	if (isa<llvm::FunctionType>(Val: DeclTy))
3616	Aliasee = GetOrCreateLLVMFunction(MangledName: AA->getAliasee(), Ty: DeclTy,
3617	D: GlobalDecl(cast<FunctionDecl>(Val: VD)),
3618	/*ForVTable=*/false);
3619	else
3620	Aliasee = GetOrCreateLLVMGlobal(MangledName: AA->getAliasee(), Ty: DeclTy, AddrSpace: LangAS::Default,
3621	D: nullptr);
3622
3623	auto *F = cast<llvm::GlobalValue>(Val: Aliasee);
3624	F->setLinkage(llvm::Function::ExternalWeakLinkage);
3625	WeakRefReferences.insert(Ptr: F);
3626
3627	return ConstantAddress(Aliasee, DeclTy, Alignment);
3628	}
3629
3630	template <typename AttrT> static bool hasImplicitAttr(const ValueDecl *D) {
3631	if (!D)
3632	return false;
3633	if (auto *A = D->getAttr<AttrT>())
3634	return A->isImplicit();
3635	return D->isImplicit();
3636	}
3637
3638	void CodeGenModule::EmitGlobal(GlobalDecl GD) {
3639	const auto *Global = cast<ValueDecl>(Val: GD.getDecl());
3640
3641	// Weak references don't produce any output by themselves.
3642	if (Global->hasAttr<WeakRefAttr>())
3643	return;
3644
3645	// If this is an alias definition (which otherwise looks like a declaration)
3646	// emit it now.
3647	if (Global->hasAttr<AliasAttr>())
3648	return EmitAliasDefinition(GD);
3649
3650	// IFunc like an alias whose value is resolved at runtime by calling resolver.
3651	if (Global->hasAttr<IFuncAttr>())
3652	return emitIFuncDefinition(GD);
3653
3654	// If this is a cpu_dispatch multiversion function, emit the resolver.
3655	if (Global->hasAttr<CPUDispatchAttr>())
3656	return emitCPUDispatchDefinition(GD);
3657
3658	// If this is CUDA, be selective about which declarations we emit.
3659	// Non-constexpr non-lambda implicit host device functions are not emitted
3660	// unless they are used on device side.
3661	if (LangOpts.CUDA) {
3662	if (LangOpts.CUDAIsDevice) {
3663	const auto *FD = dyn_cast<FunctionDecl>(Val: Global);
3664	if ((!Global->hasAttr<CUDADeviceAttr>() ||
3665	(LangOpts.OffloadImplicitHostDeviceTemplates && FD &&
3666	hasImplicitAttr<CUDAHostAttr>(FD) &&
3667	hasImplicitAttr<CUDADeviceAttr>(FD) && !FD->isConstexpr() &&
3668	!isLambdaCallOperator(FD) &&
3669	!getContext().CUDAImplicitHostDeviceFunUsedByDevice.count(FD))) &&
3670	!Global->hasAttr<CUDAGlobalAttr>() &&
3671	!Global->hasAttr<CUDAConstantAttr>() &&
3672	!Global->hasAttr<CUDASharedAttr>() &&
3673	!Global->getType()->isCUDADeviceBuiltinSurfaceType() &&
3674	!Global->getType()->isCUDADeviceBuiltinTextureType() &&
3675	!(LangOpts.HIPStdPar && isa<FunctionDecl>(Global) &&
3676	!Global->hasAttr<CUDAHostAttr>()))
3677	return;
3678	} else {
3679	// We need to emit host-side 'shadows' for all global
3680	// device-side variables because the CUDA runtime needs their
3681	// size and host-side address in order to provide access to
3682	// their device-side incarnations.
3683
3684	// So device-only functions are the only things we skip.
3685	if (isa<FunctionDecl>(Global) && !Global->hasAttr<CUDAHostAttr>() &&
3686	Global->hasAttr<CUDADeviceAttr>())
3687	return;
3688
3689	assert((isa<FunctionDecl>(Global) || isa<VarDecl>(Global)) &&
3690	"Expected Variable or Function");
3691	}
3692	}
3693
3694	if (LangOpts.OpenMP) {
3695	// If this is OpenMP, check if it is legal to emit this global normally.
3696	if (OpenMPRuntime && OpenMPRuntime->emitTargetGlobal(GD))
3697	return;
3698	if (auto *DRD = dyn_cast<OMPDeclareReductionDecl>(Val: Global)) {
3699	if (MustBeEmitted(Global))
3700	EmitOMPDeclareReduction(D: DRD);
3701	return;
3702	}
3703	if (auto *DMD = dyn_cast<OMPDeclareMapperDecl>(Val: Global)) {
3704	if (MustBeEmitted(Global))
3705	EmitOMPDeclareMapper(D: DMD);
3706	return;
3707	}
3708	}
3709
3710	// Ignore declarations, they will be emitted on their first use.
3711	if (const auto *FD = dyn_cast<FunctionDecl>(Val: Global)) {
3712	// Update deferred annotations with the latest declaration if the function
3713	// function was already used or defined.
3714	if (FD->hasAttr<AnnotateAttr>()) {
3715	StringRef MangledName = getMangledName(GD);
3716	if (GetGlobalValue(Name: MangledName))
3717	DeferredAnnotations[MangledName] = FD;
3718	}
3719
3720	// Forward declarations are emitted lazily on first use.
3721	if (!FD->doesThisDeclarationHaveABody()) {
3722	if (!FD->doesDeclarationForceExternallyVisibleDefinition() &&
3723	(!FD->isMultiVersion() ||
3724	!FD->getASTContext().getTargetInfo().getTriple().isAArch64()))
3725	return;
3726
3727	StringRef MangledName = getMangledName(GD);
3728
3729	// Compute the function info and LLVM type.
3730	const CGFunctionInfo &FI = getTypes().arrangeGlobalDeclaration(GD);
3731	llvm::Type *Ty = getTypes().GetFunctionType(Info: FI);
3732
3733	GetOrCreateLLVMFunction(MangledName, Ty, D: GD, /*ForVTable=*/false,
3734	/*DontDefer=*/false);
3735	return;
3736	}
3737	} else {
3738	const auto *VD = cast<VarDecl>(Val: Global);
3739	assert(VD->isFileVarDecl() && "Cannot emit local var decl as global.");
3740	if (VD->isThisDeclarationADefinition() != VarDecl::Definition &&
3741	!Context.isMSStaticDataMemberInlineDefinition(VD)) {
3742	if (LangOpts.OpenMP) {
3743	// Emit declaration of the must-be-emitted declare target variable.
3744	if (std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
3745	OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD)) {
3746
3747	// If this variable has external storage and doesn't require special
3748	// link handling we defer to its canonical definition.
3749	if (VD->hasExternalStorage() &&
3750	Res != OMPDeclareTargetDeclAttr::MT_Link)
3751	return;
3752
3753	bool UnifiedMemoryEnabled =
3754	getOpenMPRuntime().hasRequiresUnifiedSharedMemory();
3755	if ((*Res == OMPDeclareTargetDeclAttr::MT_To ||
3756	*Res == OMPDeclareTargetDeclAttr::MT_Enter) &&
3757	!UnifiedMemoryEnabled) {
3758	(void)GetAddrOfGlobalVar(D: VD);
3759	} else {
3760	assert(((*Res == OMPDeclareTargetDeclAttr::MT_Link) ||
3761	((*Res == OMPDeclareTargetDeclAttr::MT_To ||
3762	*Res == OMPDeclareTargetDeclAttr::MT_Enter) &&
3763	UnifiedMemoryEnabled)) &&
3764	"Link clause or to clause with unified memory expected.");
3765	(void)getOpenMPRuntime().getAddrOfDeclareTargetVar(VD);
3766	}
3767
3768	return;
3769	}
3770	}
3771	// If this declaration may have caused an inline variable definition to
3772	// change linkage, make sure that it's emitted.
3773	if (Context.getInlineVariableDefinitionKind(VD) ==
3774	ASTContext::InlineVariableDefinitionKind::Strong)
3775	GetAddrOfGlobalVar(D: VD);
3776	return;
3777	}
3778	}
3779
3780	// Defer code generation to first use when possible, e.g. if this is an inline
3781	// function. If the global must always be emitted, do it eagerly if possible
3782	// to benefit from cache locality.
3783	if (MustBeEmitted(Global) && MayBeEmittedEagerly(Global)) {
3784	// Emit the definition if it can't be deferred.
3785	EmitGlobalDefinition(D: GD);
3786	addEmittedDeferredDecl(GD);
3787	return;
3788	}
3789
3790	// If we're deferring emission of a C++ variable with an
3791	// initializer, remember the order in which it appeared in the file.
3792	if (getLangOpts().CPlusPlus && isa<VarDecl>(Val: Global) &&
3793	cast<VarDecl>(Val: Global)->hasInit()) {
3794	DelayedCXXInitPosition[Global] = CXXGlobalInits.size();
3795	CXXGlobalInits.push_back(x: nullptr);
3796	}
3797
3798	StringRef MangledName = getMangledName(GD);
3799	if (GetGlobalValue(Name: MangledName) != nullptr) {
3800	// The value has already been used and should therefore be emitted.
3801	addDeferredDeclToEmit(GD);
3802	} else if (MustBeEmitted(Global)) {
3803	// The value must be emitted, but cannot be emitted eagerly.
3804	assert(!MayBeEmittedEagerly(Global));
3805	addDeferredDeclToEmit(GD);
3806	} else {
3807	// Otherwise, remember that we saw a deferred decl with this name. The
3808	// first use of the mangled name will cause it to move into
3809	// DeferredDeclsToEmit.
3810	DeferredDecls[MangledName] = GD;
3811	}
3812	}
3813
3814	// Check if T is a class type with a destructor that's not dllimport.
3815	static bool HasNonDllImportDtor(QualType T) {
3816	if (const auto *RT = T->getBaseElementTypeUnsafe()->getAs<RecordType>())
3817	if (CXXRecordDecl *RD = dyn_cast<CXXRecordDecl>(Val: RT->getDecl()))
3818	if (RD->getDestructor() && !RD->getDestructor()->hasAttr<DLLImportAttr>())
3819	return true;
3820
3821	return false;
3822	}
3823
3824	namespace {
3825	struct FunctionIsDirectlyRecursive
3826	: public ConstStmtVisitor<FunctionIsDirectlyRecursive, bool> {
3827	const StringRef Name;
3828	const Builtin::Context &BI;
3829	FunctionIsDirectlyRecursive(StringRef N, const Builtin::Context &C)
3830	: Name(N), BI(C) {}
3831
3832	bool VisitCallExpr(const CallExpr *E) {
3833	const FunctionDecl *FD = E->getDirectCallee();
3834	if (!FD)
3835	return false;
3836	AsmLabelAttr *Attr = FD->getAttr<AsmLabelAttr>();
3837	if (Attr && Name == Attr->getLabel())
3838	return true;
3839	unsigned BuiltinID = FD->getBuiltinID();
3840	if (!BuiltinID || !BI.isLibFunction(ID: BuiltinID))
3841	return false;
3842	StringRef BuiltinName = BI.getName(ID: BuiltinID);
3843	if (BuiltinName.starts_with(Prefix: "__builtin_") &&
3844	Name == BuiltinName.slice(Start: strlen(s: "__builtin_"), End: StringRef::npos)) {
3845	return true;
3846	}
3847	return false;
3848	}
3849
3850	bool VisitStmt(const Stmt *S) {
3851	for (const Stmt *Child : S->children())
3852	if (Child && this->Visit(Child))
3853	return true;
3854	return false;
3855	}
3856	};
3857
3858	// Make sure we're not referencing non-imported vars or functions.
3859	struct DLLImportFunctionVisitor
3860	: public RecursiveASTVisitor<DLLImportFunctionVisitor> {
3861	bool SafeToInline = true;
3862
3863	bool shouldVisitImplicitCode() const { return true; }
3864
3865	bool VisitVarDecl(VarDecl *VD) {
3866	if (VD->getTLSKind()) {
3867	// A thread-local variable cannot be imported.
3868	SafeToInline = false;
3869	return SafeToInline;
3870	}
3871
3872	// A variable definition might imply a destructor call.
3873	if (VD->isThisDeclarationADefinition())
3874	SafeToInline = !HasNonDllImportDtor(VD->getType());
3875
3876	return SafeToInline;
3877	}
3878
3879	bool VisitCXXBindTemporaryExpr(CXXBindTemporaryExpr *E) {
3880	if (const auto *D = E->getTemporary()->getDestructor())
3881	SafeToInline = D->hasAttr<DLLImportAttr>();
3882	return SafeToInline;
3883	}
3884
3885	bool VisitDeclRefExpr(DeclRefExpr *E) {
3886	ValueDecl *VD = E->getDecl();
3887	if (isa<FunctionDecl>(VD))
3888	SafeToInline = VD->hasAttr<DLLImportAttr>();
3889	else if (VarDecl *V = dyn_cast<VarDecl>(VD))
3890	SafeToInline = !V->hasGlobalStorage() || V->hasAttr<DLLImportAttr>();
3891	return SafeToInline;
3892	}
3893
3894	bool VisitCXXConstructExpr(CXXConstructExpr *E) {
3895	SafeToInline = E->getConstructor()->hasAttr<DLLImportAttr>();
3896	return SafeToInline;
3897	}
3898
3899	bool VisitCXXMemberCallExpr(CXXMemberCallExpr *E) {
3900	CXXMethodDecl *M = E->getMethodDecl();
3901	if (!M) {
3902	// Call through a pointer to member function. This is safe to inline.
3903	SafeToInline = true;
3904	} else {
3905	SafeToInline = M->hasAttr<DLLImportAttr>();
3906	}
3907	return SafeToInline;
3908	}
3909
3910	bool VisitCXXDeleteExpr(CXXDeleteExpr *E) {
3911	SafeToInline = E->getOperatorDelete()->hasAttr<DLLImportAttr>();
3912	return SafeToInline;
3913	}
3914
3915	bool VisitCXXNewExpr(CXXNewExpr *E) {
3916	SafeToInline = E->getOperatorNew()->hasAttr<DLLImportAttr>();
3917	return SafeToInline;
3918	}
3919	};
3920	}
3921
3922	// isTriviallyRecursive - Check if this function calls another
3923	// decl that, because of the asm attribute or the other decl being a builtin,
3924	// ends up pointing to itself.
3925	bool
3926	CodeGenModule::isTriviallyRecursive(const FunctionDecl *FD) {
3927	StringRef Name;
3928	if (getCXXABI().getMangleContext().shouldMangleDeclName(FD)) {
3929	// asm labels are a special kind of mangling we have to support.
3930	AsmLabelAttr *Attr = FD->getAttr<AsmLabelAttr>();
3931	if (!Attr)
3932	return false;
3933	Name = Attr->getLabel();
3934	} else {
3935	Name = FD->getName();
3936	}
3937
3938	FunctionIsDirectlyRecursive Walker(Name, Context.BuiltinInfo);
3939	const Stmt *Body = FD->getBody();
3940	return Body ? Walker.Visit(Body) : false;
3941	}
3942
3943	bool CodeGenModule::shouldEmitFunction(GlobalDecl GD) {
3944	if (getFunctionLinkage(GD) != llvm::Function::AvailableExternallyLinkage)
3945	return true;
3946
3947	const auto *F = cast<FunctionDecl>(Val: GD.getDecl());
3948	if (CodeGenOpts.OptimizationLevel == 0 && !F->hasAttr<AlwaysInlineAttr>())
3949	return false;
3950
3951	// We don't import function bodies from other named module units since that
3952	// behavior may break ABI compatibility of the current unit.
3953	if (const Module *M = F->getOwningModule();
3954	M && M->getTopLevelModule()->isNamedModule() &&
3955	getContext().getCurrentNamedModule() != M->getTopLevelModule()) {
3956	// There are practices to mark template member function as always-inline
3957	// and mark the template as extern explicit instantiation but not give
3958	// the definition for member function. So we have to emit the function
3959	// from explicitly instantiation with always-inline.
3960	//
3961	// See https://github.com/llvm/llvm-project/issues/86893 for details.
3962	//
3963	// TODO: Maybe it is better to give it a warning if we call a non-inline
3964	// function from other module units which is marked as always-inline.
3965	if (!F->isTemplateInstantiation() || !F->hasAttr<AlwaysInlineAttr>()) {
3966	return false;
3967	}
3968	}
3969
3970	if (F->hasAttr<NoInlineAttr>())
3971	return false;
3972
3973	if (F->hasAttr<DLLImportAttr>() && !F->hasAttr<AlwaysInlineAttr>()) {
3974	// Check whether it would be safe to inline this dllimport function.
3975	DLLImportFunctionVisitor Visitor;
3976	Visitor.TraverseFunctionDecl(const_cast<FunctionDecl*>(F));
3977	if (!Visitor.SafeToInline)
3978	return false;
3979
3980	if (const CXXDestructorDecl *Dtor = dyn_cast<CXXDestructorDecl>(Val: F)) {
3981	// Implicit destructor invocations aren't captured in the AST, so the
3982	// check above can't see them. Check for them manually here.
3983	for (const Decl *Member : Dtor->getParent()->decls())
3984	if (isa<FieldDecl>(Member))
3985	if (HasNonDllImportDtor(cast<FieldDecl>(Member)->getType()))
3986	return false;
3987	for (const CXXBaseSpecifier &B : Dtor->getParent()->bases())
3988	if (HasNonDllImportDtor(B.getType()))
3989	return false;
3990	}
3991	}
3992
3993	// Inline builtins declaration must be emitted. They often are fortified
3994	// functions.
3995	if (F->isInlineBuiltinDeclaration())
3996	return true;
3997
3998	// PR9614. Avoid cases where the source code is lying to us. An available
3999	// externally function should have an equivalent function somewhere else,
4000	// but a function that calls itself through asm label/`__builtin_` trickery is
4001	// clearly not equivalent to the real implementation.
4002	// This happens in glibc's btowc and in some configure checks.
4003	return !isTriviallyRecursive(FD: F);
4004	}
4005
4006	bool CodeGenModule::shouldOpportunisticallyEmitVTables() {
4007	return CodeGenOpts.OptimizationLevel > 0;
4008	}
4009
4010	void CodeGenModule::EmitMultiVersionFunctionDefinition(GlobalDecl GD,
4011	llvm::GlobalValue *GV) {
4012	const auto *FD = cast<FunctionDecl>(Val: GD.getDecl());
4013
4014	if (FD->isCPUSpecificMultiVersion()) {
4015	auto *Spec = FD->getAttr<CPUSpecificAttr>();
4016	for (unsigned I = 0; I < Spec->cpus_size(); ++I)
4017	EmitGlobalFunctionDefinition(GD: GD.getWithMultiVersionIndex(Index: I), GV: nullptr);
4018	} else if (auto *TC = FD->getAttr<TargetClonesAttr>()) {
4019	for (unsigned I = 0; I < TC->featuresStrs_size(); ++I)
4020	// AArch64 favors the default target version over the clone if any.
4021	if ((!TC->isDefaultVersion(I) || !getTarget().getTriple().isAArch64()) &&
4022	TC->isFirstOfVersion(I))
4023	EmitGlobalFunctionDefinition(GD: GD.getWithMultiVersionIndex(Index: I), GV: nullptr);
4024	// Ensure that the resolver function is also emitted.
4025	GetOrCreateMultiVersionResolver(GD);
4026	} else
4027	EmitGlobalFunctionDefinition(GD, GV);
4028
4029	// Defer the resolver emission until we can reason whether the TU
4030	// contains a default target version implementation.
4031	if (FD->isTargetVersionMultiVersion())
4032	AddDeferredMultiVersionResolverToEmit(GD);
4033	}
4034
4035	void CodeGenModule::EmitGlobalDefinition(GlobalDecl GD, llvm::GlobalValue *GV) {
4036	const auto *D = cast<ValueDecl>(Val: GD.getDecl());
4037
4038	PrettyStackTraceDecl CrashInfo(const_cast<ValueDecl *>(D), D->getLocation(),
4039	Context.getSourceManager(),
4040	"Generating code for declaration");
4041
4042	if (const auto *FD = dyn_cast<FunctionDecl>(Val: D)) {
4043	// At -O0, don't generate IR for functions with available_externally
4044	// linkage.
4045	if (!shouldEmitFunction(GD))
4046	return;
4047
4048	llvm::TimeTraceScope TimeScope("CodeGen Function", [&]() {
4049	std::string Name;
4050	llvm::raw_string_ostream OS(Name);
4051	FD->getNameForDiagnostic(OS, Policy: getContext().getPrintingPolicy(),
4052	/*Qualified=*/true);
4053	return Name;
4054	});
4055
4056	if (const auto *Method = dyn_cast<CXXMethodDecl>(Val: D)) {
4057	// Make sure to emit the definition(s) before we emit the thunks.
4058	// This is necessary for the generation of certain thunks.
4059	if (isa<CXXConstructorDecl>(Val: Method) || isa<CXXDestructorDecl>(Val: Method))
4060	ABI->emitCXXStructor(GD);
4061	else if (FD->isMultiVersion())
4062	EmitMultiVersionFunctionDefinition(GD, GV);
4063	else
4064	EmitGlobalFunctionDefinition(GD, GV);
4065
4066	if (Method->isVirtual())
4067	getVTables().EmitThunks(GD);
4068
4069	return;
4070	}
4071
4072	if (FD->isMultiVersion())
4073	return EmitMultiVersionFunctionDefinition(GD, GV);
4074	return EmitGlobalFunctionDefinition(GD, GV);
4075	}
4076
4077	if (const auto *VD = dyn_cast<VarDecl>(Val: D))
4078	return EmitGlobalVarDefinition(D: VD, IsTentative: !VD->hasDefinition());
4079
4080	llvm_unreachable("Invalid argument to EmitGlobalDefinition()");
4081	}
4082
4083	static void ReplaceUsesOfNonProtoTypeWithRealFunction(llvm::GlobalValue *Old,
4084	llvm::Function *NewFn);
4085
4086	static unsigned
4087	TargetMVPriority(const TargetInfo &TI,
4088	const CodeGenFunction::MultiVersionResolverOption &RO) {
4089	unsigned Priority = 0;
4090	unsigned NumFeatures = 0;
4091	for (StringRef Feat : RO.Conditions.Features) {
4092	Priority = std::max(a: Priority, b: TI.multiVersionSortPriority(Name: Feat));
4093	NumFeatures++;
4094	}
4095
4096	if (!RO.Conditions.Architecture.empty())
4097	Priority = std::max(
4098	a: Priority, b: TI.multiVersionSortPriority(Name: RO.Conditions.Architecture));
4099
4100	Priority += TI.multiVersionFeatureCost() * NumFeatures;
4101
4102	return Priority;
4103	}
4104
4105	// Multiversion functions should be at most 'WeakODRLinkage' so that a different
4106	// TU can forward declare the function without causing problems. Particularly
4107	// in the cases of CPUDispatch, this causes issues. This also makes sure we
4108	// work with internal linkage functions, so that the same function name can be
4109	// used with internal linkage in multiple TUs.
4110	llvm::GlobalValue::LinkageTypes getMultiversionLinkage(CodeGenModule &CGM,
4111	GlobalDecl GD) {
4112	const FunctionDecl *FD = cast<FunctionDecl>(Val: GD.getDecl());
4113	if (FD->getFormalLinkage() == Linkage::Internal)
4114	return llvm::GlobalValue::InternalLinkage;
4115	return llvm::GlobalValue::WeakODRLinkage;
4116	}
4117
4118	static FunctionDecl *createDefaultTargetVersionFrom(const FunctionDecl *FD) {
4119	DeclContext *DeclCtx = FD->getASTContext().getTranslationUnitDecl();
4120	TypeSourceInfo *TInfo = FD->getTypeSourceInfo();
4121	StorageClass SC = FD->getStorageClass();
4122	DeclarationName Name = FD->getNameInfo().getName();
4123
4124	FunctionDecl *NewDecl =
4125	FunctionDecl::Create(FD->getASTContext(), DeclCtx, FD->getBeginLoc(),
4126	FD->getEndLoc(), Name, TInfo->getType(), TInfo, SC);
4127
4128	NewDecl->setIsMultiVersion();
4129	NewDecl->addAttr(TargetVersionAttr::CreateImplicit(
4130	NewDecl->getASTContext(), "default", NewDecl->getSourceRange()));
4131
4132	return NewDecl;
4133	}
4134
4135	void CodeGenModule::emitMultiVersionFunctions() {
4136	std::vector<GlobalDecl> MVFuncsToEmit;
4137	MultiVersionFuncs.swap(x&: MVFuncsToEmit);
4138	for (GlobalDecl GD : MVFuncsToEmit) {
4139	const auto *FD = cast<FunctionDecl>(Val: GD.getDecl());
4140	assert(FD && "Expected a FunctionDecl");
4141
4142	auto createFunction = [&](const FunctionDecl *Decl, unsigned MVIdx = 0) {
4143	GlobalDecl CurGD{Decl->isDefined() ? Decl->getDefinition() : Decl, MVIdx};
4144	StringRef MangledName = getMangledName(GD: CurGD);
4145	llvm::Constant *Func = GetGlobalValue(Name: MangledName);
4146	if (!Func) {
4147	if (Decl->isDefined()) {
4148	EmitGlobalFunctionDefinition(GD: CurGD, GV: nullptr);
4149	Func = GetGlobalValue(Name: MangledName);
4150	} else {
4151	const CGFunctionInfo &FI = getTypes().arrangeGlobalDeclaration(GD: CurGD);
4152	llvm::FunctionType *Ty = getTypes().GetFunctionType(Info: FI);
4153	Func = GetAddrOfFunction(GD: CurGD, Ty, /*ForVTable=*/false,
4154	/*DontDefer=*/false, IsForDefinition: ForDefinition);
4155	}
4156	assert(Func && "This should have just been created");
4157	}
4158	return cast<llvm::Function>(Val: Func);
4159	};
4160
4161	bool HasDefaultDecl = !FD->isTargetVersionMultiVersion();
4162	bool ShouldEmitResolver =
4163	!getContext().getTargetInfo().getTriple().isAArch64();
4164	SmallVector<CodeGenFunction::MultiVersionResolverOption, 10> Options;
4165
4166	getContext().forEachMultiversionedFunctionVersion(
4167	FD, Pred: [&](const FunctionDecl *CurFD) {
4168	llvm::SmallVector<StringRef, 8> Feats;
4169
4170	if (const auto *TA = CurFD->getAttr<TargetAttr>()) {
4171	TA->getAddedFeatures(Feats);
4172	llvm::Function *Func = createFunction(CurFD);
4173	Options.emplace_back(Func, TA->getArchitecture(), Feats);
4174	} else if (const auto *TVA = CurFD->getAttr<TargetVersionAttr>()) {
4175	bool HasDefaultDef = TVA->isDefaultVersion() &&
4176	CurFD->doesThisDeclarationHaveABody();
4177	HasDefaultDecl |= TVA->isDefaultVersion();
4178	ShouldEmitResolver |= (CurFD->isUsed() || HasDefaultDef);
4179	TVA->getFeatures(Feats);
4180	llvm::Function *Func = createFunction(CurFD);
4181	Options.emplace_back(Args&: Func, /*Architecture*/ Args: "", Args&: Feats);
4182	} else if (const auto *TC = CurFD->getAttr<TargetClonesAttr>()) {
4183	ShouldEmitResolver |= CurFD->doesThisDeclarationHaveABody();
4184	for (unsigned I = 0; I < TC->featuresStrs_size(); ++I) {
4185	if (!TC->isFirstOfVersion(I))
4186	continue;
4187
4188	llvm::Function *Func = createFunction(CurFD, I);
4189	StringRef Architecture;
4190	Feats.clear();
4191	if (getTarget().getTriple().isAArch64())
4192	TC->getFeatures(Feats, I);
4193	else {
4194	StringRef Version = TC->getFeatureStr(I);
4195	if (Version.starts_with(Prefix: "arch="))
4196	Architecture = Version.drop_front(N: sizeof("arch=") - 1);
4197	else if (Version != "default")
4198	Feats.push_back(Elt: Version);
4199	}
4200	Options.emplace_back(Args&: Func, Args&: Architecture, Args&: Feats);
4201	}
4202	} else
4203	llvm_unreachable("unexpected MultiVersionKind");
4204	});
4205
4206	if (!ShouldEmitResolver)
4207	continue;
4208
4209	if (!HasDefaultDecl) {
4210	FunctionDecl *NewFD = createDefaultTargetVersionFrom(FD);
4211	llvm::Function *Func = createFunction(NewFD);
4212	llvm::SmallVector<StringRef, 1> Feats;
4213	Options.emplace_back(Args&: Func, /*Architecture*/ Args: "", Args&: Feats);
4214	}
4215
4216	llvm::Constant *ResolverConstant = GetOrCreateMultiVersionResolver(GD);
4217	if (auto *IFunc = dyn_cast<llvm::GlobalIFunc>(Val: ResolverConstant)) {
4218	ResolverConstant = IFunc->getResolver();
4219	if (FD->isTargetClonesMultiVersion() ||
4220	FD->isTargetVersionMultiVersion()) {
4221	std::string MangledName = getMangledNameImpl(
4222	*this, GD, FD, /*OmitMultiVersionMangling=*/true);
4223	if (!GetGlobalValue(Name: MangledName + ".ifunc")) {
4224	const CGFunctionInfo &FI = getTypes().arrangeGlobalDeclaration(GD);
4225	llvm::FunctionType *DeclTy = getTypes().GetFunctionType(Info: FI);
4226	// In prior versions of Clang, the mangling for ifuncs incorrectly
4227	// included an .ifunc suffix. This alias is generated for backward
4228	// compatibility. It is deprecated, and may be removed in the future.
4229	auto *Alias = llvm::GlobalAlias::create(
4230	Ty: DeclTy, AddressSpace: 0, Linkage: getMultiversionLinkage(CGM&: *this, GD),
4231	Name: MangledName + ".ifunc", Aliasee: IFunc, Parent: &getModule());
4232	SetCommonAttributes(GD: FD, GV: Alias);
4233	}
4234	}
4235	}
4236	llvm::Function *ResolverFunc = cast<llvm::Function>(Val: ResolverConstant);
4237
4238	ResolverFunc->setLinkage(getMultiversionLinkage(CGM&: *this, GD));
4239
4240	if (!ResolverFunc->hasLocalLinkage() && supportsCOMDAT())
4241	ResolverFunc->setComdat(
4242	getModule().getOrInsertComdat(Name: ResolverFunc->getName()));
4243
4244	const TargetInfo &TI = getTarget();
4245	llvm::stable_sort(
4246	Range&: Options, C: [&TI](const CodeGenFunction::MultiVersionResolverOption &LHS,
4247	const CodeGenFunction::MultiVersionResolverOption &RHS) {
4248	return TargetMVPriority(TI, RO: LHS) > TargetMVPriority(TI, RO: RHS);
4249	});
4250	CodeGenFunction CGF(*this);
4251	CGF.EmitMultiVersionResolver(Resolver: ResolverFunc, Options);
4252	}
4253
4254	// Ensure that any additions to the deferred decls list caused by emitting a
4255	// variant are emitted. This can happen when the variant itself is inline and
4256	// calls a function without linkage.
4257	if (!MVFuncsToEmit.empty())
4258	EmitDeferred();
4259
4260	// Ensure that any additions to the multiversion funcs list from either the
4261	// deferred decls or the multiversion functions themselves are emitted.
4262	if (!MultiVersionFuncs.empty())
4263	emitMultiVersionFunctions();
4264	}
4265
4266	void CodeGenModule::emitCPUDispatchDefinition(GlobalDecl GD) {
4267	const auto *FD = cast<FunctionDecl>(Val: GD.getDecl());
4268	assert(FD && "Not a FunctionDecl?");
4269	assert(FD->isCPUDispatchMultiVersion() && "Not a multiversion function?");
4270	const auto *DD = FD->getAttr<CPUDispatchAttr>();
4271	assert(DD && "Not a cpu_dispatch Function?");
4272
4273	const CGFunctionInfo &FI = getTypes().arrangeGlobalDeclaration(GD);
4274	llvm::FunctionType *DeclTy = getTypes().GetFunctionType(Info: FI);
4275
4276	StringRef ResolverName = getMangledName(GD);
4277	UpdateMultiVersionNames(GD, FD, CurName&: ResolverName);
4278
4279	llvm::Type *ResolverType;
4280	GlobalDecl ResolverGD;
4281	if (getTarget().supportsIFunc()) {
4282	ResolverType = llvm::FunctionType::get(
4283	llvm::PointerType::get(DeclTy,
4284	getTypes().getTargetAddressSpace(T: FD->getType())),
4285	false);
4286	}
4287	else {
4288	ResolverType = DeclTy;
4289	ResolverGD = GD;
4290	}
4291
4292	auto *ResolverFunc = cast<llvm::Function>(Val: GetOrCreateLLVMFunction(
4293	MangledName: ResolverName, Ty: ResolverType, D: ResolverGD, /*ForVTable=*/false));
4294	ResolverFunc->setLinkage(getMultiversionLinkage(CGM&: *this, GD));
4295	if (supportsCOMDAT())
4296	ResolverFunc->setComdat(
4297	getModule().getOrInsertComdat(Name: ResolverFunc->getName()));
4298
4299	SmallVector<CodeGenFunction::MultiVersionResolverOption, 10> Options;
4300	const TargetInfo &Target = getTarget();
4301	unsigned Index = 0;
4302	for (const IdentifierInfo *II : DD->cpus()) {
4303	// Get the name of the target function so we can look it up/create it.
4304	std::string MangledName = getMangledNameImpl(*this, GD, FD, true) +
4305	getCPUSpecificMangling(*this, II->getName());
4306
4307	llvm::Constant *Func = GetGlobalValue(MangledName);
4308
4309	if (!Func) {
4310	GlobalDecl ExistingDecl = Manglings.lookup(MangledName);
4311	if (ExistingDecl.getDecl() &&
4312	ExistingDecl.getDecl()->getAsFunction()->isDefined()) {
4313	EmitGlobalFunctionDefinition(ExistingDecl, nullptr);
4314	Func = GetGlobalValue(MangledName);
4315	} else {
4316	if (!ExistingDecl.getDecl())
4317	ExistingDecl = GD.getWithMultiVersionIndex(Index);
4318
4319	Func = GetOrCreateLLVMFunction(
4320	MangledName, DeclTy, ExistingDecl,
4321	/*ForVTable=*/false, /*DontDefer=*/true,
4322	/*IsThunk=*/false, llvm::AttributeList(), ForDefinition);
4323	}
4324	}
4325
4326	llvm::SmallVector<StringRef, 32> Features;
4327	Target.getCPUSpecificCPUDispatchFeatures(II->getName(), Features);
4328	llvm::transform(Features, Features.begin(),
4329	[](StringRef Str) { return Str.substr(1); });
4330	llvm::erase_if(Features, [&Target](StringRef Feat) {
4331	return !Target.validateCpuSupports(Feat);
4332	});
4333	Options.emplace_back(cast<llvm::Function>(Func), StringRef{}, Features);
4334	++Index;
4335	}
4336
4337	llvm::stable_sort(
4338	Range&: Options, C: [](const CodeGenFunction::MultiVersionResolverOption &LHS,
4339	const CodeGenFunction::MultiVersionResolverOption &RHS) {
4340	return llvm::X86::getCpuSupportsMask(FeatureStrs: LHS.Conditions.Features) >
4341	llvm::X86::getCpuSupportsMask(FeatureStrs: RHS.Conditions.Features);
4342	});
4343
4344	// If the list contains multiple 'default' versions, such as when it contains
4345	// 'pentium' and 'generic', don't emit the call to the generic one (since we
4346	// always run on at least a 'pentium'). We do this by deleting the 'least
4347	// advanced' (read, lowest mangling letter).
4348	while (Options.size() > 1 &&
4349	llvm::all_of(Range: llvm::X86::getCpuSupportsMask(
4350	FeatureStrs: (Options.end() - 2)->Conditions.Features),
4351	P: [](auto X) { return X == 0; })) {
4352	StringRef LHSName = (Options.end() - 2)->Function->getName();
4353	StringRef RHSName = (Options.end() - 1)->Function->getName();
4354	if (LHSName.compare(RHS: RHSName) < 0)
4355	Options.erase(CI: Options.end() - 2);
4356	else
4357	Options.erase(CI: Options.end() - 1);
4358	}
4359
4360	CodeGenFunction CGF(*this);
4361	CGF.EmitMultiVersionResolver(Resolver: ResolverFunc, Options);
4362
4363	if (getTarget().supportsIFunc()) {
4364	llvm::GlobalValue::LinkageTypes Linkage = getMultiversionLinkage(CGM&: *this, GD);
4365	auto *IFunc = cast<llvm::GlobalValue>(Val: GetOrCreateMultiVersionResolver(GD));
4366
4367	// Fix up function declarations that were created for cpu_specific before
4368	// cpu_dispatch was known
4369	if (!isa<llvm::GlobalIFunc>(Val: IFunc)) {
4370	assert(cast<llvm::Function>(IFunc)->isDeclaration());
4371	auto *GI = llvm::GlobalIFunc::create(Ty: DeclTy, AddressSpace: 0, Linkage, Name: "", Resolver: ResolverFunc,
4372	Parent: &getModule());
4373	GI->takeName(V: IFunc);
4374	IFunc->replaceAllUsesWith(V: GI);
4375	IFunc->eraseFromParent();
4376	IFunc = GI;
4377	}
4378
4379	std::string AliasName = getMangledNameImpl(
4380	*this, GD, FD, /*OmitMultiVersionMangling=*/true);
4381	llvm::Constant *AliasFunc = GetGlobalValue(Name: AliasName);
4382	if (!AliasFunc) {
4383	auto *GA = llvm::GlobalAlias::create(Ty: DeclTy, AddressSpace: 0, Linkage, Name: AliasName, Aliasee: IFunc,
4384	Parent: &getModule());
4385	SetCommonAttributes(GD, GV: GA);
4386	}
4387	}
4388	}
4389
4390	/// Adds a declaration to the list of multi version functions if not present.
4391	void CodeGenModule::AddDeferredMultiVersionResolverToEmit(GlobalDecl GD) {
4392	const auto *FD = cast<FunctionDecl>(Val: GD.getDecl());
4393	assert(FD && "Not a FunctionDecl?");
4394
4395	if (FD->isTargetVersionMultiVersion() || FD->isTargetClonesMultiVersion()) {
4396	std::string MangledName =
4397	getMangledNameImpl(*this, GD, FD, /*OmitMultiVersionMangling=*/true);
4398	if (!DeferredResolversToEmit.insert(key: MangledName).second)
4399	return;
4400	}
4401	MultiVersionFuncs.push_back(x: GD);
4402	}
4403
4404	/// If a dispatcher for the specified mangled name is not in the module, create
4405	/// and return an llvm Function with the specified type.
4406	llvm::Constant *CodeGenModule::GetOrCreateMultiVersionResolver(GlobalDecl GD) {
4407	const auto *FD = cast<FunctionDecl>(Val: GD.getDecl());
4408	assert(FD && "Not a FunctionDecl?");
4409
4410	std::string MangledName =
4411	getMangledNameImpl(*this, GD, FD, /*OmitMultiVersionMangling=*/true);
4412
4413	// Holds the name of the resolver, in ifunc mode this is the ifunc (which has
4414	// a separate resolver).
4415	std::string ResolverName = MangledName;
4416	if (getTarget().supportsIFunc()) {
4417	switch (FD->getMultiVersionKind()) {
4418	case MultiVersionKind::None:
4419	llvm_unreachable("unexpected MultiVersionKind::None for resolver");
4420	case MultiVersionKind::Target:
4421	case MultiVersionKind::CPUSpecific:
4422	case MultiVersionKind::CPUDispatch:
4423	ResolverName += ".ifunc";
4424	break;
4425	case MultiVersionKind::TargetClones:
4426	case MultiVersionKind::TargetVersion:
4427	break;
4428	}
4429	} else if (FD->isTargetMultiVersion()) {
4430	ResolverName += ".resolver";
4431	}
4432
4433	// If the resolver has already been created, just return it.
4434	if (llvm::GlobalValue *ResolverGV = GetGlobalValue(Name: ResolverName))
4435	return ResolverGV;
4436
4437	const CGFunctionInfo &FI = getTypes().arrangeGlobalDeclaration(GD);
4438	llvm::FunctionType *DeclTy = getTypes().GetFunctionType(Info: FI);
4439
4440	// The resolver needs to be created. For target and target_clones, defer
4441	// creation until the end of the TU.
4442	if (FD->isTargetMultiVersion() || FD->isTargetClonesMultiVersion())
4443	AddDeferredMultiVersionResolverToEmit(GD);
4444
4445	// For cpu_specific, don't create an ifunc yet because we don't know if the
4446	// cpu_dispatch will be emitted in this translation unit.
4447	if (getTarget().supportsIFunc() && !FD->isCPUSpecificMultiVersion()) {
4448	llvm::Type *ResolverType = llvm::FunctionType::get(
4449	llvm::PointerType::get(DeclTy,
4450	getTypes().getTargetAddressSpace(T: FD->getType())),
4451	false);
4452	llvm::Constant *Resolver = GetOrCreateLLVMFunction(
4453	MangledName: MangledName + ".resolver", Ty: ResolverType, D: GlobalDecl{},
4454	/*ForVTable=*/false);
4455	llvm::GlobalIFunc *GIF =
4456	llvm::GlobalIFunc::create(Ty: DeclTy, AddressSpace: 0, Linkage: getMultiversionLinkage(CGM&: *this, GD),
4457	Name: "", Resolver, Parent: &getModule());
4458	GIF->setName(ResolverName);
4459	SetCommonAttributes(GD: FD, GV: GIF);
4460
4461	return GIF;
4462	}
4463
4464	llvm::Constant *Resolver = GetOrCreateLLVMFunction(
4465	MangledName: ResolverName, Ty: DeclTy, D: GlobalDecl{}, /*ForVTable=*/false);
4466	assert(isa<llvm::GlobalValue>(Resolver) &&
4467	"Resolver should be created for the first time");
4468	SetCommonAttributes(GD: FD, GV: cast<llvm::GlobalValue>(Val: Resolver));
4469	return Resolver;
4470	}
4471
4472	/// GetOrCreateLLVMFunction - If the specified mangled name is not in the
4473	/// module, create and return an llvm Function with the specified type. If there
4474	/// is something in the module with the specified name, return it potentially
4475	/// bitcasted to the right type.
4476	///
4477	/// If D is non-null, it specifies a decl that correspond to this. This is used
4478	/// to set the attributes on the function when it is first created.
4479	llvm::Constant *CodeGenModule::GetOrCreateLLVMFunction(
4480	StringRef MangledName, llvm::Type *Ty, GlobalDecl GD, bool ForVTable,
4481	bool DontDefer, bool IsThunk, llvm::AttributeList ExtraAttrs,
4482	ForDefinition_t IsForDefinition) {
4483	const Decl *D = GD.getDecl();
4484
4485	// Any attempts to use a MultiVersion function should result in retrieving
4486	// the iFunc instead. Name Mangling will handle the rest of the changes.
4487	if (const FunctionDecl *FD = cast_or_null<FunctionDecl>(Val: D)) {
4488	// For the device mark the function as one that should be emitted.
4489	if (getLangOpts().OpenMPIsTargetDevice && OpenMPRuntime &&
4490	!OpenMPRuntime->markAsGlobalTarget(GD) && FD->isDefined() &&
4491	!DontDefer && !IsForDefinition) {
4492	if (const FunctionDecl *FDDef = FD->getDefinition()) {
4493	GlobalDecl GDDef;
4494	if (const auto *CD = dyn_cast<CXXConstructorDecl>(Val: FDDef))
4495	GDDef = GlobalDecl(CD, GD.getCtorType());
4496	else if (const auto *DD = dyn_cast<CXXDestructorDecl>(Val: FDDef))
4497	GDDef = GlobalDecl(DD, GD.getDtorType());
4498	else
4499	GDDef = GlobalDecl(FDDef);
4500	EmitGlobal(GD: GDDef);
4501	}
4502	}
4503
4504	if (FD->isMultiVersion()) {
4505	UpdateMultiVersionNames(GD, FD, CurName&: MangledName);
4506	if (FD->getASTContext().getTargetInfo().getTriple().isAArch64() &&
4507	!FD->isUsed())
4508	AddDeferredMultiVersionResolverToEmit(GD);
4509	else if (!IsForDefinition)
4510	return GetOrCreateMultiVersionResolver(GD);
4511	}
4512	}
4513
4514	// Lookup the entry, lazily creating it if necessary.
4515	llvm::GlobalValue *Entry = GetGlobalValue(Name: MangledName);
4516	if (Entry) {
4517	if (WeakRefReferences.erase(Ptr: Entry)) {
4518	const FunctionDecl *FD = cast_or_null<FunctionDecl>(Val: D);
4519	if (FD && !FD->hasAttr<WeakAttr>())
4520	Entry->setLinkage(llvm::Function::ExternalLinkage);
4521	}
4522
4523	// Handle dropped DLL attributes.
4524	if (D && !D->hasAttr<DLLImportAttr>() && !D->hasAttr<DLLExportAttr>() &&
4525	!shouldMapVisibilityToDLLExport(cast_or_null<NamedDecl>(D))) {
4526	Entry->setDLLStorageClass(llvm::GlobalValue::DefaultStorageClass);
4527	setDSOLocal(Entry);
4528	}
4529
4530	// If there are two attempts to define the same mangled name, issue an
4531	// error.
4532	if (IsForDefinition && !Entry->isDeclaration()) {
4533	GlobalDecl OtherGD;
4534	// Check that GD is not yet in DiagnosedConflictingDefinitions is required
4535	// to make sure that we issue an error only once.
4536	if (lookupRepresentativeDecl(MangledName, Result&: OtherGD) &&
4537	(GD.getCanonicalDecl().getDecl() !=
4538	OtherGD.getCanonicalDecl().getDecl()) &&
4539	DiagnosedConflictingDefinitions.insert(V: GD).second) {
4540	getDiags().Report(D->getLocation(), diag::err_duplicate_mangled_name)
4541	<< MangledName;
4542	getDiags().Report(OtherGD.getDecl()->getLocation(),
4543	diag::note_previous_definition);
4544	}
4545	}
4546
4547	if ((isa<llvm::Function>(Val: Entry) || isa<llvm::GlobalAlias>(Val: Entry)) &&
4548	(Entry->getValueType() == Ty)) {
4549	return Entry;
4550	}
4551
4552	// Make sure the result is of the correct type.
4553	// (If function is requested for a definition, we always need to create a new
4554	// function, not just return a bitcast.)
4555	if (!IsForDefinition)
4556	return Entry;
4557	}
4558
4559	// This function doesn't have a complete type (for example, the return
4560	// type is an incomplete struct). Use a fake type instead, and make
4561	// sure not to try to set attributes.
4562	bool IsIncompleteFunction = false;
4563
4564	llvm::FunctionType *FTy;
4565	if (isa<llvm::FunctionType>(Val: Ty)) {
4566	FTy = cast<llvm::FunctionType>(Val: Ty);
4567	} else {
4568	FTy = llvm::FunctionType::get(Result: VoidTy, isVarArg: false);
4569	IsIncompleteFunction = true;
4570	}
4571
4572	llvm::Function *F =
4573	llvm::Function::Create(Ty: FTy, Linkage: llvm::Function::ExternalLinkage,
4574	N: Entry ? StringRef() : MangledName, M: &getModule());
4575
4576	// Store the declaration associated with this function so it is potentially
4577	// updated by further declarations or definitions and emitted at the end.
4578	if (D && D->hasAttr<AnnotateAttr>())
4579	DeferredAnnotations[MangledName] = cast<ValueDecl>(Val: D);
4580
4581	// If we already created a function with the same mangled name (but different
4582	// type) before, take its name and add it to the list of functions to be
4583	// replaced with F at the end of CodeGen.
4584	//
4585	// This happens if there is a prototype for a function (e.g. "int f()") and
4586	// then a definition of a different type (e.g. "int f(int x)").
4587	if (Entry) {
4588	F->takeName(V: Entry);
4589
4590	// This might be an implementation of a function without a prototype, in
4591	// which case, try to do special replacement of calls which match the new
4592	// prototype. The really key thing here is that we also potentially drop
4593	// arguments from the call site so as to make a direct call, which makes the
4594	// inliner happier and suppresses a number of optimizer warnings (!) about
4595	// dropping arguments.
4596	if (!Entry->use_empty()) {
4597	ReplaceUsesOfNonProtoTypeWithRealFunction(Old: Entry, NewFn: F);
4598	Entry->removeDeadConstantUsers();
4599	}
4600
4601	addGlobalValReplacement(GV: Entry, C: F);
4602	}
4603
4604	assert(F->getName() == MangledName && "name was uniqued!");
4605	if (D)
4606	SetFunctionAttributes(GD, F, IsIncompleteFunction, IsThunk);
4607	if (ExtraAttrs.hasFnAttrs()) {
4608	llvm::AttrBuilder B(F->getContext(), ExtraAttrs.getFnAttrs());
4609	F->addFnAttrs(Attrs: B);
4610	}
4611
4612	if (!DontDefer) {
4613	// All MSVC dtors other than the base dtor are linkonce_odr and delegate to
4614	// each other bottoming out with the base dtor. Therefore we emit non-base
4615	// dtors on usage, even if there is no dtor definition in the TU.
4616	if (isa_and_nonnull<CXXDestructorDecl>(Val: D) &&
4617	getCXXABI().useThunkForDtorVariant(Dtor: cast<CXXDestructorDecl>(Val: D),
4618	DT: GD.getDtorType()))
4619	addDeferredDeclToEmit(GD);
4620
4621	// This is the first use or definition of a mangled name. If there is a
4622	// deferred decl with this name, remember that we need to emit it at the end
4623	// of the file.
4624	auto DDI = DeferredDecls.find(Val: MangledName);
4625	if (DDI != DeferredDecls.end()) {
4626	// Move the potentially referenced deferred decl to the
4627	// DeferredDeclsToEmit list, and remove it from DeferredDecls (since we
4628	// don't need it anymore).
4629	addDeferredDeclToEmit(GD: DDI->second);
4630	DeferredDecls.erase(I: DDI);
4631
4632	// Otherwise, there are cases we have to worry about where we're
4633	// using a declaration for which we must emit a definition but where
4634	// we might not find a top-level definition:
4635	// - member functions defined inline in their classes
4636	// - friend functions defined inline in some class
4637	// - special member functions with implicit definitions
4638	// If we ever change our AST traversal to walk into class methods,
4639	// this will be unnecessary.
4640	//
4641	// We also don't emit a definition for a function if it's going to be an
4642	// entry in a vtable, unless it's already marked as used.
4643	} else if (getLangOpts().CPlusPlus && D) {
4644	// Look for a declaration that's lexically in a record.
4645	for (const auto *FD = cast<FunctionDecl>(Val: D)->getMostRecentDecl(); FD;
4646	FD = FD->getPreviousDecl()) {
4647	if (isa<CXXRecordDecl>(FD->getLexicalDeclContext())) {
4648	if (FD->doesThisDeclarationHaveABody()) {
4649	addDeferredDeclToEmit(GD: GD.getWithDecl(D: FD));
4650	break;
4651	}
4652	}
4653	}
4654	}
4655	}
4656
4657	// Make sure the result is of the requested type.
4658	if (!IsIncompleteFunction) {
4659	assert(F->getFunctionType() == Ty);
4660	return F;
4661	}
4662
4663	return F;
4664	}
4665
4666	/// GetAddrOfFunction - Return the address of the given function. If Ty is
4667	/// non-null, then this function will use the specified type if it has to
4668	/// create it (this occurs when we see a definition of the function).
4669	llvm::Constant *
4670	CodeGenModule::GetAddrOfFunction(GlobalDecl GD, llvm::Type *Ty, bool ForVTable,
4671	bool DontDefer,
4672	ForDefinition_t IsForDefinition) {
4673	// If there was no specific requested type, just convert it now.
4674	if (!Ty) {
4675	const auto *FD = cast<FunctionDecl>(Val: GD.getDecl());
4676	Ty = getTypes().ConvertType(T: FD->getType());
4677	}
4678
4679	// Devirtualized destructor calls may come through here instead of via
4680	// getAddrOfCXXStructor. Make sure we use the MS ABI base destructor instead
4681	// of the complete destructor when necessary.
4682	if (const auto *DD = dyn_cast<CXXDestructorDecl>(Val: GD.getDecl())) {
4683	if (getTarget().getCXXABI().isMicrosoft() &&
4684	GD.getDtorType() == Dtor_Complete &&
4685	DD->getParent()->getNumVBases() == 0)
4686	GD = GlobalDecl(DD, Dtor_Base);
4687	}
4688
4689	StringRef MangledName = getMangledName(GD);
4690	auto *F = GetOrCreateLLVMFunction(MangledName, Ty, GD, ForVTable, DontDefer,
4691	/*IsThunk=*/false, ExtraAttrs: llvm::AttributeList(),
4692	IsForDefinition);
4693	// Returns kernel handle for HIP kernel stub function.
4694	if (LangOpts.CUDA && !LangOpts.CUDAIsDevice &&
4695	cast<FunctionDecl>(GD.getDecl())->hasAttr<CUDAGlobalAttr>()) {
4696	auto *Handle = getCUDARuntime().getKernelHandle(
4697	Stub: cast<llvm::Function>(Val: F->stripPointerCasts()), GD);
4698	if (IsForDefinition)
4699	return F;
4700	return Handle;
4701	}
4702	return F;
4703	}
4704
4705	llvm::Constant *CodeGenModule::GetFunctionStart(const ValueDecl *Decl) {
4706	llvm::GlobalValue *F =
4707	cast<llvm::GlobalValue>(Val: GetAddrOfFunction(Decl)->stripPointerCasts());
4708
4709	return llvm::NoCFIValue::get(GV: F);
4710	}
4711
4712	static const FunctionDecl *
4713	GetRuntimeFunctionDecl(ASTContext &C, StringRef Name) {
4714	TranslationUnitDecl *TUDecl = C.getTranslationUnitDecl();
4715	DeclContext *DC = TranslationUnitDecl::castToDeclContext(D: TUDecl);
4716
4717	IdentifierInfo &CII = C.Idents.get(Name);
4718	for (const auto *Result : DC->lookup(Name: &CII))
4719	if (const auto *FD = dyn_cast<FunctionDecl>(Val: Result))
4720	return FD;
4721
4722	if (!C.getLangOpts().CPlusPlus)
4723	return nullptr;
4724
4725	// Demangle the premangled name from getTerminateFn()
4726	IdentifierInfo &CXXII =
4727	(Name == "_ZSt9terminatev" || Name == "?terminate@@YAXXZ")
4728	? C.Idents.get(Name: "terminate")
4729	: C.Idents.get(Name);
4730
4731	for (const auto &N : {"__cxxabiv1", "std"}) {
4732	IdentifierInfo &NS = C.Idents.get(Name: N);
4733	for (const auto *Result : DC->lookup(Name: &NS)) {
4734	const NamespaceDecl *ND = dyn_cast<NamespaceDecl>(Val: Result);
4735	if (auto *LSD = dyn_cast<LinkageSpecDecl>(Result))
4736	for (const auto *Result : LSD->lookup(&NS))
4737	if ((ND = dyn_cast<NamespaceDecl>(Result)))
4738	break;
4739
4740	if (ND)
4741	for (const auto *Result : ND->lookup(&CXXII))
4742	if (const auto *FD = dyn_cast<FunctionDecl>(Result))
4743	return FD;
4744	}
4745	}
4746
4747	return nullptr;
4748	}
4749
4750	/// CreateRuntimeFunction - Create a new runtime function with the specified
4751	/// type and name.
4752	llvm::FunctionCallee
4753	CodeGenModule::CreateRuntimeFunction(llvm::FunctionType *FTy, StringRef Name,
4754	llvm::AttributeList ExtraAttrs, bool Local,
4755	bool AssumeConvergent) {
4756	if (AssumeConvergent) {
4757	ExtraAttrs =
4758	ExtraAttrs.addFnAttribute(VMContext, llvm::Attribute::Convergent);
4759	}
4760
4761	llvm::Constant *C =
4762	GetOrCreateLLVMFunction(MangledName: Name, Ty: FTy, GD: GlobalDecl(), /*ForVTable=*/false,
4763	/*DontDefer=*/false, /*IsThunk=*/false,
4764	ExtraAttrs);
4765
4766	if (auto *F = dyn_cast<llvm::Function>(Val: C)) {
4767	if (F->empty()) {
4768	F->setCallingConv(getRuntimeCC());
4769
4770	// In Windows Itanium environments, try to mark runtime functions
4771	// dllimport. For Mingw and MSVC, don't. We don't really know if the user
4772	// will link their standard library statically or dynamically. Marking
4773	// functions imported when they are not imported can cause linker errors
4774	// and warnings.
4775	if (!Local && getTriple().isWindowsItaniumEnvironment() &&
4776	!getCodeGenOpts().LTOVisibilityPublicStd) {
4777	const FunctionDecl *FD = GetRuntimeFunctionDecl(C&: Context, Name);
4778	if (!FD || FD->hasAttr<DLLImportAttr>()) {
4779	F->setDLLStorageClass(llvm::GlobalValue::DLLImportStorageClass);
4780	F->setLinkage(llvm::GlobalValue::ExternalLinkage);
4781	}
4782	}
4783	setDSOLocal(F);
4784	}
4785	}
4786
4787	return {FTy, C};
4788	}
4789
4790	/// GetOrCreateLLVMGlobal - If the specified mangled name is not in the module,
4791	/// create and return an llvm GlobalVariable with the specified type and address
4792	/// space. If there is something in the module with the specified name, return
4793	/// it potentially bitcasted to the right type.
4794	///
4795	/// If D is non-null, it specifies a decl that correspond to this. This is used
4796	/// to set the attributes on the global when it is first created.
4797	///
4798	/// If IsForDefinition is true, it is guaranteed that an actual global with
4799	/// type Ty will be returned, not conversion of a variable with the same
4800	/// mangled name but some other type.
4801	llvm::Constant *
4802	CodeGenModule::GetOrCreateLLVMGlobal(StringRef MangledName, llvm::Type *Ty,
4803	LangAS AddrSpace, const VarDecl *D,
4804	ForDefinition_t IsForDefinition) {
4805	// Lookup the entry, lazily creating it if necessary.
4806	llvm::GlobalValue *Entry = GetGlobalValue(Name: MangledName);
4807	unsigned TargetAS = getContext().getTargetAddressSpace(AS: AddrSpace);
4808	if (Entry) {
4809	if (WeakRefReferences.erase(Ptr: Entry)) {
4810	if (D && !D->hasAttr<WeakAttr>())
4811	Entry->setLinkage(llvm::Function::ExternalLinkage);
4812	}
4813
4814	// Handle dropped DLL attributes.
4815	if (D && !D->hasAttr<DLLImportAttr>() && !D->hasAttr<DLLExportAttr>() &&
4816	!shouldMapVisibilityToDLLExport(D))
4817	Entry->setDLLStorageClass(llvm::GlobalValue::DefaultStorageClass);
4818
4819	if (LangOpts.OpenMP && !LangOpts.OpenMPSimd && D)
4820	getOpenMPRuntime().registerTargetGlobalVariable(VD: D, Addr: Entry);
4821
4822	if (Entry->getValueType() == Ty && Entry->getAddressSpace() == TargetAS)
4823	return Entry;
4824
4825	// If there are two attempts to define the same mangled name, issue an
4826	// error.
4827	if (IsForDefinition && !Entry->isDeclaration()) {
4828	GlobalDecl OtherGD;
4829	const VarDecl *OtherD;
4830
4831	// Check that D is not yet in DiagnosedConflictingDefinitions is required
4832	// to make sure that we issue an error only once.
4833	if (D && lookupRepresentativeDecl(MangledName, Result&: OtherGD) &&
4834	(D->getCanonicalDecl() != OtherGD.getCanonicalDecl().getDecl()) &&
4835	(OtherD = dyn_cast<VarDecl>(Val: OtherGD.getDecl())) &&
4836	OtherD->hasInit() &&
4837	DiagnosedConflictingDefinitions.insert(V: D).second) {
4838	getDiags().Report(D->getLocation(), diag::err_duplicate_mangled_name)
4839	<< MangledName;
4840	getDiags().Report(OtherGD.getDecl()->getLocation(),
4841	diag::note_previous_definition);
4842	}
4843	}
4844
4845	// Make sure the result is of the correct type.
4846	if (Entry->getType()->getAddressSpace() != TargetAS)
4847	return llvm::ConstantExpr::getAddrSpaceCast(
4848	C: Entry, Ty: llvm::PointerType::get(C&: Ty->getContext(), AddressSpace: TargetAS));
4849
4850	// (If global is requested for a definition, we always need to create a new
4851	// global, not just return a bitcast.)
4852	if (!IsForDefinition)
4853	return Entry;
4854	}
4855
4856	auto DAddrSpace = GetGlobalVarAddressSpace(D);
4857
4858	auto *GV = new llvm::GlobalVariable(
4859	getModule(), Ty, false, llvm::GlobalValue::ExternalLinkage, nullptr,
4860	MangledName, nullptr, llvm::GlobalVariable::NotThreadLocal,
4861	getContext().getTargetAddressSpace(AS: DAddrSpace));
4862
4863	// If we already created a global with the same mangled name (but different
4864	// type) before, take its name and remove it from its parent.
4865	if (Entry) {
4866	GV->takeName(V: Entry);
4867
4868	if (!Entry->use_empty()) {
4869	Entry->replaceAllUsesWith(V: GV);
4870	}
4871
4872	Entry->eraseFromParent();
4873	}
4874
4875	// This is the first use or definition of a mangled name. If there is a
4876	// deferred decl with this name, remember that we need to emit it at the end
4877	// of the file.
4878	auto DDI = DeferredDecls.find(Val: MangledName);
4879	if (DDI != DeferredDecls.end()) {
4880	// Move the potentially referenced deferred decl to the DeferredDeclsToEmit
4881	// list, and remove it from DeferredDecls (since we don't need it anymore).
4882	addDeferredDeclToEmit(GD: DDI->second);
4883	DeferredDecls.erase(I: DDI);
4884	}
4885
4886	// Handle things which are present even on external declarations.
4887	if (D) {
4888	if (LangOpts.OpenMP && !LangOpts.OpenMPSimd)
4889	getOpenMPRuntime().registerTargetGlobalVariable(VD: D, Addr: GV);
4890
4891	// FIXME: This code is overly simple and should be merged with other global
4892	// handling.
4893	GV->setConstant(D->getType().isConstantStorage(getContext(), false, false));
4894
4895	GV->setAlignment(getContext().getDeclAlign(D).getAsAlign());
4896
4897	setLinkageForGV(GV, D);
4898
4899	if (D->getTLSKind()) {
4900	if (D->getTLSKind() == VarDecl::TLS_Dynamic)
4901	CXXThreadLocals.push_back(x: D);
4902	setTLSMode(GV, D: *D);
4903	}
4904
4905	setGVProperties(GV, GD: D);
4906
4907	// If required by the ABI, treat declarations of static data members with
4908	// inline initializers as definitions.
4909	if (getContext().isMSStaticDataMemberInlineDefinition(VD: D)) {
4910	EmitGlobalVarDefinition(D);
4911	}
4912
4913	// Emit section information for extern variables.
4914	if (D->hasExternalStorage()) {
4915	if (const SectionAttr *SA = D->getAttr<SectionAttr>())
4916	GV->setSection(SA->getName());
4917	}
4918
4919	// Handle XCore specific ABI requirements.
4920	if (getTriple().getArch() == llvm::Triple::xcore &&
4921	D->getLanguageLinkage() == CLanguageLinkage &&
4922	D->getType().isConstant(Context) &&
4923	isExternallyVisible(D->getLinkageAndVisibility().getLinkage()))
4924	GV->setSection(".cp.rodata");
4925
4926	// Handle code model attribute
4927	if (const auto *CMA = D->getAttr<CodeModelAttr>())
4928	GV->setCodeModel(CMA->getModel());
4929
4930	// Check if we a have a const declaration with an initializer, we may be
4931	// able to emit it as available_externally to expose it's value to the
4932	// optimizer.
4933	if (Context.getLangOpts().CPlusPlus && GV->hasExternalLinkage() &&
4934	D->getType().isConstQualified() && !GV->hasInitializer() &&
4935	!D->hasDefinition() && D->hasInit() && !D->hasAttr<DLLImportAttr>()) {
4936	const auto *Record =
4937	Context.getBaseElementType(D->getType())->getAsCXXRecordDecl();
4938	bool HasMutableFields = Record && Record->hasMutableFields();
4939	if (!HasMutableFields) {
4940	const VarDecl *InitDecl;
4941	const Expr *InitExpr = D->getAnyInitializer(D&: InitDecl);
4942	if (InitExpr) {
4943	ConstantEmitter emitter(*this);
4944	llvm::Constant *Init = emitter.tryEmitForInitializer(D: *InitDecl);
4945	if (Init) {
4946	auto *InitType = Init->getType();
4947	if (GV->getValueType() != InitType) {
4948	// The type of the initializer does not match the definition.
4949	// This happens when an initializer has a different type from
4950	// the type of the global (because of padding at the end of a
4951	// structure for instance).
4952	GV->setName(StringRef());
4953	// Make a new global with the correct type, this is now guaranteed
4954	// to work.
4955	auto *NewGV = cast<llvm::GlobalVariable>(
4956	Val: GetAddrOfGlobalVar(D, Ty: InitType, IsForDefinition)
4957	->stripPointerCasts());
4958
4959	// Erase the old global, since it is no longer used.
4960	GV->eraseFromParent();
4961	GV = NewGV;
4962	} else {
4963	GV->setInitializer(Init);
4964	GV->setConstant(true);
4965	GV->setLinkage(llvm::GlobalValue::AvailableExternallyLinkage);
4966	}
4967	emitter.finalize(global: GV);
4968	}
4969	}
4970	}
4971	}
4972	}
4973
4974	if (D &&
4975	D->isThisDeclarationADefinition(Context) == VarDecl::DeclarationOnly) {
4976	getTargetCodeGenInfo().setTargetAttributes(D, GV, *this);
4977	// External HIP managed variables needed to be recorded for transformation
4978	// in both device and host compilations.
4979	if (getLangOpts().CUDA && D && D->hasAttr<HIPManagedAttr>() &&
4980	D->hasExternalStorage())
4981	getCUDARuntime().handleVarRegistration(VD: D, Var&: *GV);
4982	}
4983
4984	if (D)
4985	SanitizerMD->reportGlobal(GV, D: *D);
4986
4987	LangAS ExpectedAS =
4988	D ? D->getType().getAddressSpace()
4989	: (LangOpts.OpenCL ? LangAS::opencl_global : LangAS::Default);
4990	assert(getContext().getTargetAddressSpace(ExpectedAS) == TargetAS);
4991	if (DAddrSpace != ExpectedAS) {
4992	return getTargetCodeGenInfo().performAddrSpaceCast(
4993	CGM&: *this, V: GV, SrcAddr: DAddrSpace, DestAddr: ExpectedAS,
4994	DestTy: llvm::PointerType::get(C&: getLLVMContext(), AddressSpace: TargetAS));
4995	}
4996
4997	return GV;
4998	}
4999
5000	llvm::Constant *
5001	CodeGenModule::GetAddrOfGlobal(GlobalDecl GD, ForDefinition_t IsForDefinition) {
5002	const Decl *D = GD.getDecl();
5003
5004	if (isa<CXXConstructorDecl>(Val: D) || isa<CXXDestructorDecl>(Val: D))
5005	return getAddrOfCXXStructor(GD, /*FnInfo=*/nullptr, /*FnType=*/nullptr,
5006	/*DontDefer=*/false, IsForDefinition);
5007
5008	if (isa<CXXMethodDecl>(Val: D)) {
5009	auto FInfo =
5010	&getTypes().arrangeCXXMethodDeclaration(MD: cast<CXXMethodDecl>(Val: D));
5011	auto Ty = getTypes().GetFunctionType(Info: *FInfo);
5012	return GetAddrOfFunction(GD, Ty, /*ForVTable=*/false, /*DontDefer=*/false,
5013	IsForDefinition);
5014	}
5015
5016	if (isa<FunctionDecl>(Val: D)) {
5017	const CGFunctionInfo &FI = getTypes().arrangeGlobalDeclaration(GD);
5018	llvm::FunctionType *Ty = getTypes().GetFunctionType(Info: FI);
5019	return GetAddrOfFunction(GD, Ty, /*ForVTable=*/false, /*DontDefer=*/false,
5020	IsForDefinition);
5021	}
5022
5023	return GetAddrOfGlobalVar(D: cast<VarDecl>(Val: D), /*Ty=*/nullptr, IsForDefinition);
5024	}
5025
5026	llvm::GlobalVariable *CodeGenModule::CreateOrReplaceCXXRuntimeVariable(
5027	StringRef Name, llvm::Type *Ty, llvm::GlobalValue::LinkageTypes Linkage,
5028	llvm::Align Alignment) {
5029	llvm::GlobalVariable *GV = getModule().getNamedGlobal(Name);
5030	llvm::GlobalVariable *OldGV = nullptr;
5031
5032	if (GV) {
5033	// Check if the variable has the right type.
5034	if (GV->getValueType() == Ty)
5035	return GV;
5036
5037	// Because C++ name mangling, the only way we can end up with an already
5038	// existing global with the same name is if it has been declared extern "C".
5039	assert(GV->isDeclaration() && "Declaration has wrong type!");
5040	OldGV = GV;
5041	}
5042
5043	// Create a new variable.
5044	GV = new llvm::GlobalVariable(getModule(), Ty, /*isConstant=*/true,
5045	Linkage, nullptr, Name);
5046
5047	if (OldGV) {
5048	// Replace occurrences of the old variable if needed.
5049	GV->takeName(V: OldGV);
5050
5051	if (!OldGV->use_empty()) {
5052	OldGV->replaceAllUsesWith(V: GV);
5053	}
5054
5055	OldGV->eraseFromParent();
5056	}
5057
5058	if (supportsCOMDAT() && GV->isWeakForLinker() &&
5059	!GV->hasAvailableExternallyLinkage())
5060	GV->setComdat(TheModule.getOrInsertComdat(Name: GV->getName()));
5061
5062	GV->setAlignment(Alignment);
5063
5064	return GV;
5065	}
5066
5067	/// GetAddrOfGlobalVar - Return the llvm::Constant for the address of the
5068	/// given global variable. If Ty is non-null and if the global doesn't exist,
5069	/// then it will be created with the specified type instead of whatever the
5070	/// normal requested type would be. If IsForDefinition is true, it is guaranteed
5071	/// that an actual global with type Ty will be returned, not conversion of a
5072	/// variable with the same mangled name but some other type.
5073	llvm::Constant *CodeGenModule::GetAddrOfGlobalVar(const VarDecl *D,
5074	llvm::Type *Ty,
5075	ForDefinition_t IsForDefinition) {
5076	assert(D->hasGlobalStorage() && "Not a global variable");
5077	QualType ASTTy = D->getType();
5078	if (!Ty)
5079	Ty = getTypes().ConvertTypeForMem(T: ASTTy);
5080
5081	StringRef MangledName = getMangledName(GD: D);
5082	return GetOrCreateLLVMGlobal(MangledName, Ty, AddrSpace: ASTTy.getAddressSpace(), D,
5083	IsForDefinition);
5084	}
5085
5086	/// CreateRuntimeVariable - Create a new runtime global variable with the
5087	/// specified type and name.
5088	llvm::Constant *
5089	CodeGenModule::CreateRuntimeVariable(llvm::Type *Ty,
5090	StringRef Name) {
5091	LangAS AddrSpace = getContext().getLangOpts().OpenCL ? LangAS::opencl_global
5092	: LangAS::Default;
5093	auto *Ret = GetOrCreateLLVMGlobal(MangledName: Name, Ty, AddrSpace, D: nullptr);
5094	setDSOLocal(cast<llvm::GlobalValue>(Val: Ret->stripPointerCasts()));
5095	return Ret;
5096	}
5097
5098	void CodeGenModule::EmitTentativeDefinition(const VarDecl *D) {
5099	assert(!D->getInit() && "Cannot emit definite definitions here!");
5100
5101	StringRef MangledName = getMangledName(GD: D);
5102	llvm::GlobalValue *GV = GetGlobalValue(Name: MangledName);
5103
5104	// We already have a definition, not declaration, with the same mangled name.
5105	// Emitting of declaration is not required (and actually overwrites emitted
5106	// definition).
5107	if (GV && !GV->isDeclaration())
5108	return;
5109
5110	// If we have not seen a reference to this variable yet, place it into the
5111	// deferred declarations table to be emitted if needed later.
5112	if (!MustBeEmitted(D) && !GV) {
5113	DeferredDecls[MangledName] = D;
5114	return;
5115	}
5116
5117	// The tentative definition is the only definition.
5118	EmitGlobalVarDefinition(D);
5119	}
5120
5121	void CodeGenModule::EmitExternalDeclaration(const VarDecl *D) {
5122	EmitExternalVarDeclaration(D);
5123	}
5124
5125	CharUnits CodeGenModule::GetTargetTypeStoreSize(llvm::Type *Ty) const {
5126	return Context.toCharUnitsFromBits(
5127	BitSize: getDataLayout().getTypeStoreSizeInBits(Ty));
5128	}
5129
5130	LangAS CodeGenModule::GetGlobalVarAddressSpace(const VarDecl *D) {
5131	if (LangOpts.OpenCL) {
5132	LangAS AS = D ? D->getType().getAddressSpace() : LangAS::opencl_global;
5133	assert(AS == LangAS::opencl_global ||
5134	AS == LangAS::opencl_global_device ||
5135	AS == LangAS::opencl_global_host ||
5136	AS == LangAS::opencl_constant ||
5137	AS == LangAS::opencl_local ||
5138	AS >= LangAS::FirstTargetAddressSpace);
5139	return AS;
5140	}
5141
5142	if (LangOpts.SYCLIsDevice &&
5143	(!D || D->getType().getAddressSpace() == LangAS::Default))
5144	return LangAS::sycl_global;
5145
5146	if (LangOpts.CUDA && LangOpts.CUDAIsDevice) {
5147	if (D) {
5148	if (D->hasAttr<CUDAConstantAttr>())
5149	return LangAS::cuda_constant;
5150	if (D->hasAttr<CUDASharedAttr>())
5151	return LangAS::cuda_shared;
5152	if (D->hasAttr<CUDADeviceAttr>())
5153	return LangAS::cuda_device;
5154	if (D->getType().isConstQualified())
5155	return LangAS::cuda_constant;
5156	}
5157	return LangAS::cuda_device;
5158	}
5159
5160	if (LangOpts.OpenMP) {
5161	LangAS AS;
5162	if (OpenMPRuntime->hasAllocateAttributeForGlobalVar(VD: D, AS))
5163	return AS;
5164	}
5165	return getTargetCodeGenInfo().getGlobalVarAddressSpace(CGM&: *this, D);
5166	}
5167
5168	LangAS CodeGenModule::GetGlobalConstantAddressSpace() const {
5169	// OpenCL v1.2 s6.5.3: a string literal is in the constant address space.
5170	if (LangOpts.OpenCL)
5171	return LangAS::opencl_constant;
5172	if (LangOpts.SYCLIsDevice)
5173	return LangAS::sycl_global;
5174	if (LangOpts.HIP && LangOpts.CUDAIsDevice && getTriple().isSPIRV())
5175	// For HIPSPV map literals to cuda_device (maps to CrossWorkGroup in SPIR-V)
5176	// instead of default AS (maps to Generic in SPIR-V). Otherwise, we end up
5177	// with OpVariable instructions with Generic storage class which is not
5178	// allowed (SPIR-V V1.6 s3.42.8). Also, mapping literals to SPIR-V
5179	// UniformConstant storage class is not viable as pointers to it may not be
5180	// casted to Generic pointers which are used to model HIP's "flat" pointers.
5181	return LangAS::cuda_device;
5182	if (auto AS = getTarget().getConstantAddressSpace())
5183	return *AS;
5184	return LangAS::Default;
5185	}
5186
5187	// In address space agnostic languages, string literals are in default address
5188	// space in AST. However, certain targets (e.g. amdgcn) request them to be
5189	// emitted in constant address space in LLVM IR. To be consistent with other
5190	// parts of AST, string literal global variables in constant address space
5191	// need to be casted to default address space before being put into address
5192	// map and referenced by other part of CodeGen.
5193	// In OpenCL, string literals are in constant address space in AST, therefore
5194	// they should not be casted to default address space.
5195	static llvm::Constant *
5196	castStringLiteralToDefaultAddressSpace(CodeGenModule &CGM,
5197	llvm::GlobalVariable *GV) {
5198	llvm::Constant *Cast = GV;
5199	if (!CGM.getLangOpts().OpenCL) {
5200	auto AS = CGM.GetGlobalConstantAddressSpace();
5201	if (AS != LangAS::Default)
5202	Cast = CGM.getTargetCodeGenInfo().performAddrSpaceCast(
5203	CGM, V: GV, SrcAddr: AS, DestAddr: LangAS::Default,
5204	DestTy: llvm::PointerType::get(
5205	C&: CGM.getLLVMContext(),
5206	AddressSpace: CGM.getContext().getTargetAddressSpace(AS: LangAS::Default)));
5207	}
5208	return Cast;
5209	}
5210
5211	template<typename SomeDecl>
5212	void CodeGenModule::MaybeHandleStaticInExternC(const SomeDecl *D,
5213	llvm::GlobalValue *GV) {
5214	if (!getLangOpts().CPlusPlus)
5215	return;
5216
5217	// Must have 'used' attribute, or else inline assembly can't rely on
5218	// the name existing.
5219	if (!D->template hasAttr<UsedAttr>())
5220	return;
5221
5222	// Must have internal linkage and an ordinary name.
5223	if (!D->getIdentifier() || D->getFormalLinkage() != Linkage::Internal)
5224	return;
5225
5226	// Must be in an extern "C" context. Entities declared directly within
5227	// a record are not extern "C" even if the record is in such a context.
5228	const SomeDecl *First = D->getFirstDecl();
5229	if (First->getDeclContext()->isRecord() || !First->isInExternCContext())
5230	return;
5231
5232	// OK, this is an internal linkage entity inside an extern "C" linkage
5233	// specification. Make a note of that so we can give it the "expected"
5234	// mangled name if nothing else is using that name.
5235	std::pair<StaticExternCMap::iterator, bool> R =
5236	StaticExternCValues.insert(std::make_pair(D->getIdentifier(), GV));
5237
5238	// If we have multiple internal linkage entities with the same name
5239	// in extern "C" regions, none of them gets that name.
5240	if (!R.second)
5241	R.first->second = nullptr;
5242	}
5243
5244	static bool shouldBeInCOMDAT(CodeGenModule &CGM, const Decl &D) {
5245	if (!CGM.supportsCOMDAT())
5246	return false;
5247
5248	if (D.hasAttr<SelectAnyAttr>())
5249	return true;
5250
5251	GVALinkage Linkage;
5252	if (auto *VD = dyn_cast<VarDecl>(Val: &D))
5253	Linkage = CGM.getContext().GetGVALinkageForVariable(VD);
5254	else
5255	Linkage = CGM.getContext().GetGVALinkageForFunction(FD: cast<FunctionDecl>(Val: &D));
5256
5257	switch (Linkage) {
5258	case GVA_Internal:
5259	case GVA_AvailableExternally:
5260	case GVA_StrongExternal:
5261	return false;
5262	case GVA_DiscardableODR:
5263	case GVA_StrongODR:
5264	return true;
5265	}
5266	llvm_unreachable("No such linkage");
5267	}
5268
5269	bool CodeGenModule::supportsCOMDAT() const {
5270	return getTriple().supportsCOMDAT();
5271	}
5272
5273	void CodeGenModule::maybeSetTrivialComdat(const Decl &D,
5274	llvm::GlobalObject &GO) {
5275	if (!shouldBeInCOMDAT(CGM&: *this, D))
5276	return;
5277	GO.setComdat(TheModule.getOrInsertComdat(Name: GO.getName()));
5278	}
5279
5280	/// Pass IsTentative as true if you want to create a tentative definition.
5281	void CodeGenModule::EmitGlobalVarDefinition(const VarDecl *D,
5282	bool IsTentative) {
5283	// OpenCL global variables of sampler type are translated to function calls,
5284	// therefore no need to be translated.
5285	QualType ASTTy = D->getType();
5286	if (getLangOpts().OpenCL && ASTTy->isSamplerT())
5287	return;
5288
5289	// If this is OpenMP device, check if it is legal to emit this global
5290	// normally.
5291	if (LangOpts.OpenMPIsTargetDevice && OpenMPRuntime &&
5292	OpenMPRuntime->emitTargetGlobalVariable(GD: D))
5293	return;
5294
5295	llvm::TrackingVH<llvm::Constant> Init;
5296	bool NeedsGlobalCtor = false;
5297	// Whether the definition of the variable is available externally.
5298	// If yes, we shouldn't emit the GloablCtor and GlobalDtor for the variable
5299	// since this is the job for its original source.
5300	bool IsDefinitionAvailableExternally =
5301	getContext().GetGVALinkageForVariable(VD: D) == GVA_AvailableExternally;
5302	bool NeedsGlobalDtor =
5303	!IsDefinitionAvailableExternally &&
5304	D->needsDestruction(Ctx: getContext()) == QualType::DK_cxx_destructor;
5305
5306	const VarDecl *InitDecl;
5307	const Expr *InitExpr = D->getAnyInitializer(D&: InitDecl);
5308
5309	std::optional<ConstantEmitter> emitter;
5310
5311	// CUDA E.2.4.1 "__shared__ variables cannot have an initialization
5312	// as part of their declaration." Sema has already checked for
5313	// error cases, so we just need to set Init to UndefValue.
5314	bool IsCUDASharedVar =
5315	getLangOpts().CUDAIsDevice && D->hasAttr<CUDASharedAttr>();
5316	// Shadows of initialized device-side global variables are also left
5317	// undefined.
5318	// Managed Variables should be initialized on both host side and device side.
5319	bool IsCUDAShadowVar =
5320	!getLangOpts().CUDAIsDevice && !D->hasAttr<HIPManagedAttr>() &&
5321	(D->hasAttr<CUDAConstantAttr>() || D->hasAttr<CUDADeviceAttr>() ||
5322	D->hasAttr<CUDASharedAttr>());
5323	bool IsCUDADeviceShadowVar =
5324	getLangOpts().CUDAIsDevice && !D->hasAttr<HIPManagedAttr>() &&
5325	(D->getType()->isCUDADeviceBuiltinSurfaceType() ||
5326	D->getType()->isCUDADeviceBuiltinTextureType());
5327	if (getLangOpts().CUDA &&
5328	(IsCUDASharedVar || IsCUDAShadowVar || IsCUDADeviceShadowVar))
5329	Init = llvm::UndefValue::get(T: getTypes().ConvertTypeForMem(T: ASTTy));
5330	else if (D->hasAttr<LoaderUninitializedAttr>())
5331	Init = llvm::UndefValue::get(T: getTypes().ConvertTypeForMem(T: ASTTy));
5332	else if (!InitExpr) {
5333	// This is a tentative definition; tentative definitions are
5334	// implicitly initialized with { 0 }.
5335	//
5336	// Note that tentative definitions are only emitted at the end of
5337	// a translation unit, so they should never have incomplete
5338	// type. In addition, EmitTentativeDefinition makes sure that we
5339	// never attempt to emit a tentative definition if a real one
5340	// exists. A use may still exists, however, so we still may need
5341	// to do a RAUW.
5342	assert(!ASTTy->isIncompleteType() && "Unexpected incomplete type");
5343	Init = EmitNullConstant(T: D->getType());
5344	} else {
5345	initializedGlobalDecl = GlobalDecl(D);
5346	emitter.emplace(args&: *this);
5347	llvm::Constant *Initializer = emitter->tryEmitForInitializer(D: *InitDecl);
5348	if (!Initializer) {
5349	QualType T = InitExpr->getType();
5350	if (D->getType()->isReferenceType())
5351	T = D->getType();
5352
5353	if (getLangOpts().CPlusPlus) {
5354	if (InitDecl->hasFlexibleArrayInit(Ctx: getContext()))
5355	ErrorUnsupported(D, "flexible array initializer");
5356	Init = EmitNullConstant(T);
5357
5358	if (!IsDefinitionAvailableExternally)
5359	NeedsGlobalCtor = true;
5360	} else {
5361	ErrorUnsupported(D, "static initializer");
5362	Init = llvm::UndefValue::get(T: getTypes().ConvertType(T));
5363	}
5364	} else {
5365	Init = Initializer;
5366	// We don't need an initializer, so remove the entry for the delayed
5367	// initializer position (just in case this entry was delayed) if we
5368	// also don't need to register a destructor.
5369	if (getLangOpts().CPlusPlus && !NeedsGlobalDtor)
5370	DelayedCXXInitPosition.erase(D);
5371
5372	#ifndef NDEBUG
5373	CharUnits VarSize = getContext().getTypeSizeInChars(T: ASTTy) +
5374	InitDecl->getFlexibleArrayInitChars(Ctx: getContext());
5375	CharUnits CstSize = CharUnits::fromQuantity(
5376	Quantity: getDataLayout().getTypeAllocSize(Ty: Init->getType()));
5377	assert(VarSize == CstSize && "Emitted constant has unexpected size");
5378	#endif
5379	}
5380	}
5381
5382	llvm::Type* InitType = Init->getType();
5383	llvm::Constant *Entry =
5384	GetAddrOfGlobalVar(D, Ty: InitType, IsForDefinition: ForDefinition_t(!IsTentative));
5385
5386	// Strip off pointer casts if we got them.
5387	Entry = Entry->stripPointerCasts();
5388
5389	// Entry is now either a Function or GlobalVariable.
5390	auto *GV = dyn_cast<llvm::GlobalVariable>(Val: Entry);
5391
5392	// We have a definition after a declaration with the wrong type.
5393	// We must make a new GlobalVariable* and update everything that used OldGV
5394	// (a declaration or tentative definition) with the new GlobalVariable*
5395	// (which will be a definition).
5396	//
5397	// This happens if there is a prototype for a global (e.g.
5398	// "extern int x[];") and then a definition of a different type (e.g.
5399	// "int x[10];"). This also happens when an initializer has a different type
5400	// from the type of the global (this happens with unions).
5401	if (!GV || GV->getValueType() != InitType ||
5402	GV->getType()->getAddressSpace() !=
5403	getContext().getTargetAddressSpace(AS: GetGlobalVarAddressSpace(D))) {
5404
5405	// Move the old entry aside so that we'll create a new one.
5406	Entry->setName(StringRef());
5407
5408	// Make a new global with the correct type, this is now guaranteed to work.
5409	GV = cast<llvm::GlobalVariable>(
5410	Val: GetAddrOfGlobalVar(D, Ty: InitType, IsForDefinition: ForDefinition_t(!IsTentative))
5411	->stripPointerCasts());
5412
5413	// Replace all uses of the old global with the new global
5414	llvm::Constant *NewPtrForOldDecl =
5415	llvm::ConstantExpr::getPointerBitCastOrAddrSpaceCast(C: GV,
5416	Ty: Entry->getType());
5417	Entry->replaceAllUsesWith(V: NewPtrForOldDecl);
5418
5419	// Erase the old global, since it is no longer used.
5420	cast<llvm::GlobalValue>(Val: Entry)->eraseFromParent();
5421	}
5422
5423	MaybeHandleStaticInExternC(D, GV);
5424
5425	if (D->hasAttr<AnnotateAttr>())
5426	AddGlobalAnnotations(D, GV);
5427
5428	// Set the llvm linkage type as appropriate.
5429	llvm::GlobalValue::LinkageTypes Linkage = getLLVMLinkageVarDefinition(VD: D);
5430
5431	// CUDA B.2.1 "The __device__ qualifier declares a variable that resides on
5432	// the device. [...]"
5433	// CUDA B.2.2 "The __constant__ qualifier, optionally used together with
5434	// __device__, declares a variable that: [...]
5435	// Is accessible from all the threads within the grid and from the host
5436	// through the runtime library (cudaGetSymbolAddress() / cudaGetSymbolSize()
5437	// / cudaMemcpyToSymbol() / cudaMemcpyFromSymbol())."
5438	if (LangOpts.CUDA) {
5439	if (LangOpts.CUDAIsDevice) {
5440	if (Linkage != llvm::GlobalValue::InternalLinkage &&
5441	(D->hasAttr<CUDADeviceAttr>() || D->hasAttr<CUDAConstantAttr>() ||
5442	D->getType()->isCUDADeviceBuiltinSurfaceType() ||
5443	D->getType()->isCUDADeviceBuiltinTextureType()))
5444	GV->setExternallyInitialized(true);
5445	} else {
5446	getCUDARuntime().internalizeDeviceSideVar(D, Linkage);
5447	}
5448	getCUDARuntime().handleVarRegistration(VD: D, Var&: *GV);
5449	}
5450
5451	GV->setInitializer(Init);
5452	if (emitter)
5453	emitter->finalize(global: GV);
5454
5455	// If it is safe to mark the global 'constant', do so now.
5456	GV->setConstant(!NeedsGlobalCtor && !NeedsGlobalDtor &&
5457	D->getType().isConstantStorage(getContext(), true, true));
5458
5459	// If it is in a read-only section, mark it 'constant'.
5460	if (const SectionAttr *SA = D->getAttr<SectionAttr>()) {
5461	const ASTContext::SectionInfo &SI = Context.SectionInfos[SA->getName()];
5462	if ((SI.SectionFlags & ASTContext::PSF_Write) == 0)
5463	GV->setConstant(true);
5464	}
5465
5466	CharUnits AlignVal = getContext().getDeclAlign(D);
5467	// Check for alignment specifed in an 'omp allocate' directive.
5468	if (std::optional<CharUnits> AlignValFromAllocate =
5469	getOMPAllocateAlignment(VD: D))
5470	AlignVal = *AlignValFromAllocate;
5471	GV->setAlignment(AlignVal.getAsAlign());
5472
5473	// On Darwin, unlike other Itanium C++ ABI platforms, the thread-wrapper
5474	// function is only defined alongside the variable, not also alongside
5475	// callers. Normally, all accesses to a thread_local go through the
5476	// thread-wrapper in order to ensure initialization has occurred, underlying
5477	// variable will never be used other than the thread-wrapper, so it can be
5478	// converted to internal linkage.
5479	//
5480	// However, if the variable has the 'constinit' attribute, it _can_ be
5481	// referenced directly, without calling the thread-wrapper, so the linkage
5482	// must not be changed.
5483	//
5484	// Additionally, if the variable isn't plain external linkage, e.g. if it's
5485	// weak or linkonce, the de-duplication semantics are important to preserve,
5486	// so we don't change the linkage.
5487	if (D->getTLSKind() == VarDecl::TLS_Dynamic &&
5488	Linkage == llvm::GlobalValue::ExternalLinkage &&
5489	Context.getTargetInfo().getTriple().isOSDarwin() &&
5490	!D->hasAttr<ConstInitAttr>())
5491	Linkage = llvm::GlobalValue::InternalLinkage;
5492
5493	GV->setLinkage(Linkage);
5494	if (D->hasAttr<DLLImportAttr>())
5495	GV->setDLLStorageClass(llvm::GlobalVariable::DLLImportStorageClass);
5496	else if (D->hasAttr<DLLExportAttr>())
5497	GV->setDLLStorageClass(llvm::GlobalVariable::DLLExportStorageClass);
5498	else
5499	GV->setDLLStorageClass(llvm::GlobalVariable::DefaultStorageClass);
5500
5501	if (Linkage == llvm::GlobalVariable::CommonLinkage) {
5502	// common vars aren't constant even if declared const.
5503	GV->setConstant(false);
5504	// Tentative definition of global variables may be initialized with
5505	// non-zero null pointers. In this case they should have weak linkage
5506	// since common linkage must have zero initializer and must not have
5507	// explicit section therefore cannot have non-zero initial value.
5508	if (!GV->getInitializer()->isNullValue())
5509	GV->setLinkage(llvm::GlobalVariable::WeakAnyLinkage);
5510	}
5511
5512	setNonAliasAttributes(GD: D, GO: GV);
5513
5514	if (D->getTLSKind() && !GV->isThreadLocal()) {
5515	if (D->getTLSKind() == VarDecl::TLS_Dynamic)
5516	CXXThreadLocals.push_back(x: D);
5517	setTLSMode(GV, D: *D);
5518	}
5519
5520	maybeSetTrivialComdat(*D, *GV);
5521
5522	// Emit the initializer function if necessary.
5523	if (NeedsGlobalCtor || NeedsGlobalDtor)
5524	EmitCXXGlobalVarDeclInitFunc(D, Addr: GV, PerformInit: NeedsGlobalCtor);
5525
5526	SanitizerMD->reportGlobal(GV, D: *D, IsDynInit: NeedsGlobalCtor);
5527
5528	// Emit global variable debug information.
5529	if (CGDebugInfo *DI = getModuleDebugInfo())
5530	if (getCodeGenOpts().hasReducedDebugInfo())
5531	DI->EmitGlobalVariable(GV, Decl: D);
5532	}
5533
5534	void CodeGenModule::EmitExternalVarDeclaration(const VarDecl *D) {
5535	if (CGDebugInfo *DI = getModuleDebugInfo())
5536	if (getCodeGenOpts().hasReducedDebugInfo()) {
5537	QualType ASTTy = D->getType();
5538	llvm::Type *Ty = getTypes().ConvertTypeForMem(T: D->getType());
5539	llvm::Constant *GV =
5540	GetOrCreateLLVMGlobal(MangledName: D->getName(), Ty, AddrSpace: ASTTy.getAddressSpace(), D);
5541	DI->EmitExternalVariable(
5542	GV: cast<llvm::GlobalVariable>(Val: GV->stripPointerCasts()), Decl: D);
5543	}
5544	}
5545
5546	static bool isVarDeclStrongDefinition(const ASTContext &Context,
5547	CodeGenModule &CGM, const VarDecl *D,
5548	bool NoCommon) {
5549	// Don't give variables common linkage if -fno-common was specified unless it
5550	// was overridden by a NoCommon attribute.
5551	if ((NoCommon || D->hasAttr<NoCommonAttr>()) && !D->hasAttr<CommonAttr>())
5552	return true;
5553
5554	// C11 6.9.2/2:
5555	// A declaration of an identifier for an object that has file scope without
5556	// an initializer, and without a storage-class specifier or with the
5557	// storage-class specifier static, constitutes a tentative definition.
5558	if (D->getInit() || D->hasExternalStorage())
5559	return true;
5560
5561	// A variable cannot be both common and exist in a section.
5562	if (D->hasAttr<SectionAttr>())
5563	return true;
5564
5565	// A variable cannot be both common and exist in a section.
5566	// We don't try to determine which is the right section in the front-end.
5567	// If no specialized section name is applicable, it will resort to default.
5568	if (D->hasAttr<PragmaClangBSSSectionAttr>() ||
5569	D->hasAttr<PragmaClangDataSectionAttr>() ||
5570	D->hasAttr<PragmaClangRelroSectionAttr>() ||
5571	D->hasAttr<PragmaClangRodataSectionAttr>())
5572	return true;
5573
5574	// Thread local vars aren't considered common linkage.
5575	if (D->getTLSKind())
5576	return true;
5577
5578	// Tentative definitions marked with WeakImportAttr are true definitions.
5579	if (D->hasAttr<WeakImportAttr>())
5580	return true;
5581
5582	// A variable cannot be both common and exist in a comdat.
5583	if (shouldBeInCOMDAT(CGM, *D))
5584	return true;
5585
5586	// Declarations with a required alignment do not have common linkage in MSVC
5587	// mode.
5588	if (Context.getTargetInfo().getCXXABI().isMicrosoft()) {
5589	if (D->hasAttr<AlignedAttr>())
5590	return true;
5591	QualType VarType = D->getType();
5592	if (Context.isAlignmentRequired(T: VarType))
5593	return true;
5594
5595	if (const auto *RT = VarType->getAs<RecordType>()) {
5596	const RecordDecl *RD = RT->getDecl();
5597	for (const FieldDecl *FD : RD->fields()) {
5598	if (FD->isBitField())
5599	continue;
5600	if (FD->hasAttr<AlignedAttr>())
5601	return true;
5602	if (Context.isAlignmentRequired(FD->getType()))
5603	return true;
5604	}
5605	}
5606	}
5607
5608	// Microsoft's link.exe doesn't support alignments greater than 32 bytes for
5609	// common symbols, so symbols with greater alignment requirements cannot be
5610	// common.
5611	// Other COFF linkers (ld.bfd and LLD) support arbitrary power-of-two
5612	// alignments for common symbols via the aligncomm directive, so this
5613	// restriction only applies to MSVC environments.
5614	if (Context.getTargetInfo().getTriple().isKnownWindowsMSVCEnvironment() &&
5615	Context.getTypeAlignIfKnown(T: D->getType()) >
5616	Context.toBits(CharSize: CharUnits::fromQuantity(Quantity: 32)))
5617	return true;
5618
5619	return false;
5620	}
5621
5622	llvm::GlobalValue::LinkageTypes
5623	CodeGenModule::getLLVMLinkageForDeclarator(const DeclaratorDecl *D,
5624	GVALinkage Linkage) {
5625	if (Linkage == GVA_Internal)
5626	return llvm::Function::InternalLinkage;
5627
5628	if (D->hasAttr<WeakAttr>())
5629	return llvm::GlobalVariable::WeakAnyLinkage;
5630
5631	if (const auto *FD = D->getAsFunction())
5632	if (FD->isMultiVersion() && Linkage == GVA_AvailableExternally)
5633	return llvm::GlobalVariable::LinkOnceAnyLinkage;
5634
5635	// We are guaranteed to have a strong definition somewhere else,
5636	// so we can use available_externally linkage.
5637	if (Linkage == GVA_AvailableExternally)
5638	return llvm::GlobalValue::AvailableExternallyLinkage;
5639
5640	// Note that Apple's kernel linker doesn't support symbol
5641	// coalescing, so we need to avoid linkonce and weak linkages there.
5642	// Normally, this means we just map to internal, but for explicit
5643	// instantiations we'll map to external.
5644
5645	// In C++, the compiler has to emit a definition in every translation unit
5646	// that references the function. We should use linkonce_odr because
5647	// a) if all references in this translation unit are optimized away, we
5648	// don't need to codegen it. b) if the function persists, it needs to be
5649	// merged with other definitions. c) C++ has the ODR, so we know the
5650	// definition is dependable.
5651	if (Linkage == GVA_DiscardableODR)
5652	return !Context.getLangOpts().AppleKext ? llvm::Function::LinkOnceODRLinkage
5653	: llvm::Function::InternalLinkage;
5654
5655	// An explicit instantiation of a template has weak linkage, since
5656	// explicit instantiations can occur in multiple translation units
5657	// and must all be equivalent. However, we are not allowed to
5658	// throw away these explicit instantiations.
5659	//
5660	// CUDA/HIP: For -fno-gpu-rdc case, device code is limited to one TU,
5661	// so say that CUDA templates are either external (for kernels) or internal.
5662	// This lets llvm perform aggressive inter-procedural optimizations. For
5663	// -fgpu-rdc case, device function calls across multiple TU's are allowed,
5664	// therefore we need to follow the normal linkage paradigm.
5665	if (Linkage == GVA_StrongODR) {
5666	if (getLangOpts().AppleKext)
5667	return llvm::Function::ExternalLinkage;
5668	if (getLangOpts().CUDA && getLangOpts().CUDAIsDevice &&
5669	!getLangOpts().GPURelocatableDeviceCode)
5670	return D->hasAttr<CUDAGlobalAttr>() ? llvm::Function::ExternalLinkage
5671	: llvm::Function::InternalLinkage;
5672	return llvm::Function::WeakODRLinkage;
5673	}
5674
5675	// C++ doesn't have tentative definitions and thus cannot have common
5676	// linkage.
5677	if (!getLangOpts().CPlusPlus && isa<VarDecl>(Val: D) &&
5678	!isVarDeclStrongDefinition(Context, CGM&: *this, D: cast<VarDecl>(Val: D),
5679	NoCommon: CodeGenOpts.NoCommon))
5680	return llvm::GlobalVariable::CommonLinkage;
5681
5682	// selectany symbols are externally visible, so use weak instead of
5683	// linkonce. MSVC optimizes away references to const selectany globals, so
5684	// all definitions should be the same and ODR linkage should be used.
5685	// http://msdn.microsoft.com/en-us/library/5tkz6s71.aspx
5686	if (D->hasAttr<SelectAnyAttr>())
5687	return llvm::GlobalVariable::WeakODRLinkage;
5688
5689	// Otherwise, we have strong external linkage.
5690	assert(Linkage == GVA_StrongExternal);
5691	return llvm::GlobalVariable::ExternalLinkage;
5692	}
5693
5694	llvm::GlobalValue::LinkageTypes
5695	CodeGenModule::getLLVMLinkageVarDefinition(const VarDecl *VD) {
5696	GVALinkage Linkage = getContext().GetGVALinkageForVariable(VD);
5697	return getLLVMLinkageForDeclarator(VD, Linkage);
5698	}
5699
5700	/// Replace the uses of a function that was declared with a non-proto type.
5701	/// We want to silently drop extra arguments from call sites
5702	static void replaceUsesOfNonProtoConstant(llvm::Constant *old,
5703	llvm::Function *newFn) {
5704	// Fast path.
5705	if (old->use_empty()) return;
5706
5707	llvm::Type *newRetTy = newFn->getReturnType();
5708	SmallVector<llvm::Value*, 4> newArgs;
5709
5710	for (llvm::Value::use_iterator ui = old->use_begin(), ue = old->use_end();
5711	ui != ue; ) {
5712	llvm::Value::use_iterator use = ui++; // Increment before the use is erased.
5713	llvm::User *user = use->getUser();
5714
5715	// Recognize and replace uses of bitcasts. Most calls to
5716	// unprototyped functions will use bitcasts.
5717	if (auto *bitcast = dyn_cast<llvm::ConstantExpr>(Val: user)) {
5718	if (bitcast->getOpcode() == llvm::Instruction::BitCast)
5719	replaceUsesOfNonProtoConstant(old: bitcast, newFn);
5720	continue;
5721	}
5722
5723	// Recognize calls to the function.
5724	llvm::CallBase *callSite = dyn_cast<llvm::CallBase>(Val: user);
5725	if (!callSite) continue;
5726	if (!callSite->isCallee(U: &*use))
5727	continue;
5728
5729	// If the return types don't match exactly, then we can't
5730	// transform this call unless it's dead.
5731	if (callSite->getType() != newRetTy && !callSite->use_empty())
5732	continue;
5733
5734	// Get the call site's attribute list.
5735	SmallVector<llvm::AttributeSet, 8> newArgAttrs;
5736	llvm::AttributeList oldAttrs = callSite->getAttributes();
5737
5738	// If the function was passed too few arguments, don't transform.
5739	unsigned newNumArgs = newFn->arg_size();
5740	if (callSite->arg_size() < newNumArgs)
5741	continue;
5742
5743	// If extra arguments were passed, we silently drop them.
5744	// If any of the types mismatch, we don't transform.
5745	unsigned argNo = 0;
5746	bool dontTransform = false;
5747	for (llvm::Argument &A : newFn->args()) {
5748	if (callSite->getArgOperand(i: argNo)->getType() != A.getType()) {
5749	dontTransform = true;
5750	break;
5751	}
5752
5753	// Add any parameter attributes.
5754	newArgAttrs.push_back(Elt: oldAttrs.getParamAttrs(ArgNo: argNo));
5755	argNo++;
5756	}
5757	if (dontTransform)
5758	continue;
5759
5760	// Okay, we can transform this. Create the new call instruction and copy
5761	// over the required information.
5762	newArgs.append(in_start: callSite->arg_begin(), in_end: callSite->arg_begin() + argNo);
5763
5764	// Copy over any operand bundles.
5765	SmallVector<llvm::OperandBundleDef, 1> newBundles;
5766	callSite->getOperandBundlesAsDefs(Defs&: newBundles);
5767
5768	llvm::CallBase *newCall;
5769	if (isa<llvm::CallInst>(Val: callSite)) {
5770	newCall =
5771	llvm::CallInst::Create(Func: newFn, Args: newArgs, Bundles: newBundles, NameStr: "", InsertBefore: callSite);
5772	} else {
5773	auto *oldInvoke = cast<llvm::InvokeInst>(Val: callSite);
5774	newCall = llvm::InvokeInst::Create(Func: newFn, IfNormal: oldInvoke->getNormalDest(),
5775	IfException: oldInvoke->getUnwindDest(), Args: newArgs,
5776	Bundles: newBundles, NameStr: "", InsertBefore: callSite);
5777	}
5778	newArgs.clear(); // for the next iteration
5779
5780	if (!newCall->getType()->isVoidTy())
5781	newCall->takeName(V: callSite);
5782	newCall->setAttributes(
5783	llvm::AttributeList::get(C&: newFn->getContext(), FnAttrs: oldAttrs.getFnAttrs(),
5784	RetAttrs: oldAttrs.getRetAttrs(), ArgAttrs: newArgAttrs));
5785	newCall->setCallingConv(callSite->getCallingConv());
5786
5787	// Finally, remove the old call, replacing any uses with the new one.
5788	if (!callSite->use_empty())
5789	callSite->replaceAllUsesWith(V: newCall);
5790
5791	// Copy debug location attached to CI.
5792	if (callSite->getDebugLoc())
5793	newCall->setDebugLoc(callSite->getDebugLoc());
5794
5795	callSite->eraseFromParent();
5796	}
5797	}
5798
5799	/// ReplaceUsesOfNonProtoTypeWithRealFunction - This function is called when we
5800	/// implement a function with no prototype, e.g. "int foo() {}". If there are
5801	/// existing call uses of the old function in the module, this adjusts them to
5802	/// call the new function directly.
5803	///
5804	/// This is not just a cleanup: the always_inline pass requires direct calls to
5805	/// functions to be able to inline them. If there is a bitcast in the way, it
5806	/// won't inline them. Instcombine normally deletes these calls, but it isn't
5807	/// run at -O0.
5808	static void ReplaceUsesOfNonProtoTypeWithRealFunction(llvm::GlobalValue *Old,
5809	llvm::Function *NewFn) {
5810	// If we're redefining a global as a function, don't transform it.
5811	if (!isa<llvm::Function>(Val: Old)) return;
5812
5813	replaceUsesOfNonProtoConstant(old: Old, newFn: NewFn);
5814	}
5815
5816	void CodeGenModule::HandleCXXStaticMemberVarInstantiation(VarDecl *VD) {
5817	auto DK = VD->isThisDeclarationADefinition();
5818	if (DK == VarDecl::Definition && VD->hasAttr<DLLImportAttr>())
5819	return;
5820
5821	TemplateSpecializationKind TSK = VD->getTemplateSpecializationKind();
5822	// If we have a definition, this might be a deferred decl. If the
5823	// instantiation is explicit, make sure we emit it at the end.
5824	if (VD->getDefinition() && TSK == TSK_ExplicitInstantiationDefinition)
5825	GetAddrOfGlobalVar(D: VD);
5826
5827	EmitTopLevelDecl(VD);
5828	}
5829
5830	void CodeGenModule::EmitGlobalFunctionDefinition(GlobalDecl GD,
5831	llvm::GlobalValue *GV) {
5832	const auto *D = cast<FunctionDecl>(Val: GD.getDecl());
5833
5834	// Compute the function info and LLVM type.
5835	const CGFunctionInfo &FI = getTypes().arrangeGlobalDeclaration(GD);
5836	llvm::FunctionType *Ty = getTypes().GetFunctionType(Info: FI);
5837
5838	// Get or create the prototype for the function.
5839	if (!GV || (GV->getValueType() != Ty))
5840	GV = cast<llvm::GlobalValue>(Val: GetAddrOfFunction(GD, Ty, /*ForVTable=*/false,
5841	/*DontDefer=*/true,
5842	IsForDefinition: ForDefinition));
5843
5844	// Already emitted.
5845	if (!GV->isDeclaration())
5846	return;
5847
5848	// We need to set linkage and visibility on the function before
5849	// generating code for it because various parts of IR generation
5850	// want to propagate this information down (e.g. to local static
5851	// declarations).
5852	auto *Fn = cast<llvm::Function>(Val: GV);
5853	setFunctionLinkage(GD, F: Fn);
5854
5855	// FIXME: this is redundant with part of setFunctionDefinitionAttributes
5856	setGVProperties(GV: Fn, GD);
5857
5858	MaybeHandleStaticInExternC(D, GV: Fn);
5859
5860	maybeSetTrivialComdat(*D, *Fn);
5861
5862	CodeGenFunction(*this).GenerateCode(GD, Fn, FnInfo: FI);
5863
5864	setNonAliasAttributes(GD, GO: Fn);
5865	SetLLVMFunctionAttributesForDefinition(D, Fn);
5866
5867	if (const ConstructorAttr *CA = D->getAttr<ConstructorAttr>())
5868	AddGlobalCtor(Ctor: Fn, Priority: CA->getPriority());
5869	if (const DestructorAttr *DA = D->getAttr<DestructorAttr>())
5870	AddGlobalDtor(Dtor: Fn, Priority: DA->getPriority(), IsDtorAttrFunc: true);
5871	if (getLangOpts().OpenMP && D->hasAttr<OMPDeclareTargetDeclAttr>())
5872	getOpenMPRuntime().emitDeclareTargetFunction(FD: D, GV);
5873	}
5874
5875	void CodeGenModule::EmitAliasDefinition(GlobalDecl GD) {
5876	const auto *D = cast<ValueDecl>(Val: GD.getDecl());
5877	const AliasAttr *AA = D->getAttr<AliasAttr>();
5878	assert(AA && "Not an alias?");
5879
5880	StringRef MangledName = getMangledName(GD);
5881
5882	if (AA->getAliasee() == MangledName) {
5883	Diags.Report(AA->getLocation(), diag::err_cyclic_alias) << 0;
5884	return;
5885	}
5886
5887	// If there is a definition in the module, then it wins over the alias.
5888	// This is dubious, but allow it to be safe. Just ignore the alias.
5889	llvm::GlobalValue *Entry = GetGlobalValue(Name: MangledName);
5890	if (Entry && !Entry->isDeclaration())
5891	return;
5892
5893	Aliases.push_back(x: GD);
5894
5895	llvm::Type *DeclTy = getTypes().ConvertTypeForMem(T: D->getType());
5896
5897	// Create a reference to the named value. This ensures that it is emitted
5898	// if a deferred decl.
5899	llvm::Constant *Aliasee;
5900	llvm::GlobalValue::LinkageTypes LT;
5901	if (isa<llvm::FunctionType>(Val: DeclTy)) {
5902	Aliasee = GetOrCreateLLVMFunction(MangledName: AA->getAliasee(), Ty: DeclTy, GD,
5903	/*ForVTable=*/false);
5904	LT = getFunctionLinkage(GD);
5905	} else {
5906	Aliasee = GetOrCreateLLVMGlobal(MangledName: AA->getAliasee(), Ty: DeclTy, AddrSpace: LangAS::Default,
5907	/*D=*/nullptr);
5908	if (const auto *VD = dyn_cast<VarDecl>(Val: GD.getDecl()))
5909	LT = getLLVMLinkageVarDefinition(VD);
5910	else
5911	LT = getFunctionLinkage(GD);
5912	}
5913
5914	// Create the new alias itself, but don't set a name yet.
5915	unsigned AS = Aliasee->getType()->getPointerAddressSpace();
5916	auto *GA =
5917	llvm::GlobalAlias::create(Ty: DeclTy, AddressSpace: AS, Linkage: LT, Name: "", Aliasee, Parent: &getModule());
5918
5919	if (Entry) {
5920	if (GA->getAliasee() == Entry) {
5921	Diags.Report(AA->getLocation(), diag::err_cyclic_alias) << 0;
5922	return;
5923	}
5924
5925	assert(Entry->isDeclaration());
5926
5927	// If there is a declaration in the module, then we had an extern followed
5928	// by the alias, as in:
5929	// extern int test6();
5930	// ...
5931	// int test6() __attribute__((alias("test7")));
5932	//
5933	// Remove it and replace uses of it with the alias.
5934	GA->takeName(V: Entry);
5935
5936	Entry->replaceAllUsesWith(V: GA);
5937	Entry->eraseFromParent();
5938	} else {
5939	GA->setName(MangledName);
5940	}
5941
5942	// Set attributes which are particular to an alias; this is a
5943	// specialization of the attributes which may be set on a global
5944	// variable/function.
5945	if (D->hasAttr<WeakAttr>() || D->hasAttr<WeakRefAttr>() ||
5946	D->isWeakImported()) {
5947	GA->setLinkage(llvm::Function::WeakAnyLinkage);
5948	}
5949
5950	if (const auto *VD = dyn_cast<VarDecl>(Val: D))
5951	if (VD->getTLSKind())
5952	setTLSMode(GV: GA, D: *VD);
5953
5954	SetCommonAttributes(GD, GV: GA);
5955
5956	// Emit global alias debug information.
5957	if (isa<VarDecl>(Val: D))
5958	if (CGDebugInfo *DI = getModuleDebugInfo())
5959	DI->EmitGlobalAlias(GV: cast<llvm::GlobalValue>(Val: GA->getAliasee()->stripPointerCasts()), Decl: GD);
5960	}
5961
5962	void CodeGenModule::emitIFuncDefinition(GlobalDecl GD) {
5963	const auto *D = cast<ValueDecl>(Val: GD.getDecl());
5964	const IFuncAttr *IFA = D->getAttr<IFuncAttr>();
5965	assert(IFA && "Not an ifunc?");
5966
5967	StringRef MangledName = getMangledName(GD);
5968
5969	if (IFA->getResolver() == MangledName) {
5970	Diags.Report(IFA->getLocation(), diag::err_cyclic_alias) << 1;
5971	return;
5972	}
5973
5974	// Report an error if some definition overrides ifunc.
5975	llvm::GlobalValue *Entry = GetGlobalValue(Name: MangledName);
5976	if (Entry && !Entry->isDeclaration()) {
5977	GlobalDecl OtherGD;
5978	if (lookupRepresentativeDecl(MangledName, Result&: OtherGD) &&
5979	DiagnosedConflictingDefinitions.insert(V: GD).second) {
5980	Diags.Report(D->getLocation(), diag::err_duplicate_mangled_name)
5981	<< MangledName;
5982	Diags.Report(OtherGD.getDecl()->getLocation(),
5983	diag::note_previous_definition);
5984	}
5985	return;
5986	}
5987
5988	Aliases.push_back(x: GD);
5989
5990	llvm::Type *DeclTy = getTypes().ConvertTypeForMem(T: D->getType());
5991	llvm::Type *ResolverTy = llvm::GlobalIFunc::getResolverFunctionType(IFuncValTy: DeclTy);
5992	llvm::Constant *Resolver =
5993	GetOrCreateLLVMFunction(MangledName: IFA->getResolver(), Ty: ResolverTy, GD: {},
5994	/*ForVTable=*/false);
5995	llvm::GlobalIFunc *GIF =
5996	llvm::GlobalIFunc::create(Ty: DeclTy, AddressSpace: 0, Linkage: llvm::Function::ExternalLinkage,
5997	Name: "", Resolver, Parent: &getModule());
5998	if (Entry) {
5999	if (GIF->getResolver() == Entry) {
6000	Diags.Report(IFA->getLocation(), diag::err_cyclic_alias) << 1;
6001	return;
6002	}
6003	assert(Entry->isDeclaration());
6004
6005	// If there is a declaration in the module, then we had an extern followed
6006	// by the ifunc, as in:
6007	// extern int test();
6008	// ...
6009	// int test() __attribute__((ifunc("resolver")));
6010	//
6011	// Remove it and replace uses of it with the ifunc.
6012	GIF->takeName(V: Entry);
6013
6014	Entry->replaceAllUsesWith(V: GIF);
6015	Entry->eraseFromParent();
6016	} else
6017	GIF->setName(MangledName);
6018	if (auto *F = dyn_cast<llvm::Function>(Resolver)) {
6019	F->addFnAttr(llvm::Attribute::DisableSanitizerInstrumentation);
6020	}
6021	SetCommonAttributes(GD, GV: GIF);
6022	}
6023
6024	llvm::Function *CodeGenModule::getIntrinsic(unsigned IID,
6025	ArrayRef<llvm::Type*> Tys) {
6026	return llvm::Intrinsic::getDeclaration(M: &getModule(), id: (llvm::Intrinsic::ID)IID,
6027	Tys);
6028	}
6029
6030	static llvm::StringMapEntry<llvm::GlobalVariable *> &
6031	GetConstantCFStringEntry(llvm::StringMap<llvm::GlobalVariable *> &Map,
6032	const StringLiteral *Literal, bool TargetIsLSB,
6033	bool &IsUTF16, unsigned &StringLength) {
6034	StringRef String = Literal->getString();
6035	unsigned NumBytes = String.size();
6036
6037	// Check for simple case.
6038	if (!Literal->containsNonAsciiOrNull()) {
6039	StringLength = NumBytes;
6040	return *Map.insert(KV: std::make_pair(x&: String, y: nullptr)).first;
6041	}
6042
6043	// Otherwise, convert the UTF8 literals into a string of shorts.
6044	IsUTF16 = true;
6045
6046	SmallVector<llvm::UTF16, 128> ToBuf(NumBytes + 1); // +1 for ending nulls.
6047	const llvm::UTF8 *FromPtr = (const llvm::UTF8 *)String.data();
6048	llvm::UTF16 *ToPtr = &ToBuf[0];
6049
6050	(void)llvm::ConvertUTF8toUTF16(sourceStart: &FromPtr, sourceEnd: FromPtr + NumBytes, targetStart: &ToPtr,
6051	targetEnd: ToPtr + NumBytes, flags: llvm::strictConversion);
6052
6053	// ConvertUTF8toUTF16 returns the length in ToPtr.
6054	StringLength = ToPtr - &ToBuf[0];
6055
6056	// Add an explicit null.
6057	*ToPtr = 0;
6058	return *Map.insert(KV: std::make_pair(
6059	x: StringRef(reinterpret_cast<const char *>(ToBuf.data()),
6060	(StringLength + 1) * 2),
6061	y: nullptr)).first;
6062	}
6063
6064	ConstantAddress
6065	CodeGenModule::GetAddrOfConstantCFString(const StringLiteral *Literal) {
6066	unsigned StringLength = 0;
6067	bool isUTF16 = false;
6068	llvm::StringMapEntry<llvm::GlobalVariable *> &Entry =
6069	GetConstantCFStringEntry(Map&: CFConstantStringMap, Literal,
6070	TargetIsLSB: getDataLayout().isLittleEndian(), IsUTF16&: isUTF16,
6071	StringLength);
6072
6073	if (auto *C = Entry.second)
6074	return ConstantAddress(
6075	C, C->getValueType(), CharUnits::fromQuantity(Quantity: C->getAlignment()));
6076
6077	llvm::Constant *Zero = llvm::Constant::getNullValue(Ty: Int32Ty);
6078	llvm::Constant *Zeros[] = { Zero, Zero };
6079
6080	const ASTContext &Context = getContext();
6081	const llvm::Triple &Triple = getTriple();
6082
6083	const auto CFRuntime = getLangOpts().CFRuntime;
6084	const bool IsSwiftABI =
6085	static_cast<unsigned>(CFRuntime) >=
6086	static_cast<unsigned>(LangOptions::CoreFoundationABI::Swift);
6087	const bool IsSwift4_1 = CFRuntime == LangOptions::CoreFoundationABI::Swift4_1;
6088
6089	// If we don't already have it, get __CFConstantStringClassReference.
6090	if (!CFConstantStringClassRef) {
6091	const char *CFConstantStringClassName = "__CFConstantStringClassReference";
6092	llvm::Type *Ty = getTypes().ConvertType(T: getContext().IntTy);
6093	Ty = llvm::ArrayType::get(ElementType: Ty, NumElements: 0);
6094
6095	switch (CFRuntime) {
6096	default: break;
6097	case LangOptions::CoreFoundationABI::Swift: [[fallthrough]];
6098	case LangOptions::CoreFoundationABI::Swift5_0:
6099	CFConstantStringClassName =
6100	Triple.isOSDarwin() ? "$s15SwiftFoundation19_NSCFConstantStringCN"
6101	: "$s10Foundation19_NSCFConstantStringCN";
6102	Ty = IntPtrTy;
6103	break;
6104	case LangOptions::CoreFoundationABI::Swift4_2:
6105	CFConstantStringClassName =
6106	Triple.isOSDarwin() ? "$S15SwiftFoundation19_NSCFConstantStringCN"
6107	: "$S10Foundation19_NSCFConstantStringCN";
6108	Ty = IntPtrTy;
6109	break;
6110	case LangOptions::CoreFoundationABI::Swift4_1:
6111	CFConstantStringClassName =
6112	Triple.isOSDarwin() ? "__T015SwiftFoundation19_NSCFConstantStringCN"
6113	: "__T010Foundation19_NSCFConstantStringCN";
6114	Ty = IntPtrTy;
6115	break;
6116	}
6117
6118	llvm::Constant *C = CreateRuntimeVariable(Ty, Name: CFConstantStringClassName);
6119
6120	if (Triple.isOSBinFormatELF() || Triple.isOSBinFormatCOFF()) {
6121	llvm::GlobalValue *GV = nullptr;
6122
6123	if ((GV = dyn_cast<llvm::GlobalValue>(Val: C))) {
6124	IdentifierInfo &II = Context.Idents.get(Name: GV->getName());
6125	TranslationUnitDecl *TUDecl = Context.getTranslationUnitDecl();
6126	DeclContext *DC = TranslationUnitDecl::castToDeclContext(D: TUDecl);
6127
6128	const VarDecl *VD = nullptr;
6129	for (const auto *Result : DC->lookup(Name: &II))
6130	if ((VD = dyn_cast<VarDecl>(Val: Result)))
6131	break;
6132
6133	if (Triple.isOSBinFormatELF()) {
6134	if (!VD)
6135	GV->setLinkage(llvm::GlobalValue::ExternalLinkage);
6136	} else {
6137	GV->setLinkage(llvm::GlobalValue::ExternalLinkage);
6138	if (!VD || !VD->hasAttr<DLLExportAttr>())
6139	GV->setDLLStorageClass(llvm::GlobalValue::DLLImportStorageClass);
6140	else
6141	GV->setDLLStorageClass(llvm::GlobalValue::DLLExportStorageClass);
6142	}
6143
6144	setDSOLocal(GV);
6145	}
6146	}
6147
6148	// Decay array -> ptr
6149	CFConstantStringClassRef =
6150	IsSwiftABI ? llvm::ConstantExpr::getPtrToInt(C, Ty)
6151	: llvm::ConstantExpr::getGetElementPtr(Ty, C, IdxList: Zeros);
6152	}
6153
6154	QualType CFTy = Context.getCFConstantStringType();
6155
6156	auto *STy = cast<llvm::StructType>(Val: getTypes().ConvertType(T: CFTy));
6157
6158	ConstantInitBuilder Builder(*this);
6159	auto Fields = Builder.beginStruct(structTy: STy);
6160
6161	// Class pointer.
6162	Fields.add(value: cast<llvm::Constant>(Val&: CFConstantStringClassRef));
6163
6164	// Flags.
6165	if (IsSwiftABI) {
6166	Fields.addInt(intTy: IntPtrTy, value: IsSwift4_1 ? 0x05 : 0x01);
6167	Fields.addInt(intTy: Int64Ty, value: isUTF16 ? 0x07d0 : 0x07c8);
6168	} else {
6169	Fields.addInt(intTy: IntTy, value: isUTF16 ? 0x07d0 : 0x07C8);
6170	}
6171
6172	// String pointer.
6173	llvm::Constant *C = nullptr;
6174	if (isUTF16) {
6175	auto Arr = llvm::ArrayRef(
6176	reinterpret_cast<uint16_t *>(const_cast<char *>(Entry.first().data())),
6177	Entry.first().size() / 2);
6178	C = llvm::ConstantDataArray::get(Context&: VMContext, Elts: Arr);
6179	} else {
6180	C = llvm::ConstantDataArray::getString(Context&: VMContext, Initializer: Entry.first());
6181	}
6182
6183	// Note: -fwritable-strings doesn't make the backing store strings of
6184	// CFStrings writable.
6185	auto *GV =
6186	new llvm::GlobalVariable(getModule(), C->getType(), /*isConstant=*/true,
6187	llvm::GlobalValue::PrivateLinkage, C, ".str");
6188	GV->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
6189	// Don't enforce the target's minimum global alignment, since the only use
6190	// of the string is via this class initializer.
6191	CharUnits Align = isUTF16 ? Context.getTypeAlignInChars(Context.ShortTy)
6192	: Context.getTypeAlignInChars(Context.CharTy);
6193	GV->setAlignment(Align.getAsAlign());
6194
6195	// FIXME: We set the section explicitly to avoid a bug in ld64 224.1.
6196	// Without it LLVM can merge the string with a non unnamed_addr one during
6197	// LTO. Doing that changes the section it ends in, which surprises ld64.
6198	if (Triple.isOSBinFormatMachO())
6199	GV->setSection(isUTF16 ? "__TEXT,__ustring"
6200	: "__TEXT,__cstring,cstring_literals");
6201	// Make sure the literal ends up in .rodata to allow for safe ICF and for
6202	// the static linker to adjust permissions to read-only later on.
6203	else if (Triple.isOSBinFormatELF())
6204	GV->setSection(".rodata");
6205
6206	// String.
6207	llvm::Constant *Str =
6208	llvm::ConstantExpr::getGetElementPtr(Ty: GV->getValueType(), C: GV, IdxList: Zeros);
6209
6210	Fields.add(value: Str);
6211
6212	// String length.
6213	llvm::IntegerType *LengthTy =
6214	llvm::IntegerType::get(C&: getModule().getContext(),
6215	NumBits: Context.getTargetInfo().getLongWidth());
6216	if (IsSwiftABI) {
6217	if (CFRuntime == LangOptions::CoreFoundationABI::Swift4_1 ||
6218	CFRuntime == LangOptions::CoreFoundationABI::Swift4_2)
6219	LengthTy = Int32Ty;
6220	else
6221	LengthTy = IntPtrTy;
6222	}
6223	Fields.addInt(intTy: LengthTy, value: StringLength);
6224
6225	// Swift ABI requires 8-byte alignment to ensure that the _Atomic(uint64_t) is
6226	// properly aligned on 32-bit platforms.
6227	CharUnits Alignment =
6228	IsSwiftABI ? Context.toCharUnitsFromBits(BitSize: 64) : getPointerAlign();
6229
6230	// The struct.
6231	GV = Fields.finishAndCreateGlobal(args: "_unnamed_cfstring_", args&: Alignment,
6232	/*isConstant=*/args: false,
6233	args: llvm::GlobalVariable::PrivateLinkage);
6234	GV->addAttribute(Kind: "objc_arc_inert");
6235	switch (Triple.getObjectFormat()) {
6236	case llvm::Triple::UnknownObjectFormat:
6237	llvm_unreachable("unknown file format");
6238	case llvm::Triple::DXContainer:
6239	case llvm::Triple::GOFF:
6240	case llvm::Triple::SPIRV:
6241	case llvm::Triple::XCOFF:
6242	llvm_unreachable("unimplemented");
6243	case llvm::Triple::COFF:
6244	case llvm::Triple::ELF:
6245	case llvm::Triple::Wasm:
6246	GV->setSection("cfstring");
6247	break;
6248	case llvm::Triple::MachO:
6249	GV->setSection("__DATA,__cfstring");
6250	break;
6251	}
6252	Entry.second = GV;
6253
6254	return ConstantAddress(GV, GV->getValueType(), Alignment);
6255	}
6256
6257	bool CodeGenModule::getExpressionLocationsEnabled() const {
6258	return !CodeGenOpts.EmitCodeView || CodeGenOpts.DebugColumnInfo;
6259	}
6260
6261	QualType CodeGenModule::getObjCFastEnumerationStateType() {
6262	if (ObjCFastEnumerationStateType.isNull()) {
6263	RecordDecl *D = Context.buildImplicitRecord(Name: "__objcFastEnumerationState");
6264	D->startDefinition();
6265
6266	QualType FieldTypes[] = {
6267	Context.UnsignedLongTy, Context.getPointerType(T: Context.getObjCIdType()),
6268	Context.getPointerType(Context.UnsignedLongTy),
6269	Context.getConstantArrayType(EltTy: Context.UnsignedLongTy, ArySize: llvm::APInt(32, 5),
6270	SizeExpr: nullptr, ASM: ArraySizeModifier::Normal, IndexTypeQuals: 0)};
6271
6272	for (size_t i = 0; i < 4; ++i) {
6273	FieldDecl *Field = FieldDecl::Create(C: Context,
6274	DC: D,
6275	StartLoc: SourceLocation(),
6276	IdLoc: SourceLocation(), Id: nullptr,
6277	T: FieldTypes[i], /*TInfo=*/nullptr,
6278	/*BitWidth=*/BW: nullptr,
6279	/*Mutable=*/false,
6280	InitStyle: ICIS_NoInit);
6281	Field->setAccess(AS_public);
6282	D->addDecl(Field);
6283	}
6284
6285	D->completeDefinition();
6286	ObjCFastEnumerationStateType = Context.getTagDeclType(D);
6287	}
6288
6289	return ObjCFastEnumerationStateType;
6290	}
6291
6292	llvm::Constant *
6293	CodeGenModule::GetConstantArrayFromStringLiteral(const StringLiteral *E) {
6294	assert(!E->getType()->isPointerType() && "Strings are always arrays");
6295
6296	// Don't emit it as the address of the string, emit the string data itself
6297	// as an inline array.
6298	if (E->getCharByteWidth() == 1) {
6299	SmallString<64> Str(E->getString());
6300
6301	// Resize the string to the right size, which is indicated by its type.
6302	const ConstantArrayType *CAT = Context.getAsConstantArrayType(T: E->getType());
6303	assert(CAT && "String literal not of constant array type!");
6304	Str.resize(N: CAT->getZExtSize());
6305	return llvm::ConstantDataArray::getString(Context&: VMContext, Initializer: Str, AddNull: false);
6306	}
6307
6308	auto *AType = cast<llvm::ArrayType>(getTypes().ConvertType(T: E->getType()));
6309	llvm::Type *ElemTy = AType->getElementType();
6310	unsigned NumElements = AType->getNumElements();
6311
6312	// Wide strings have either 2-byte or 4-byte elements.
6313	if (ElemTy->getPrimitiveSizeInBits() == 16) {
6314	SmallVector<uint16_t, 32> Elements;
6315	Elements.reserve(N: NumElements);
6316
6317	for(unsigned i = 0, e = E->getLength(); i != e; ++i)
6318	Elements.push_back(Elt: E->getCodeUnit(i));
6319	Elements.resize(N: NumElements);
6320	return llvm::ConstantDataArray::get(Context&: VMContext, Elts&: Elements);
6321	}
6322
6323	assert(ElemTy->getPrimitiveSizeInBits() == 32);
6324	SmallVector<uint32_t, 32> Elements;
6325	Elements.reserve(N: NumElements);
6326
6327	for(unsigned i = 0, e = E->getLength(); i != e; ++i)
6328	Elements.push_back(Elt: E->getCodeUnit(i));
6329	Elements.resize(N: NumElements);
6330	return llvm::ConstantDataArray::get(Context&: VMContext, Elts&: Elements);
6331	}
6332
6333	static llvm::GlobalVariable *
6334	GenerateStringLiteral(llvm::Constant *C, llvm::GlobalValue::LinkageTypes LT,
6335	CodeGenModule &CGM, StringRef GlobalName,
6336	CharUnits Alignment) {
6337	unsigned AddrSpace = CGM.getContext().getTargetAddressSpace(
6338	AS: CGM.GetGlobalConstantAddressSpace());
6339
6340	llvm::Module &M = CGM.getModule();
6341	// Create a global variable for this string
6342	auto *GV = new llvm::GlobalVariable(
6343	M, C->getType(), !CGM.getLangOpts().WritableStrings, LT, C, GlobalName,
6344	nullptr, llvm::GlobalVariable::NotThreadLocal, AddrSpace);
6345	GV->setAlignment(Alignment.getAsAlign());
6346	GV->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
6347	if (GV->isWeakForLinker()) {
6348	assert(CGM.supportsCOMDAT() && "Only COFF uses weak string literals");
6349	GV->setComdat(M.getOrInsertComdat(Name: GV->getName()));
6350	}
6351	CGM.setDSOLocal(GV);
6352
6353	return GV;
6354	}
6355
6356	/// GetAddrOfConstantStringFromLiteral - Return a pointer to a
6357	/// constant array for the given string literal.
6358	ConstantAddress
6359	CodeGenModule::GetAddrOfConstantStringFromLiteral(const StringLiteral *S,
6360	StringRef Name) {
6361	CharUnits Alignment =
6362	getContext().getAlignOfGlobalVarInChars(T: S->getType(), /*VD=*/nullptr);
6363
6364	llvm::Constant *C = GetConstantArrayFromStringLiteral(E: S);
6365	llvm::GlobalVariable **Entry = nullptr;
6366	if (!LangOpts.WritableStrings) {
6367	Entry = &ConstantStringMap[C];
6368	if (auto GV = *Entry) {
6369	if (uint64_t(Alignment.getQuantity()) > GV->getAlignment())
6370	GV->setAlignment(Alignment.getAsAlign());
6371	return ConstantAddress(castStringLiteralToDefaultAddressSpace(CGM&: *this, GV),
6372	GV->getValueType(), Alignment);
6373	}
6374	}
6375
6376	SmallString<256> MangledNameBuffer;
6377	StringRef GlobalVariableName;
6378	llvm::GlobalValue::LinkageTypes LT;
6379
6380	// Mangle the string literal if that's how the ABI merges duplicate strings.
6381	// Don't do it if they are writable, since we don't want writes in one TU to
6382	// affect strings in another.
6383	if (getCXXABI().getMangleContext().shouldMangleStringLiteral(SL: S) &&
6384	!LangOpts.WritableStrings) {
6385	llvm::raw_svector_ostream Out(MangledNameBuffer);
6386	getCXXABI().getMangleContext().mangleStringLiteral(SL: S, Out);
6387	LT = llvm::GlobalValue::LinkOnceODRLinkage;
6388	GlobalVariableName = MangledNameBuffer;
6389	} else {
6390	LT = llvm::GlobalValue::PrivateLinkage;
6391	GlobalVariableName = Name;
6392	}
6393
6394	auto GV = GenerateStringLiteral(C, LT, CGM&: *this, GlobalName: GlobalVariableName, Alignment);
6395
6396	CGDebugInfo *DI = getModuleDebugInfo();
6397	if (DI && getCodeGenOpts().hasReducedDebugInfo())
6398	DI->AddStringLiteralDebugInfo(GV: GV, S);
6399
6400	if (Entry)
6401	*Entry = GV;
6402
6403	SanitizerMD->reportGlobal(GV, S->getStrTokenLoc(TokNum: 0), "<string literal>");
6404
6405	return ConstantAddress(castStringLiteralToDefaultAddressSpace(*this, GV),
6406	GV->getValueType(), Alignment);
6407	}
6408
6409	/// GetAddrOfConstantStringFromObjCEncode - Return a pointer to a constant
6410	/// array for the given ObjCEncodeExpr node.
6411	ConstantAddress
6412	CodeGenModule::GetAddrOfConstantStringFromObjCEncode(const ObjCEncodeExpr *E) {
6413	std::string Str;
6414	getContext().getObjCEncodingForType(T: E->getEncodedType(), S&: Str);
6415
6416	return GetAddrOfConstantCString(Str);
6417	}
6418
6419	/// GetAddrOfConstantCString - Returns a pointer to a character array containing
6420	/// the literal and a terminating '\0' character.
6421	/// The result has pointer to array type.
6422	ConstantAddress CodeGenModule::GetAddrOfConstantCString(
6423	const std::string &Str, const char *GlobalName) {
6424	StringRef StrWithNull(Str.c_str(), Str.size() + 1);
6425	CharUnits Alignment = getContext().getAlignOfGlobalVarInChars(
6426	T: getContext().CharTy, /*VD=*/nullptr);
6427
6428	llvm::Constant *C =
6429	llvm::ConstantDataArray::getString(Context&: getLLVMContext(), Initializer: StrWithNull, AddNull: false);
6430
6431	// Don't share any string literals if strings aren't constant.
6432	llvm::GlobalVariable **Entry = nullptr;
6433	if (!LangOpts.WritableStrings) {
6434	Entry = &ConstantStringMap[C];
6435	if (auto GV = *Entry) {
6436	if (uint64_t(Alignment.getQuantity()) > GV->getAlignment())
6437	GV->setAlignment(Alignment.getAsAlign());
6438	return ConstantAddress(castStringLiteralToDefaultAddressSpace(CGM&: *this, GV),
6439	GV->getValueType(), Alignment);
6440	}
6441	}
6442
6443	// Get the default prefix if a name wasn't specified.
6444	if (!GlobalName)
6445	GlobalName = ".str";
6446	// Create a global variable for this.
6447	auto GV = GenerateStringLiteral(C, LT: llvm::GlobalValue::PrivateLinkage, CGM&: *this,
6448	GlobalName, Alignment);
6449	if (Entry)
6450	*Entry = GV;
6451
6452	return ConstantAddress(castStringLiteralToDefaultAddressSpace(*this, GV),
6453	GV->getValueType(), Alignment);
6454	}
6455
6456	ConstantAddress CodeGenModule::GetAddrOfGlobalTemporary(
6457	const MaterializeTemporaryExpr *E, const Expr *Init) {
6458	assert((E->getStorageDuration() == SD_Static ||
6459	E->getStorageDuration() == SD_Thread) && "not a global temporary");
6460	const auto *VD = cast<VarDecl>(Val: E->getExtendingDecl());
6461
6462	// If we're not materializing a subobject of the temporary, keep the
6463	// cv-qualifiers from the type of the MaterializeTemporaryExpr.
6464	QualType MaterializedType = Init->getType();
6465	if (Init == E->getSubExpr())
6466	MaterializedType = E->getType();
6467
6468	CharUnits Align = getContext().getTypeAlignInChars(T: MaterializedType);
6469
6470	auto InsertResult = MaterializedGlobalTemporaryMap.insert({E, nullptr});
6471	if (!InsertResult.second) {
6472	// We've seen this before: either we already created it or we're in the
6473	// process of doing so.
6474	if (!InsertResult.first->second) {
6475	// We recursively re-entered this function, probably during emission of
6476	// the initializer. Create a placeholder. We'll clean this up in the
6477	// outer call, at the end of this function.
6478	llvm::Type *Type = getTypes().ConvertTypeForMem(T: MaterializedType);
6479	InsertResult.first->second = new llvm::GlobalVariable(
6480	getModule(), Type, false, llvm::GlobalVariable::InternalLinkage,
6481	nullptr);
6482	}
6483	return ConstantAddress(InsertResult.first->second,
6484	llvm::cast<llvm::GlobalVariable>(
6485	InsertResult.first->second->stripPointerCasts())
6486	->getValueType(),
6487	Align);
6488	}
6489
6490	// FIXME: If an externally-visible declaration extends multiple temporaries,
6491	// we need to give each temporary the same name in every translation unit (and
6492	// we also need to make the temporaries externally-visible).
6493	SmallString<256> Name;
6494	llvm::raw_svector_ostream Out(Name);
6495	getCXXABI().getMangleContext().mangleReferenceTemporary(
6496	D: VD, ManglingNumber: E->getManglingNumber(), Out);
6497
6498	APValue *Value = nullptr;
6499	if (E->getStorageDuration() == SD_Static && VD->evaluateValue()) {
6500	// If the initializer of the extending declaration is a constant
6501	// initializer, we should have a cached constant initializer for this
6502	// temporary. Note that this might have a different value from the value
6503	// computed by evaluating the initializer if the surrounding constant
6504	// expression modifies the temporary.
6505	Value = E->getOrCreateValue(MayCreate: false);
6506	}
6507
6508	// Try evaluating it now, it might have a constant initializer.
6509	Expr::EvalResult EvalResult;
6510	if (!Value && Init->EvaluateAsRValue(Result&: EvalResult, Ctx: getContext()) &&
6511	!EvalResult.hasSideEffects())
6512	Value = &EvalResult.Val;
6513
6514	LangAS AddrSpace = GetGlobalVarAddressSpace(D: VD);
6515
6516	std::optional<ConstantEmitter> emitter;
6517	llvm::Constant *InitialValue = nullptr;
6518	bool Constant = false;
6519	llvm::Type *Type;
6520	if (Value) {
6521	// The temporary has a constant initializer, use it.
6522	emitter.emplace(args&: *this);
6523	InitialValue = emitter->emitForInitializer(value: *Value, destAddrSpace: AddrSpace,
6524	destType: MaterializedType);
6525	Constant =
6526	MaterializedType.isConstantStorage(Ctx: getContext(), /*ExcludeCtor*/ Value,
6527	/*ExcludeDtor*/ false);
6528	Type = InitialValue->getType();
6529	} else {
6530	// No initializer, the initialization will be provided when we
6531	// initialize the declaration which performed lifetime extension.
6532	Type = getTypes().ConvertTypeForMem(T: MaterializedType);
6533	}
6534
6535	// Create a global variable for this lifetime-extended temporary.
6536	llvm::GlobalValue::LinkageTypes Linkage = getLLVMLinkageVarDefinition(VD);
6537	if (Linkage == llvm::GlobalVariable::ExternalLinkage) {
6538	const VarDecl *InitVD;
6539	if (VD->isStaticDataMember() && VD->getAnyInitializer(D&: InitVD) &&
6540	isa<CXXRecordDecl>(InitVD->getLexicalDeclContext())) {
6541	// Temporaries defined inside a class get linkonce_odr linkage because the
6542	// class can be defined in multiple translation units.
6543	Linkage = llvm::GlobalVariable::LinkOnceODRLinkage;
6544	} else {
6545	// There is no need for this temporary to have external linkage if the
6546	// VarDecl has external linkage.
6547	Linkage = llvm::GlobalVariable::InternalLinkage;
6548	}
6549	}
6550	auto TargetAS = getContext().getTargetAddressSpace(AS: AddrSpace);
6551	auto *GV = new llvm::GlobalVariable(
6552	getModule(), Type, Constant, Linkage, InitialValue, Name.c_str(),
6553	/*InsertBefore=*/nullptr, llvm::GlobalVariable::NotThreadLocal, TargetAS);
6554	if (emitter) emitter->finalize(global: GV);
6555	// Don't assign dllimport or dllexport to local linkage globals.
6556	if (!llvm::GlobalValue::isLocalLinkage(Linkage)) {
6557	setGVProperties(GV, GD: VD);
6558	if (GV->getDLLStorageClass() == llvm::GlobalVariable::DLLExportStorageClass)
6559	// The reference temporary should never be dllexport.
6560	GV->setDLLStorageClass(llvm::GlobalVariable::DefaultStorageClass);
6561	}
6562	GV->setAlignment(Align.getAsAlign());
6563	if (supportsCOMDAT() && GV->isWeakForLinker())
6564	GV->setComdat(TheModule.getOrInsertComdat(Name: GV->getName()));
6565	if (VD->getTLSKind())
6566	setTLSMode(GV, D: *VD);
6567	llvm::Constant *CV = GV;
6568	if (AddrSpace != LangAS::Default)
6569	CV = getTargetCodeGenInfo().performAddrSpaceCast(
6570	CGM&: *this, V: GV, SrcAddr: AddrSpace, DestAddr: LangAS::Default,
6571	DestTy: llvm::PointerType::get(
6572	C&: getLLVMContext(),
6573	AddressSpace: getContext().getTargetAddressSpace(AS: LangAS::Default)));
6574
6575	// Update the map with the new temporary. If we created a placeholder above,
6576	// replace it with the new global now.
6577	llvm::Constant *&Entry = MaterializedGlobalTemporaryMap[E];
6578	if (Entry) {
6579	Entry->replaceAllUsesWith(V: CV);
6580	llvm::cast<llvm::GlobalVariable>(Val: Entry)->eraseFromParent();
6581	}
6582	Entry = CV;
6583
6584	return ConstantAddress(CV, Type, Align);
6585	}
6586
6587	/// EmitObjCPropertyImplementations - Emit information for synthesized
6588	/// properties for an implementation.
6589	void CodeGenModule::EmitObjCPropertyImplementations(const
6590	ObjCImplementationDecl *D) {
6591	for (const auto *PID : D->property_impls()) {
6592	// Dynamic is just for type-checking.
6593	if (PID->getPropertyImplementation() == ObjCPropertyImplDecl::Synthesize) {
6594	ObjCPropertyDecl *PD = PID->getPropertyDecl();
6595
6596	// Determine which methods need to be implemented, some may have
6597	// been overridden. Note that ::isPropertyAccessor is not the method
6598	// we want, that just indicates if the decl came from a
6599	// property. What we want to know is if the method is defined in
6600	// this implementation.
6601	auto *Getter = PID->getGetterMethodDecl();
6602	if (!Getter || Getter->isSynthesizedAccessorStub())
6603	CodeGenFunction(*this).GenerateObjCGetter(
6604	const_cast<ObjCImplementationDecl *>(D), PID);
6605	auto *Setter = PID->getSetterMethodDecl();
6606	if (!PD->isReadOnly() && (!Setter || Setter->isSynthesizedAccessorStub()))
6607	CodeGenFunction(*this).GenerateObjCSetter(
6608	const_cast<ObjCImplementationDecl *>(D), PID);
6609	}
6610	}
6611	}
6612
6613	static bool needsDestructMethod(ObjCImplementationDecl *impl) {
6614	const ObjCInterfaceDecl *iface = impl->getClassInterface();
6615	for (const ObjCIvarDecl *ivar = iface->all_declared_ivar_begin();
6616	ivar; ivar = ivar->getNextIvar())
6617	if (ivar->getType().isDestructedType())
6618	return true;
6619
6620	return false;
6621	}
6622
6623	static bool AllTrivialInitializers(CodeGenModule &CGM,
6624	ObjCImplementationDecl *D) {
6625	CodeGenFunction CGF(CGM);
6626	for (ObjCImplementationDecl::init_iterator B = D->init_begin(),
6627	E = D->init_end(); B != E; ++B) {
6628	CXXCtorInitializer *CtorInitExp = *B;
6629	Expr *Init = CtorInitExp->getInit();
6630	if (!CGF.isTrivialInitializer(Init))
6631	return false;
6632	}
6633	return true;
6634	}
6635
6636	/// EmitObjCIvarInitializations - Emit information for ivar initialization
6637	/// for an implementation.
6638	void CodeGenModule::EmitObjCIvarInitializations(ObjCImplementationDecl *D) {
6639	// We might need a .cxx_destruct even if we don't have any ivar initializers.
6640	if (needsDestructMethod(impl: D)) {
6641	const IdentifierInfo *II = &getContext().Idents.get(Name: ".cxx_destruct");
6642	Selector cxxSelector = getContext().Selectors.getSelector(NumArgs: 0, IIV: &II);
6643	ObjCMethodDecl *DTORMethod = ObjCMethodDecl::Create(
6644	C&: getContext(), beginLoc: D->getLocation(), endLoc: D->getLocation(), SelInfo: cxxSelector,
6645	T: getContext().VoidTy, ReturnTInfo: nullptr, contextDecl: D,
6646	/*isInstance=*/true, /*isVariadic=*/false,
6647	/*isPropertyAccessor=*/true, /*isSynthesizedAccessorStub=*/false,
6648	/*isImplicitlyDeclared=*/true,
6649	/*isDefined=*/false, impControl: ObjCImplementationControl::Required);
6650	D->addInstanceMethod(DTORMethod);
6651	CodeGenFunction(*this).GenerateObjCCtorDtorMethod(IMP: D, MD: DTORMethod, ctor: false);
6652	D->setHasDestructors(true);
6653	}
6654
6655	// If the implementation doesn't have any ivar initializers, we don't need
6656	// a .cxx_construct.
6657	if (D->getNumIvarInitializers() == 0 ||
6658	AllTrivialInitializers(CGM&: *this, D))
6659	return;
6660
6661	const IdentifierInfo *II = &getContext().Idents.get(Name: ".cxx_construct");
6662	Selector cxxSelector = getContext().Selectors.getSelector(NumArgs: 0, IIV: &II);
6663	// The constructor returns 'self'.
6664	ObjCMethodDecl *CTORMethod = ObjCMethodDecl::Create(
6665	C&: getContext(), beginLoc: D->getLocation(), endLoc: D->getLocation(), SelInfo: cxxSelector,
6666	T: getContext().getObjCIdType(), ReturnTInfo: nullptr, contextDecl: D, /*isInstance=*/true,
6667	/*isVariadic=*/false,
6668	/*isPropertyAccessor=*/true, /*isSynthesizedAccessorStub=*/false,
6669	/*isImplicitlyDeclared=*/true,
6670	/*isDefined=*/false, impControl: ObjCImplementationControl::Required);
6671	D->addInstanceMethod(CTORMethod);
6672	CodeGenFunction(*this).GenerateObjCCtorDtorMethod(IMP: D, MD: CTORMethod, ctor: true);
6673	D->setHasNonZeroConstructors(true);
6674	}
6675
6676	// EmitLinkageSpec - Emit all declarations in a linkage spec.
6677	void CodeGenModule::EmitLinkageSpec(const LinkageSpecDecl *LSD) {
6678	if (LSD->getLanguage() != LinkageSpecLanguageIDs::C &&
6679	LSD->getLanguage() != LinkageSpecLanguageIDs::CXX) {
6680	ErrorUnsupported(LSD, "linkage spec");
6681	return;
6682	}
6683
6684	EmitDeclContext(LSD);
6685	}
6686
6687	void CodeGenModule::EmitTopLevelStmt(const TopLevelStmtDecl *D) {
6688	// Device code should not be at top level.
6689	if (LangOpts.CUDA && LangOpts.CUDAIsDevice)
6690	return;
6691
6692	std::unique_ptr<CodeGenFunction> &CurCGF =
6693	GlobalTopLevelStmtBlockInFlight.first;
6694
6695	// We emitted a top-level stmt but after it there is initialization.
6696	// Stop squashing the top-level stmts into a single function.
6697	if (CurCGF && CXXGlobalInits.back() != CurCGF->CurFn) {
6698	CurCGF->FinishFunction(EndLoc: D->getEndLoc());
6699	CurCGF = nullptr;
6700	}
6701
6702	if (!CurCGF) {
6703	// void __stmts__N(void)
6704	// FIXME: Ask the ABI name mangler to pick a name.
6705	std::string Name = "__stmts__" + llvm::utostr(X: CXXGlobalInits.size());
6706	FunctionArgList Args;
6707	QualType RetTy = getContext().VoidTy;
6708	const CGFunctionInfo &FnInfo =
6709	getTypes().arrangeBuiltinFunctionDeclaration(resultType: RetTy, args: Args);
6710	llvm::FunctionType *FnTy = getTypes().GetFunctionType(Info: FnInfo);
6711	llvm::Function *Fn = llvm::Function::Create(
6712	Ty: FnTy, Linkage: llvm::GlobalValue::InternalLinkage, N: Name, M: &getModule());
6713
6714	CurCGF.reset(p: new CodeGenFunction(*this));
6715	GlobalTopLevelStmtBlockInFlight.second = D;
6716	CurCGF->StartFunction(GD: GlobalDecl(), RetTy, Fn, FnInfo, Args,
6717	Loc: D->getBeginLoc(), StartLoc: D->getBeginLoc());
6718	CXXGlobalInits.push_back(x: Fn);
6719	}
6720
6721	CurCGF->EmitStmt(S: D->getStmt());
6722	}
6723
6724	void CodeGenModule::EmitDeclContext(const DeclContext *DC) {
6725	for (auto *I : DC->decls()) {
6726	// Unlike other DeclContexts, the contents of an ObjCImplDecl at TU scope
6727	// are themselves considered "top-level", so EmitTopLevelDecl on an
6728	// ObjCImplDecl does not recursively visit them. We need to do that in
6729	// case they're nested inside another construct (LinkageSpecDecl /
6730	// ExportDecl) that does stop them from being considered "top-level".
6731	if (auto *OID = dyn_cast<ObjCImplDecl>(Val: I)) {
6732	for (auto *M : OID->methods())
6733	EmitTopLevelDecl(M);
6734	}
6735
6736	EmitTopLevelDecl(D: I);
6737	}
6738	}
6739
6740	/// EmitTopLevelDecl - Emit code for a single top level declaration.
6741	void CodeGenModule::EmitTopLevelDecl(Decl *D) {
6742	// Ignore dependent declarations.
6743	if (D->isTemplated())
6744	return;
6745
6746	// Consteval function shouldn't be emitted.
6747	if (auto *FD = dyn_cast<FunctionDecl>(Val: D); FD && FD->isImmediateFunction())
6748	return;
6749
6750	switch (D->getKind()) {
6751	case Decl::CXXConversion:
6752	case Decl::CXXMethod:
6753	case Decl::Function:
6754	EmitGlobal(GD: cast<FunctionDecl>(Val: D));
6755	// Always provide some coverage mapping
6756	// even for the functions that aren't emitted.
6757	AddDeferredUnusedCoverageMapping(D);
6758	break;
6759
6760	case Decl::CXXDeductionGuide:
6761	// Function-like, but does not result in code emission.
6762	break;
6763
6764	case Decl::Var:
6765	case Decl::Decomposition:
6766	case Decl::VarTemplateSpecialization:
6767	EmitGlobal(GD: cast<VarDecl>(Val: D));
6768	if (auto *DD = dyn_cast<DecompositionDecl>(Val: D))
6769	for (auto *B : DD->bindings())
6770	if (auto *HD = B->getHoldingVar())
6771	EmitGlobal(GD: HD);
6772	break;
6773
6774	// Indirect fields from global anonymous structs and unions can be
6775	// ignored; only the actual variable requires IR gen support.
6776	case Decl::IndirectField:
6777	break;
6778
6779	// C++ Decls
6780	case Decl::Namespace:
6781	EmitDeclContext(cast<NamespaceDecl>(Val: D));
6782	break;
6783	case Decl::ClassTemplateSpecialization: {
6784	const auto *Spec = cast<ClassTemplateSpecializationDecl>(Val: D);
6785	if (CGDebugInfo *DI = getModuleDebugInfo())
6786	if (Spec->getSpecializationKind() ==
6787	TSK_ExplicitInstantiationDefinition &&
6788	Spec->hasDefinition())
6789	DI->completeTemplateDefinition(SD: *Spec);
6790	} [[fallthrough]];
6791	case Decl::CXXRecord: {
6792	CXXRecordDecl *CRD = cast<CXXRecordDecl>(Val: D);
6793	if (CGDebugInfo *DI = getModuleDebugInfo()) {
6794	if (CRD->hasDefinition())
6795	DI->EmitAndRetainType(Ty: getContext().getRecordType(Decl: cast<RecordDecl>(Val: D)));
6796	if (auto *ES = D->getASTContext().getExternalSource())
6797	if (ES->hasExternalDefinitions(D) == ExternalASTSource::EK_Never)
6798	DI->completeUnusedClass(D: *CRD);
6799	}
6800	// Emit any static data members, they may be definitions.
6801	for (auto *I : CRD->decls())
6802	if (isa<VarDecl>(I) || isa<CXXRecordDecl>(I))
6803	EmitTopLevelDecl(I);
6804	break;
6805	}
6806	// No code generation needed.
6807	case Decl::UsingShadow:
6808	case Decl::ClassTemplate:
6809	case Decl::VarTemplate:
6810	case Decl::Concept:
6811	case Decl::VarTemplatePartialSpecialization:
6812	case Decl::FunctionTemplate:
6813	case Decl::TypeAliasTemplate:
6814	case Decl::Block:
6815	case Decl::Empty:
6816	case Decl::Binding:
6817	break;
6818	case Decl::Using: // using X; [C++]
6819	if (CGDebugInfo *DI = getModuleDebugInfo())
6820	DI->EmitUsingDecl(UD: cast<UsingDecl>(Val&: *D));
6821	break;
6822	case Decl::UsingEnum: // using enum X; [C++]
6823	if (CGDebugInfo *DI = getModuleDebugInfo())
6824	DI->EmitUsingEnumDecl(UD: cast<UsingEnumDecl>(Val&: *D));
6825	break;
6826	case Decl::NamespaceAlias:
6827	if (CGDebugInfo *DI = getModuleDebugInfo())
6828	DI->EmitNamespaceAlias(NA: cast<NamespaceAliasDecl>(Val&: *D));
6829	break;
6830	case Decl::UsingDirective: // using namespace X; [C++]
6831	if (CGDebugInfo *DI = getModuleDebugInfo())
6832	DI->EmitUsingDirective(UD: cast<UsingDirectiveDecl>(Val&: *D));
6833	break;
6834	case Decl::CXXConstructor:
6835	getCXXABI().EmitCXXConstructors(D: cast<CXXConstructorDecl>(Val: D));
6836	break;
6837	case Decl::CXXDestructor:
6838	getCXXABI().EmitCXXDestructors(D: cast<CXXDestructorDecl>(Val: D));
6839	break;
6840
6841	case Decl::StaticAssert:
6842	// Nothing to do.
6843	break;
6844
6845	// Objective-C Decls
6846
6847	// Forward declarations, no (immediate) code generation.
6848	case Decl::ObjCInterface:
6849	case Decl::ObjCCategory:
6850	break;
6851
6852	case Decl::ObjCProtocol: {
6853	auto *Proto = cast<ObjCProtocolDecl>(Val: D);
6854	if (Proto->isThisDeclarationADefinition())
6855	ObjCRuntime->GenerateProtocol(OPD: Proto);
6856	break;
6857	}
6858
6859	case Decl::ObjCCategoryImpl:
6860	// Categories have properties but don't support synthesize so we
6861	// can ignore them here.
6862	ObjCRuntime->GenerateCategory(OCD: cast<ObjCCategoryImplDecl>(Val: D));
6863	break;
6864
6865	case Decl::ObjCImplementation: {
6866	auto *OMD = cast<ObjCImplementationDecl>(Val: D);
6867	EmitObjCPropertyImplementations(D: OMD);
6868	EmitObjCIvarInitializations(D: OMD);
6869	ObjCRuntime->GenerateClass(OID: OMD);
6870	// Emit global variable debug information.
6871	if (CGDebugInfo *DI = getModuleDebugInfo())
6872	if (getCodeGenOpts().hasReducedDebugInfo())
6873	DI->getOrCreateInterfaceType(Ty: getContext().getObjCInterfaceType(
6874	Decl: OMD->getClassInterface()), Loc: OMD->getLocation());
6875	break;
6876	}
6877	case Decl::ObjCMethod: {
6878	auto *OMD = cast<ObjCMethodDecl>(Val: D);
6879	// If this is not a prototype, emit the body.
6880	if (OMD->getBody())
6881	CodeGenFunction(*this).GenerateObjCMethod(OMD);
6882	break;
6883	}
6884	case Decl::ObjCCompatibleAlias:
6885	ObjCRuntime->RegisterAlias(OAD: cast<ObjCCompatibleAliasDecl>(Val: D));
6886	break;
6887
6888	case Decl::PragmaComment: {
6889	const auto *PCD = cast<PragmaCommentDecl>(Val: D);
6890	switch (PCD->getCommentKind()) {
6891	case PCK_Unknown:
6892	llvm_unreachable("unexpected pragma comment kind");
6893	case PCK_Linker:
6894	AppendLinkerOptions(Opts: PCD->getArg());
6895	break;
6896	case PCK_Lib:
6897	AddDependentLib(Lib: PCD->getArg());
6898	break;
6899	case PCK_Compiler:
6900	case PCK_ExeStr:
6901	case PCK_User:
6902	break; // We ignore all of these.
6903	}
6904	break;
6905	}
6906
6907	case Decl::PragmaDetectMismatch: {
6908	const auto *PDMD = cast<PragmaDetectMismatchDecl>(Val: D);
6909	AddDetectMismatch(Name: PDMD->getName(), Value: PDMD->getValue());
6910	break;
6911	}
6912
6913	case Decl::LinkageSpec:
6914	EmitLinkageSpec(LSD: cast<LinkageSpecDecl>(Val: D));
6915	break;
6916
6917	case Decl::FileScopeAsm: {
6918	// File-scope asm is ignored during device-side CUDA compilation.
6919	if (LangOpts.CUDA && LangOpts.CUDAIsDevice)
6920	break;
6921	// File-scope asm is ignored during device-side OpenMP compilation.
6922	if (LangOpts.OpenMPIsTargetDevice)
6923	break;
6924	// File-scope asm is ignored during device-side SYCL compilation.
6925	if (LangOpts.SYCLIsDevice)
6926	break;
6927	auto *AD = cast<FileScopeAsmDecl>(Val: D);
6928	getModule().appendModuleInlineAsm(Asm: AD->getAsmString()->getString());
6929	break;
6930	}
6931
6932	case Decl::TopLevelStmt:
6933	EmitTopLevelStmt(D: cast<TopLevelStmtDecl>(Val: D));
6934	break;
6935
6936	case Decl::Import: {
6937	auto *Import = cast<ImportDecl>(Val: D);
6938
6939	// If we've already imported this module, we're done.
6940	if (!ImportedModules.insert(X: Import->getImportedModule()))
6941	break;
6942
6943	// Emit debug information for direct imports.
6944	if (!Import->getImportedOwningModule()) {
6945	if (CGDebugInfo *DI = getModuleDebugInfo())
6946	DI->EmitImportDecl(ID: *Import);
6947	}
6948
6949	// For C++ standard modules we are done - we will call the module
6950	// initializer for imported modules, and that will likewise call those for
6951	// any imports it has.
6952	if (CXX20ModuleInits && Import->getImportedOwningModule() &&
6953	!Import->getImportedOwningModule()->isModuleMapModule())
6954	break;
6955
6956	// For clang C++ module map modules the initializers for sub-modules are
6957	// emitted here.
6958
6959	// Find all of the submodules and emit the module initializers.
6960	llvm::SmallPtrSet<clang::Module *, 16> Visited;
6961	SmallVector<clang::Module *, 16> Stack;
6962	Visited.insert(Ptr: Import->getImportedModule());
6963	Stack.push_back(Elt: Import->getImportedModule());
6964
6965	while (!Stack.empty()) {
6966	clang::Module *Mod = Stack.pop_back_val();
6967	if (!EmittedModuleInitializers.insert(Ptr: Mod).second)
6968	continue;
6969
6970	for (auto *D : Context.getModuleInitializers(M: Mod))
6971	EmitTopLevelDecl(D);
6972
6973	// Visit the submodules of this module.
6974	for (auto *Submodule : Mod->submodules()) {
6975	// Skip explicit children; they need to be explicitly imported to emit
6976	// the initializers.
6977	if (Submodule->IsExplicit)
6978	continue;
6979
6980	if (Visited.insert(Ptr: Submodule).second)
6981	Stack.push_back(Elt: Submodule);
6982	}
6983	}
6984	break;
6985	}
6986
6987	case Decl::Export:
6988	EmitDeclContext(cast<ExportDecl>(Val: D));
6989	break;
6990
6991	case Decl::OMPThreadPrivate:
6992	EmitOMPThreadPrivateDecl(D: cast<OMPThreadPrivateDecl>(Val: D));
6993	break;
6994
6995	case Decl::OMPAllocate:
6996	EmitOMPAllocateDecl(D: cast<OMPAllocateDecl>(Val: D));
6997	break;
6998
6999	case Decl::OMPDeclareReduction:
7000	EmitOMPDeclareReduction(D: cast<OMPDeclareReductionDecl>(Val: D));
7001	break;
7002
7003	case Decl::OMPDeclareMapper:
7004	EmitOMPDeclareMapper(D: cast<OMPDeclareMapperDecl>(Val: D));
7005	break;
7006
7007	case Decl::OMPRequires:
7008	EmitOMPRequiresDecl(D: cast<OMPRequiresDecl>(Val: D));
7009	break;
7010
7011	case Decl::Typedef:
7012	case Decl::TypeAlias: // using foo = bar; [C++11]
7013	if (CGDebugInfo *DI = getModuleDebugInfo())
7014	DI->EmitAndRetainType(
7015	Ty: getContext().getTypedefType(Decl: cast<TypedefNameDecl>(Val: D)));
7016	break;
7017
7018	case Decl::Record:
7019	if (CGDebugInfo *DI = getModuleDebugInfo())
7020	if (cast<RecordDecl>(Val: D)->getDefinition())
7021	DI->EmitAndRetainType(Ty: getContext().getRecordType(Decl: cast<RecordDecl>(Val: D)));
7022	break;
7023
7024	case Decl::Enum:
7025	if (CGDebugInfo *DI = getModuleDebugInfo())
7026	if (cast<EnumDecl>(Val: D)->getDefinition())
7027	DI->EmitAndRetainType(Ty: getContext().getEnumType(Decl: cast<EnumDecl>(Val: D)));
7028	break;
7029
7030	case Decl::HLSLBuffer:
7031	getHLSLRuntime().addBuffer(D: cast<HLSLBufferDecl>(Val: D));
7032	break;
7033
7034	default:
7035	// Make sure we handled everything we should, every other kind is a
7036	// non-top-level decl. FIXME: Would be nice to have an isTopLevelDeclKind
7037	// function. Need to recode Decl::Kind to do that easily.
7038	assert(isa<TypeDecl>(D) && "Unsupported decl kind");
7039	break;
7040	}
7041	}
7042
7043	void CodeGenModule::AddDeferredUnusedCoverageMapping(Decl *D) {
7044	// Do we need to generate coverage mapping?
7045	if (!CodeGenOpts.CoverageMapping)
7046	return;
7047	switch (D->getKind()) {
7048	case Decl::CXXConversion:
7049	case Decl::CXXMethod:
7050	case Decl::Function:
7051	case Decl::ObjCMethod:
7052	case Decl::CXXConstructor:
7053	case Decl::CXXDestructor: {
7054	if (!cast<FunctionDecl>(Val: D)->doesThisDeclarationHaveABody())
7055	break;
7056	SourceManager &SM = getContext().getSourceManager();
7057	if (LimitedCoverage && SM.getMainFileID() != SM.getFileID(SpellingLoc: D->getBeginLoc()))
7058	break;
7059	DeferredEmptyCoverageMappingDecls.try_emplace(Key: D, Args: true);
7060	break;
7061	}
7062	default:
7063	break;
7064	};
7065	}
7066
7067	void CodeGenModule::ClearUnusedCoverageMapping(const Decl *D) {
7068	// Do we need to generate coverage mapping?
7069	if (!CodeGenOpts.CoverageMapping)
7070	return;
7071	if (const auto *Fn = dyn_cast<FunctionDecl>(Val: D)) {
7072	if (Fn->isTemplateInstantiation())
7073	ClearUnusedCoverageMapping(Fn->getTemplateInstantiationPattern());
7074	}
7075	DeferredEmptyCoverageMappingDecls.insert_or_assign(Key: D, Val: false);
7076	}
7077
7078	void CodeGenModule::EmitDeferredUnusedCoverageMappings() {
7079	// We call takeVector() here to avoid use-after-free.
7080	// FIXME: DeferredEmptyCoverageMappingDecls is getting mutated because
7081	// we deserialize function bodies to emit coverage info for them, and that
7082	// deserializes more declarations. How should we handle that case?
7083	for (const auto &Entry : DeferredEmptyCoverageMappingDecls.takeVector()) {
7084	if (!Entry.second)
7085	continue;
7086	const Decl *D = Entry.first;
7087	switch (D->getKind()) {
7088	case Decl::CXXConversion:
7089	case Decl::CXXMethod:
7090	case Decl::Function:
7091	case Decl::ObjCMethod: {
7092	CodeGenPGO PGO(*this);
7093	GlobalDecl GD(cast<FunctionDecl>(Val: D));
7094	PGO.emitEmptyCounterMapping(D, FuncName: getMangledName(GD),
7095	Linkage: getFunctionLinkage(GD));
7096	break;
7097	}
7098	case Decl::CXXConstructor: {
7099	CodeGenPGO PGO(*this);
7100	GlobalDecl GD(cast<CXXConstructorDecl>(Val: D), Ctor_Base);
7101	PGO.emitEmptyCounterMapping(D, FuncName: getMangledName(GD),
7102	Linkage: getFunctionLinkage(GD));
7103	break;
7104	}
7105	case Decl::CXXDestructor: {
7106	CodeGenPGO PGO(*this);
7107	GlobalDecl GD(cast<CXXDestructorDecl>(Val: D), Dtor_Base);
7108	PGO.emitEmptyCounterMapping(D, FuncName: getMangledName(GD),
7109	Linkage: getFunctionLinkage(GD));
7110	break;
7111	}
7112	default:
7113	break;
7114	};
7115	}
7116	}
7117
7118	void CodeGenModule::EmitMainVoidAlias() {
7119	// In order to transition away from "__original_main" gracefully, emit an
7120	// alias for "main" in the no-argument case so that libc can detect when
7121	// new-style no-argument main is in used.
7122	if (llvm::Function *F = getModule().getFunction(Name: "main")) {
7123	if (!F->isDeclaration() && F->arg_size() == 0 && !F->isVarArg() &&
7124	F->getReturnType()->isIntegerTy(Bitwidth: Context.getTargetInfo().getIntWidth())) {
7125	auto *GA = llvm::GlobalAlias::create(Name: "__main_void", Aliasee: F);
7126	GA->setVisibility(llvm::GlobalValue::HiddenVisibility);
7127	}
7128	}
7129	}
7130
7131	/// Turns the given pointer into a constant.
7132	static llvm::Constant *GetPointerConstant(llvm::LLVMContext &Context,
7133	const void *Ptr) {
7134	uintptr_t PtrInt = reinterpret_cast<uintptr_t>(Ptr);
7135	llvm::Type *i64 = llvm::Type::getInt64Ty(C&: Context);
7136	return llvm::ConstantInt::get(Ty: i64, V: PtrInt);
7137	}
7138
7139	static void EmitGlobalDeclMetadata(CodeGenModule &CGM,
7140	llvm::NamedMDNode *&GlobalMetadata,
7141	GlobalDecl D,
7142	llvm::GlobalValue *Addr) {
7143	if (!GlobalMetadata)
7144	GlobalMetadata =
7145	CGM.getModule().getOrInsertNamedMetadata(Name: "clang.global.decl.ptrs");
7146
7147	// TODO: should we report variant information for ctors/dtors?
7148	llvm::Metadata *Ops[] = {llvm::ConstantAsMetadata::get(C: Addr),
7149	llvm::ConstantAsMetadata::get(C: GetPointerConstant(
7150	Context&: CGM.getLLVMContext(), Ptr: D.getDecl()))};
7151	GlobalMetadata->addOperand(M: llvm::MDNode::get(Context&: CGM.getLLVMContext(), MDs: Ops));
7152	}
7153
7154	bool CodeGenModule::CheckAndReplaceExternCIFuncs(llvm::GlobalValue *Elem,
7155	llvm::GlobalValue *CppFunc) {
7156	// Store the list of ifuncs we need to replace uses in.
7157	llvm::SmallVector<llvm::GlobalIFunc *> IFuncs;
7158	// List of ConstantExprs that we should be able to delete when we're done
7159	// here.
7160	llvm::SmallVector<llvm::ConstantExpr *> CEs;
7161
7162	// It isn't valid to replace the extern-C ifuncs if all we find is itself!
7163	if (Elem == CppFunc)
7164	return false;
7165
7166	// First make sure that all users of this are ifuncs (or ifuncs via a
7167	// bitcast), and collect the list of ifuncs and CEs so we can work on them
7168	// later.
7169	for (llvm::User *User : Elem->users()) {
7170	// Users can either be a bitcast ConstExpr that is used by the ifuncs, OR an
7171	// ifunc directly. In any other case, just give up, as we don't know what we
7172	// could break by changing those.
7173	if (auto *ConstExpr = dyn_cast<llvm::ConstantExpr>(Val: User)) {
7174	if (ConstExpr->getOpcode() != llvm::Instruction::BitCast)
7175	return false;
7176
7177	for (llvm::User *CEUser : ConstExpr->users()) {
7178	if (auto *IFunc = dyn_cast<llvm::GlobalIFunc>(Val: CEUser)) {
7179	IFuncs.push_back(Elt: IFunc);
7180	} else {
7181	return false;
7182	}
7183	}
7184	CEs.push_back(Elt: ConstExpr);
7185	} else if (auto *IFunc = dyn_cast<llvm::GlobalIFunc>(Val: User)) {
7186	IFuncs.push_back(Elt: IFunc);
7187	} else {
7188	// This user is one we don't know how to handle, so fail redirection. This
7189	// will result in an ifunc retaining a resolver name that will ultimately
7190	// fail to be resolved to a defined function.
7191	return false;
7192	}
7193	}
7194
7195	// Now we know this is a valid case where we can do this alias replacement, we
7196	// need to remove all of the references to Elem (and the bitcasts!) so we can
7197	// delete it.
7198	for (llvm::GlobalIFunc *IFunc : IFuncs)
7199	IFunc->setResolver(nullptr);
7200	for (llvm::ConstantExpr *ConstExpr : CEs)
7201	ConstExpr->destroyConstant();
7202
7203	// We should now be out of uses for the 'old' version of this function, so we
7204	// can erase it as well.
7205	Elem->eraseFromParent();
7206
7207	for (llvm::GlobalIFunc *IFunc : IFuncs) {
7208	// The type of the resolver is always just a function-type that returns the
7209	// type of the IFunc, so create that here. If the type of the actual
7210	// resolver doesn't match, it just gets bitcast to the right thing.
7211	auto *ResolverTy =
7212	llvm::FunctionType::get(Result: IFunc->getType(), /*isVarArg*/ false);
7213	llvm::Constant *Resolver = GetOrCreateLLVMFunction(
7214	MangledName: CppFunc->getName(), Ty: ResolverTy, GD: {}, /*ForVTable*/ false);
7215	IFunc->setResolver(Resolver);
7216	}
7217	return true;
7218	}
7219
7220	/// For each function which is declared within an extern "C" region and marked
7221	/// as 'used', but has internal linkage, create an alias from the unmangled
7222	/// name to the mangled name if possible. People expect to be able to refer
7223	/// to such functions with an unmangled name from inline assembly within the
7224	/// same translation unit.
7225	void CodeGenModule::EmitStaticExternCAliases() {
7226	if (!getTargetCodeGenInfo().shouldEmitStaticExternCAliases())
7227	return;
7228	for (auto &I : StaticExternCValues) {
7229	const IdentifierInfo *Name = I.first;
7230	llvm::GlobalValue *Val = I.second;
7231
7232	// If Val is null, that implies there were multiple declarations that each
7233	// had a claim to the unmangled name. In this case, generation of the alias
7234	// is suppressed. See CodeGenModule::MaybeHandleStaticInExternC.
7235	if (!Val)
7236	break;
7237
7238	llvm::GlobalValue *ExistingElem =
7239	getModule().getNamedValue(Name: Name->getName());
7240
7241	// If there is either not something already by this name, or we were able to
7242	// replace all uses from IFuncs, create the alias.
7243	if (!ExistingElem || CheckAndReplaceExternCIFuncs(Elem: ExistingElem, CppFunc: Val))
7244	addCompilerUsedGlobal(GV: llvm::GlobalAlias::create(Name: Name->getName(), Aliasee: Val));
7245	}
7246	}
7247
7248	bool CodeGenModule::lookupRepresentativeDecl(StringRef MangledName,
7249	GlobalDecl &Result) const {
7250	auto Res = Manglings.find(Key: MangledName);
7251	if (Res == Manglings.end())
7252	return false;
7253	Result = Res->getValue();
7254	return true;
7255	}
7256
7257	/// Emits metadata nodes associating all the global values in the
7258	/// current module with the Decls they came from. This is useful for
7259	/// projects using IR gen as a subroutine.
7260	///
7261	/// Since there's currently no way to associate an MDNode directly
7262	/// with an llvm::GlobalValue, we create a global named metadata
7263	/// with the name 'clang.global.decl.ptrs'.
7264	void CodeGenModule::EmitDeclMetadata() {
7265	llvm::NamedMDNode *GlobalMetadata = nullptr;
7266
7267	for (auto &I : MangledDeclNames) {
7268	llvm::GlobalValue *Addr = getModule().getNamedValue(Name: I.second);
7269	// Some mangled names don't necessarily have an associated GlobalValue
7270	// in this module, e.g. if we mangled it for DebugInfo.
7271	if (Addr)
7272	EmitGlobalDeclMetadata(CGM&: *this, GlobalMetadata, D: I.first, Addr);
7273	}
7274	}
7275
7276	/// Emits metadata nodes for all the local variables in the current
7277	/// function.
7278	void CodeGenFunction::EmitDeclMetadata() {
7279	if (LocalDeclMap.empty()) return;
7280
7281	llvm::LLVMContext &Context = getLLVMContext();
7282
7283	// Find the unique metadata ID for this name.
7284	unsigned DeclPtrKind = Context.getMDKindID(Name: "clang.decl.ptr");
7285
7286	llvm::NamedMDNode *GlobalMetadata = nullptr;
7287
7288	for (auto &I : LocalDeclMap) {
7289	const Decl *D = I.first;
7290	llvm::Value *Addr = I.second.emitRawPointer(CGF&: *this);
7291	if (auto *Alloca = dyn_cast<llvm::AllocaInst>(Val: Addr)) {
7292	llvm::Value *DAddr = GetPointerConstant(Context&: getLLVMContext(), Ptr: D);
7293	Alloca->setMetadata(
7294	KindID: DeclPtrKind, Node: llvm::MDNode::get(
7295	Context, MDs: llvm::ValueAsMetadata::getConstant(C: DAddr)));
7296	} else if (auto *GV = dyn_cast<llvm::GlobalValue>(Val: Addr)) {
7297	GlobalDecl GD = GlobalDecl(cast<VarDecl>(Val: D));
7298	EmitGlobalDeclMetadata(CGM, GlobalMetadata, D: GD, Addr: GV);
7299	}
7300	}
7301	}
7302
7303	void CodeGenModule::EmitVersionIdentMetadata() {
7304	llvm::NamedMDNode *IdentMetadata =
7305	TheModule.getOrInsertNamedMetadata(Name: "llvm.ident");
7306	std::string Version = getClangFullVersion();
7307	llvm::LLVMContext &Ctx = TheModule.getContext();
7308
7309	llvm::Metadata *IdentNode[] = {llvm::MDString::get(Context&: Ctx, Str: Version)};
7310	IdentMetadata->addOperand(M: llvm::MDNode::get(Context&: Ctx, MDs: IdentNode));
7311	}
7312
7313	void CodeGenModule::EmitCommandLineMetadata() {
7314	llvm::NamedMDNode *CommandLineMetadata =
7315	TheModule.getOrInsertNamedMetadata(Name: "llvm.commandline");
7316	std::string CommandLine = getCodeGenOpts().RecordCommandLine;
7317	llvm::LLVMContext &Ctx = TheModule.getContext();
7318
7319	llvm::Metadata *CommandLineNode[] = {llvm::MDString::get(Context&: Ctx, Str: CommandLine)};
7320	CommandLineMetadata->addOperand(M: llvm::MDNode::get(Context&: Ctx, MDs: CommandLineNode));
7321	}
7322
7323	void CodeGenModule::EmitCoverageFile() {
7324	llvm::NamedMDNode *CUNode = TheModule.getNamedMetadata(Name: "llvm.dbg.cu");
7325	if (!CUNode)
7326	return;
7327
7328	llvm::NamedMDNode *GCov = TheModule.getOrInsertNamedMetadata(Name: "llvm.gcov");
7329	llvm::LLVMContext &Ctx = TheModule.getContext();
7330	auto *CoverageDataFile =
7331	llvm::MDString::get(Context&: Ctx, Str: getCodeGenOpts().CoverageDataFile);
7332	auto *CoverageNotesFile =
7333	llvm::MDString::get(Context&: Ctx, Str: getCodeGenOpts().CoverageNotesFile);
7334	for (int i = 0, e = CUNode->getNumOperands(); i != e; ++i) {
7335	llvm::MDNode *CU = CUNode->getOperand(i);
7336	llvm::Metadata *Elts[] = {CoverageNotesFile, CoverageDataFile, CU};
7337	GCov->addOperand(M: llvm::MDNode::get(Context&: Ctx, MDs: Elts));
7338	}
7339	}
7340
7341	llvm::Constant *CodeGenModule::GetAddrOfRTTIDescriptor(QualType Ty,
7342	bool ForEH) {
7343	// Return a bogus pointer if RTTI is disabled, unless it's for EH.
7344	// FIXME: should we even be calling this method if RTTI is disabled
7345	// and it's not for EH?
7346	if (!shouldEmitRTTI(ForEH))
7347	return llvm::Constant::getNullValue(Ty: GlobalsInt8PtrTy);
7348
7349	if (ForEH && Ty->isObjCObjectPointerType() &&
7350	LangOpts.ObjCRuntime.isGNUFamily())
7351	return ObjCRuntime->GetEHType(T: Ty);
7352
7353	return getCXXABI().getAddrOfRTTIDescriptor(Ty);
7354	}
7355
7356	void CodeGenModule::EmitOMPThreadPrivateDecl(const OMPThreadPrivateDecl *D) {
7357	// Do not emit threadprivates in simd-only mode.
7358	if (LangOpts.OpenMP && LangOpts.OpenMPSimd)
7359	return;
7360	for (auto RefExpr : D->varlists()) {
7361	auto *VD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: RefExpr)->getDecl());
7362	bool PerformInit =
7363	VD->getAnyInitializer() &&
7364	!VD->getAnyInitializer()->isConstantInitializer(Ctx&: getContext(),
7365	/*ForRef=*/false);
7366
7367	Address Addr(GetAddrOfGlobalVar(D: VD),
7368	getTypes().ConvertTypeForMem(T: VD->getType()),
7369	getContext().getDeclAlign(VD));
7370	if (auto InitFunction = getOpenMPRuntime().emitThreadPrivateVarDefinition(
7371	VD, Addr, RefExpr->getBeginLoc(), PerformInit))
7372	CXXGlobalInits.push_back(InitFunction);
7373	}
7374	}
7375
7376	llvm::Metadata *
7377	CodeGenModule::CreateMetadataIdentifierImpl(QualType T, MetadataTypeMap &Map,
7378	StringRef Suffix) {
7379	if (auto *FnType = T->getAs<FunctionProtoType>())
7380	T = getContext().getFunctionType(
7381	ResultTy: FnType->getReturnType(), Args: FnType->getParamTypes(),
7382	EPI: FnType->getExtProtoInfo().withExceptionSpec(ESI: EST_None));
7383
7384	llvm::Metadata *&InternalId = Map[T.getCanonicalType()];
7385	if (InternalId)
7386	return InternalId;
7387
7388	if (isExternallyVisible(L: T->getLinkage())) {
7389	std::string OutName;
7390	llvm::raw_string_ostream Out(OutName);
7391	getCXXABI().getMangleContext().mangleCanonicalTypeName(
7392	T, Out, NormalizeIntegers: getCodeGenOpts().SanitizeCfiICallNormalizeIntegers);
7393
7394	if (getCodeGenOpts().SanitizeCfiICallNormalizeIntegers)
7395	Out << ".normalized";
7396
7397	Out << Suffix;
7398
7399	InternalId = llvm::MDString::get(Context&: getLLVMContext(), Str: Out.str());
7400	} else {
7401	InternalId = llvm::MDNode::getDistinct(Context&: getLLVMContext(),
7402	MDs: llvm::ArrayRef<llvm::Metadata *>());
7403	}
7404
7405	return InternalId;
7406	}
7407
7408	llvm::Metadata *CodeGenModule::CreateMetadataIdentifierForType(QualType T) {
7409	return CreateMetadataIdentifierImpl(T, Map&: MetadataIdMap, Suffix: "");
7410	}
7411
7412	llvm::Metadata *
7413	CodeGenModule::CreateMetadataIdentifierForVirtualMemPtrType(QualType T) {
7414	return CreateMetadataIdentifierImpl(T, Map&: VirtualMetadataIdMap, Suffix: ".virtual");
7415	}
7416
7417	// Generalize pointer types to a void pointer with the qualifiers of the
7418	// originally pointed-to type, e.g. 'const char *' and 'char * const *'
7419	// generalize to 'const void *' while 'char *' and 'const char **' generalize to
7420	// 'void *'.
7421	static QualType GeneralizeType(ASTContext &Ctx, QualType Ty) {
7422	if (!Ty->isPointerType())
7423	return Ty;
7424
7425	return Ctx.getPointerType(
7426	T: QualType(Ctx.VoidTy).withCVRQualifiers(
7427	CVR: Ty->getPointeeType().getCVRQualifiers()));
7428	}
7429
7430	// Apply type generalization to a FunctionType's return and argument types
7431	static QualType GeneralizeFunctionType(ASTContext &Ctx, QualType Ty) {
7432	if (auto *FnType = Ty->getAs<FunctionProtoType>()) {
7433	SmallVector<QualType, 8> GeneralizedParams;
7434	for (auto &Param : FnType->param_types())
7435	GeneralizedParams.push_back(Elt: GeneralizeType(Ctx, Ty: Param));
7436
7437	return Ctx.getFunctionType(
7438	ResultTy: GeneralizeType(Ctx, FnType->getReturnType()),
7439	Args: GeneralizedParams, EPI: FnType->getExtProtoInfo());
7440	}
7441
7442	if (auto *FnType = Ty->getAs<FunctionNoProtoType>())
7443	return Ctx.getFunctionNoProtoType(
7444	GeneralizeType(Ctx, FnType->getReturnType()));
7445
7446	llvm_unreachable("Encountered unknown FunctionType");
7447	}
7448
7449	llvm::Metadata *CodeGenModule::CreateMetadataIdentifierGeneralized(QualType T) {
7450	return CreateMetadataIdentifierImpl(T: GeneralizeFunctionType(Ctx&: getContext(), Ty: T),
7451	Map&: GeneralizedMetadataIdMap, Suffix: ".generalized");
7452	}
7453
7454	/// Returns whether this module needs the "all-vtables" type identifier.
7455	bool CodeGenModule::NeedAllVtablesTypeId() const {
7456	// Returns true if at least one of vtable-based CFI checkers is enabled and
7457	// is not in the trapping mode.
7458	return ((LangOpts.Sanitize.has(K: SanitizerKind::CFIVCall) &&
7459	!CodeGenOpts.SanitizeTrap.has(K: SanitizerKind::CFIVCall)) ||
7460	(LangOpts.Sanitize.has(K: SanitizerKind::CFINVCall) &&
7461	!CodeGenOpts.SanitizeTrap.has(K: SanitizerKind::CFINVCall)) ||
7462	(LangOpts.Sanitize.has(K: SanitizerKind::CFIDerivedCast) &&
7463	!CodeGenOpts.SanitizeTrap.has(K: SanitizerKind::CFIDerivedCast)) ||
7464	(LangOpts.Sanitize.has(K: SanitizerKind::CFIUnrelatedCast) &&
7465	!CodeGenOpts.SanitizeTrap.has(K: SanitizerKind::CFIUnrelatedCast)));
7466	}
7467
7468	void CodeGenModule::AddVTableTypeMetadata(llvm::GlobalVariable *VTable,
7469	CharUnits Offset,
7470	const CXXRecordDecl *RD) {
7471	llvm::Metadata *MD =
7472	CreateMetadataIdentifierForType(T: QualType(RD->getTypeForDecl(), 0));
7473	VTable->addTypeMetadata(Offset: Offset.getQuantity(), TypeID: MD);
7474
7475	if (CodeGenOpts.SanitizeCfiCrossDso)
7476	if (auto CrossDsoTypeId = CreateCrossDsoCfiTypeId(MD))
7477	VTable->addTypeMetadata(Offset: Offset.getQuantity(),
7478	TypeID: llvm::ConstantAsMetadata::get(C: CrossDsoTypeId));
7479
7480	if (NeedAllVtablesTypeId()) {
7481	llvm::Metadata *MD = llvm::MDString::get(Context&: getLLVMContext(), Str: "all-vtables");
7482	VTable->addTypeMetadata(Offset: Offset.getQuantity(), TypeID: MD);
7483	}
7484	}
7485
7486	llvm::SanitizerStatReport &CodeGenModule::getSanStats() {
7487	if (!SanStats)
7488	SanStats = std::make_unique<llvm::SanitizerStatReport>(args: &getModule());
7489
7490	return *SanStats;
7491	}
7492
7493	llvm::Value *
7494	CodeGenModule::createOpenCLIntToSamplerConversion(const Expr *E,
7495	CodeGenFunction &CGF) {
7496	llvm::Constant *C = ConstantEmitter(CGF).emitAbstract(E, T: E->getType());
7497	auto *SamplerT = getOpenCLRuntime().getSamplerType(T: E->getType().getTypePtr());
7498	auto *FTy = llvm::FunctionType::get(Result: SamplerT, Params: {C->getType()}, isVarArg: false);
7499	auto *Call = CGF.EmitRuntimeCall(
7500	callee: CreateRuntimeFunction(FTy, Name: "__translate_sampler_initializer"), args: {C});
7501	return Call;
7502	}
7503
7504	CharUnits CodeGenModule::getNaturalPointeeTypeAlignment(
7505	QualType T, LValueBaseInfo *BaseInfo, TBAAAccessInfo *TBAAInfo) {
7506	return getNaturalTypeAlignment(T: T->getPointeeType(), BaseInfo, TBAAInfo,
7507	/* forPointeeType= */ true);
7508	}
7509
7510	CharUnits CodeGenModule::getNaturalTypeAlignment(QualType T,
7511	LValueBaseInfo *BaseInfo,
7512	TBAAAccessInfo *TBAAInfo,
7513	bool forPointeeType) {
7514	if (TBAAInfo)
7515	*TBAAInfo = getTBAAAccessInfo(AccessType: T);
7516
7517	// FIXME: This duplicates logic in ASTContext::getTypeAlignIfKnown. But
7518	// that doesn't return the information we need to compute BaseInfo.
7519
7520	// Honor alignment typedef attributes even on incomplete types.
7521	// We also honor them straight for C++ class types, even as pointees;
7522	// there's an expressivity gap here.
7523	if (auto TT = T->getAs<TypedefType>()) {
7524	if (auto Align = TT->getDecl()->getMaxAlignment()) {
7525	if (BaseInfo)
7526	*BaseInfo = LValueBaseInfo(AlignmentSource::AttributedType);
7527	return getContext().toCharUnitsFromBits(BitSize: Align);
7528	}
7529	}
7530
7531	bool AlignForArray = T->isArrayType();
7532
7533	// Analyze the base element type, so we don't get confused by incomplete
7534	// array types.
7535	T = getContext().getBaseElementType(QT: T);
7536
7537	if (T->isIncompleteType()) {
7538	// We could try to replicate the logic from
7539	// ASTContext::getTypeAlignIfKnown, but nothing uses the alignment if the
7540	// type is incomplete, so it's impossible to test. We could try to reuse
7541	// getTypeAlignIfKnown, but that doesn't return the information we need
7542	// to set BaseInfo. So just ignore the possibility that the alignment is
7543	// greater than one.
7544	if (BaseInfo)
7545	*BaseInfo = LValueBaseInfo(AlignmentSource::Type);
7546	return CharUnits::One();
7547	}
7548
7549	if (BaseInfo)
7550	*BaseInfo = LValueBaseInfo(AlignmentSource::Type);
7551
7552	CharUnits Alignment;
7553	const CXXRecordDecl *RD;
7554	if (T.getQualifiers().hasUnaligned()) {
7555	Alignment = CharUnits::One();
7556	} else if (forPointeeType && !AlignForArray &&
7557	(RD = T->getAsCXXRecordDecl())) {
7558	// For C++ class pointees, we don't know whether we're pointing at a
7559	// base or a complete object, so we generally need to use the
7560	// non-virtual alignment.
7561	Alignment = getClassPointerAlignment(CD: RD);
7562	} else {
7563	Alignment = getContext().getTypeAlignInChars(T);
7564	}
7565
7566	// Cap to the global maximum type alignment unless the alignment
7567	// was somehow explicit on the type.
7568	if (unsigned MaxAlign = getLangOpts().MaxTypeAlign) {
7569	if (Alignment.getQuantity() > MaxAlign &&
7570	!getContext().isAlignmentRequired(T))
7571	Alignment = CharUnits::fromQuantity(Quantity: MaxAlign);
7572	}
7573	return Alignment;
7574	}
7575
7576	bool CodeGenModule::stopAutoInit() {
7577	unsigned StopAfter = getContext().getLangOpts().TrivialAutoVarInitStopAfter;
7578	if (StopAfter) {
7579	// This number is positive only when -ftrivial-auto-var-init-stop-after=* is
7580	// used
7581	if (NumAutoVarInit >= StopAfter) {
7582	return true;
7583	}
7584	if (!NumAutoVarInit) {
7585	unsigned DiagID = getDiags().getCustomDiagID(
7586	L: DiagnosticsEngine::Warning,
7587	FormatString: "-ftrivial-auto-var-init-stop-after=%0 has been enabled to limit the "
7588	"number of times ftrivial-auto-var-init=%1 gets applied.");
7589	getDiags().Report(DiagID)
7590	<< StopAfter
7591	<< (getContext().getLangOpts().getTrivialAutoVarInit() ==
7592	LangOptions::TrivialAutoVarInitKind::Zero
7593	? "zero"
7594	: "pattern");
7595	}
7596	++NumAutoVarInit;
7597	}
7598	return false;
7599	}
7600
7601	void CodeGenModule::printPostfixForExternalizedDecl(llvm::raw_ostream &OS,
7602	const Decl *D) const {
7603	// ptxas does not allow '.' in symbol names. On the other hand, HIP prefers
7604	// postfix beginning with '.' since the symbol name can be demangled.
7605	if (LangOpts.HIP)
7606	OS << (isa<VarDecl>(Val: D) ? ".static." : ".intern.");
7607	else
7608	OS << (isa<VarDecl>(Val: D) ? "__static__" : "__intern__");
7609
7610	// If the CUID is not specified we try to generate a unique postfix.
7611	if (getLangOpts().CUID.empty()) {
7612	SourceManager &SM = getContext().getSourceManager();
7613	PresumedLoc PLoc = SM.getPresumedLoc(Loc: D->getLocation());
7614	assert(PLoc.isValid() && "Source location is expected to be valid.");
7615
7616	// Get the hash of the user defined macros.
7617	llvm::MD5 Hash;
7618	llvm::MD5::MD5Result Result;
7619	for (const auto &Arg : PreprocessorOpts.Macros)
7620	Hash.update(Str: Arg.first);
7621	Hash.final(Result);
7622
7623	// Get the UniqueID for the file containing the decl.
7624	llvm::sys::fs::UniqueID ID;
7625	if (llvm::sys::fs::getUniqueID(Path: PLoc.getFilename(), Result&: ID)) {
7626	PLoc = SM.getPresumedLoc(Loc: D->getLocation(), /*UseLineDirectives=*/false);
7627	assert(PLoc.isValid() && "Source location is expected to be valid.");
7628	if (auto EC = llvm::sys::fs::getUniqueID(PLoc.getFilename(), ID))
7629	SM.getDiagnostics().Report(diag::err_cannot_open_file)
7630	<< PLoc.getFilename() << EC.message();
7631	}
7632	OS << llvm::format(Fmt: "%x", Vals: ID.getFile()) << llvm::format(Fmt: "%x", Vals: ID.getDevice())
7633	<< "_" << llvm::utohexstr(X: Result.low(), /*LowerCase=*/true, /*Width=*/8);
7634	} else {
7635	OS << getContext().getCUIDHash();
7636	}
7637	}
7638
7639	void CodeGenModule::moveLazyEmissionStates(CodeGenModule *NewBuilder) {
7640	assert(DeferredDeclsToEmit.empty() &&
7641	"Should have emitted all decls deferred to emit.");
7642	assert(NewBuilder->DeferredDecls.empty() &&
7643	"Newly created module should not have deferred decls");
7644	NewBuilder->DeferredDecls = std::move(DeferredDecls);
7645	assert(EmittedDeferredDecls.empty() &&
7646	"Still have (unmerged) EmittedDeferredDecls deferred decls");
7647
7648	assert(NewBuilder->DeferredVTables.empty() &&
7649	"Newly created module should not have deferred vtables");
7650	NewBuilder->DeferredVTables = std::move(DeferredVTables);
7651
7652	assert(NewBuilder->MangledDeclNames.empty() &&
7653	"Newly created module should not have mangled decl names");
7654	assert(NewBuilder->Manglings.empty() &&
7655	"Newly created module should not have manglings");
7656	NewBuilder->Manglings = std::move(Manglings);
7657
7658	NewBuilder->WeakRefReferences = std::move(WeakRefReferences);
7659
7660	NewBuilder->TBAA = std::move(TBAA);
7661
7662	NewBuilder->ABI->MangleCtx = std::move(ABI->MangleCtx);
7663	}
7664