1	//===-- CodeGenFunction.h - Per-Function state for LLVM CodeGen -*- C++ -*-===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This is the internal per-function state used for llvm translation.
10	//
11	//===----------------------------------------------------------------------===//
12
13	#ifndef LLVM_CLANG_LIB_CODEGEN_CODEGENFUNCTION_H
14	#define LLVM_CLANG_LIB_CODEGEN_CODEGENFUNCTION_H
15
16	#include "CGBuilder.h"
17	#include "CGLoopInfo.h"
18	#include "CGValue.h"
19	#include "CodeGenModule.h"
20	#include "EHScopeStack.h"
21	#include "SanitizerHandler.h"
22	#include "VarBypassDetector.h"
23	#include "clang/AST/CharUnits.h"
24	#include "clang/AST/CurrentSourceLocExprScope.h"
25	#include "clang/AST/ExprCXX.h"
26	#include "clang/AST/ExprObjC.h"
27	#include "clang/AST/ExprOpenMP.h"
28	#include "clang/AST/StmtOpenACC.h"
29	#include "clang/AST/StmtOpenMP.h"
30	#include "clang/AST/StmtSYCL.h"
31	#include "clang/AST/Type.h"
32	#include "clang/Basic/ABI.h"
33	#include "clang/Basic/CapturedStmt.h"
34	#include "clang/Basic/CodeGenOptions.h"
35	#include "clang/Basic/OpenMPKinds.h"
36	#include "clang/Basic/TargetInfo.h"
37	#include "llvm/ADT/ArrayRef.h"
38	#include "llvm/ADT/DenseMap.h"
39	#include "llvm/ADT/MapVector.h"
40	#include "llvm/ADT/SmallVector.h"
41	#include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
42	#include "llvm/IR/Instructions.h"
43	#include "llvm/IR/ValueHandle.h"
44	#include "llvm/Support/Debug.h"
45	#include "llvm/Transforms/Utils/SanitizerStats.h"
46	#include <optional>
47
48	namespace llvm {
49	class BasicBlock;
50	class ConvergenceControlInst;
51	class LLVMContext;
52	class MDNode;
53	class SwitchInst;
54	class Twine;
55	class Value;
56	class CanonicalLoopInfo;
57	} // namespace llvm
58
59	namespace clang {
60	class ASTContext;
61	class CXXDestructorDecl;
62	class CXXForRangeStmt;
63	class CXXTryStmt;
64	class Decl;
65	class LabelDecl;
66	class FunctionDecl;
67	class FunctionProtoType;
68	class LabelStmt;
69	class ObjCContainerDecl;
70	class ObjCInterfaceDecl;
71	class ObjCIvarDecl;
72	class ObjCMethodDecl;
73	class ObjCImplementationDecl;
74	class ObjCPropertyImplDecl;
75	class TargetInfo;
76	class VarDecl;
77	class ObjCForCollectionStmt;
78	class ObjCAtTryStmt;
79	class ObjCAtThrowStmt;
80	class ObjCAtSynchronizedStmt;
81	class ObjCAutoreleasePoolStmt;
82	class OMPUseDevicePtrClause;
83	class OMPUseDeviceAddrClause;
84	class SVETypeFlags;
85	class OMPExecutableDirective;
86
87	namespace analyze_os_log {
88	class OSLogBufferLayout;
89	}
90
91	namespace CodeGen {
92	class CodeGenTypes;
93	class CodeGenPGO;
94	class CGCallee;
95	class CGFunctionInfo;
96	class CGBlockInfo;
97	class CGCXXABI;
98	class BlockByrefHelpers;
99	class BlockByrefInfo;
100	class BlockFieldFlags;
101	class RegionCodeGenTy;
102	class TargetCodeGenInfo;
103	struct OMPTaskDataTy;
104	struct CGCoroData;
105
106	// clang-format off
107	/// The kind of evaluation to perform on values of a particular
108	/// type. Basically, is the code in CGExprScalar, CGExprComplex, or
109	/// CGExprAgg?
110	///
111	/// TODO: should vectors maybe be split out into their own thing?
112	enum TypeEvaluationKind {
113	TEK_Scalar,
114	TEK_Complex,
115	TEK_Aggregate
116	};
117	// clang-format on
118
119	/// Helper class with most of the code for saving a value for a
120	/// conditional expression cleanup.
121	struct DominatingLLVMValue {
122	typedef llvm::PointerIntPair<llvm::Value *, 1, bool> saved_type;
123
124	/// Answer whether the given value needs extra work to be saved.
125	static bool needsSaving(llvm::Value *value) {
126	if (!value)
127	return false;
128
129	// If it's not an instruction, we don't need to save.
130	if (!isa<llvm::Instruction>(Val: value))
131	return false;
132
133	// If it's an instruction in the entry block, we don't need to save.
134	llvm::BasicBlock *block = cast<llvm::Instruction>(Val: value)->getParent();
135	return (block != &block->getParent()->getEntryBlock());
136	}
137
138	static saved_type save(CodeGenFunction &CGF, llvm::Value *value);
139	static llvm::Value *restore(CodeGenFunction &CGF, saved_type value);
140	};
141
142	/// A partial specialization of DominatingValue for llvm::Values that
143	/// might be llvm::Instructions.
144	template <class T> struct DominatingPointer<T, true> : DominatingLLVMValue {
145	typedef T *type;
146	static type restore(CodeGenFunction &CGF, saved_type value) {
147	return static_cast<T *>(DominatingLLVMValue::restore(CGF, value));
148	}
149	};
150
151	/// A specialization of DominatingValue for Address.
152	template <> struct DominatingValue<Address> {
153	typedef Address type;
154
155	struct saved_type {
156	DominatingLLVMValue::saved_type BasePtr;
157	llvm::Type *ElementType;
158	CharUnits Alignment;
159	DominatingLLVMValue::saved_type Offset;
160	llvm::PointerType *EffectiveType;
161	};
162
163	static bool needsSaving(type value) {
164	if (DominatingLLVMValue::needsSaving(value: value.getBasePointer()) ||
165	DominatingLLVMValue::needsSaving(value: value.getOffset()))
166	return true;
167	return false;
168	}
169	static saved_type save(CodeGenFunction &CGF, type value) {
170	return {.BasePtr: DominatingLLVMValue::save(CGF, value: value.getBasePointer()),
171	.ElementType: value.getElementType(), .Alignment: value.getAlignment(),
172	.Offset: DominatingLLVMValue::save(CGF, value: value.getOffset()), .EffectiveType: value.getType()};
173	}
174	static type restore(CodeGenFunction &CGF, saved_type value) {
175	return Address(DominatingLLVMValue::restore(CGF, value: value.BasePtr),
176	value.ElementType, value.Alignment, CGPointerAuthInfo(),
177	DominatingLLVMValue::restore(CGF, value: value.Offset));
178	}
179	};
180
181	/// A specialization of DominatingValue for RValue.
182	template <> struct DominatingValue<RValue> {
183	typedef RValue type;
184	class saved_type {
185	enum Kind {
186	ScalarLiteral,
187	ScalarAddress,
188	AggregateLiteral,
189	AggregateAddress,
190	ComplexAddress
191	};
192	union {
193	struct {
194	DominatingLLVMValue::saved_type first, second;
195	} Vals;
196	DominatingValue<Address>::saved_type AggregateAddr;
197	};
198	LLVM_PREFERRED_TYPE(Kind)
199	unsigned K : 3;
200
201	saved_type(DominatingLLVMValue::saved_type Val1, unsigned K)
202	: Vals{.first: Val1, .second: DominatingLLVMValue::saved_type()}, K(K) {}
203
204	saved_type(DominatingLLVMValue::saved_type Val1,
205	DominatingLLVMValue::saved_type Val2)
206	: Vals{.first: Val1, .second: Val2}, K(ComplexAddress) {}
207
208	saved_type(DominatingValue<Address>::saved_type AggregateAddr, unsigned K)
209	: AggregateAddr(AggregateAddr), K(K) {}
210
211	public:
212	static bool needsSaving(RValue value);
213	static saved_type save(CodeGenFunction &CGF, RValue value);
214	RValue restore(CodeGenFunction &CGF);
215
216	// implementations in CGCleanup.cpp
217	};
218
219	static bool needsSaving(type value) { return saved_type::needsSaving(value); }
220	static saved_type save(CodeGenFunction &CGF, type value) {
221	return saved_type::save(CGF, value);
222	}
223	static type restore(CodeGenFunction &CGF, saved_type value) {
224	return value.restore(CGF);
225	}
226	};
227
228	/// A scoped helper to set the current source atom group for
229	/// CGDebugInfo::addInstToCurrentSourceAtom. A source atom is a source construct
230	/// that is "interesting" for debug stepping purposes. We use an atom group
231	/// number to track the instruction(s) that implement the functionality for the
232	/// atom, plus backup instructions/source locations.
233	class ApplyAtomGroup {
234	uint64_t OriginalAtom = 0;
235	CGDebugInfo *DI = nullptr;
236
237	public:
238	ApplyAtomGroup(CGDebugInfo *DI);
239	~ApplyAtomGroup();
240	};
241
242	/// CodeGenFunction - This class organizes the per-function state that is used
243	/// while generating LLVM code.
244	class CodeGenFunction : public CodeGenTypeCache {
245	CodeGenFunction(const CodeGenFunction &) = delete;
246	void operator=(const CodeGenFunction &) = delete;
247
248	friend class CGCXXABI;
249
250	public:
251	/// A jump destination is an abstract label, branching to which may
252	/// require a jump out through normal cleanups.
253	struct JumpDest {
254	JumpDest() : Block(nullptr), Index(0) {}
255	JumpDest(llvm::BasicBlock *Block, EHScopeStack::stable_iterator Depth,
256	unsigned Index)
257	: Block(Block), ScopeDepth(Depth), Index(Index) {}
258
259	bool isValid() const { return Block != nullptr; }
260	llvm::BasicBlock *getBlock() const { return Block; }
261	EHScopeStack::stable_iterator getScopeDepth() const { return ScopeDepth; }
262	unsigned getDestIndex() const { return Index; }
263
264	// This should be used cautiously.
265	void setScopeDepth(EHScopeStack::stable_iterator depth) {
266	ScopeDepth = depth;
267	}
268
269	private:
270	llvm::BasicBlock *Block;
271	EHScopeStack::stable_iterator ScopeDepth;
272	unsigned Index;
273	};
274
275	CodeGenModule &CGM; // Per-module state.
276	const TargetInfo &Target;
277
278	// For EH/SEH outlined funclets, this field points to parent's CGF
279	CodeGenFunction *ParentCGF = nullptr;
280
281	typedef std::pair<llvm::Value *, llvm::Value *> ComplexPairTy;
282	LoopInfoStack LoopStack;
283	CGBuilderTy Builder;
284
285	// Stores variables for which we can't generate correct lifetime markers
286	// because of jumps.
287	VarBypassDetector Bypasses;
288
289	/// List of recently emitted OMPCanonicalLoops.
290	///
291	/// Since OMPCanonicalLoops are nested inside other statements (in particular
292	/// CapturedStmt generated by OMPExecutableDirective and non-perfectly nested
293	/// loops), we cannot directly call OMPEmitOMPCanonicalLoop and receive its
294	/// llvm::CanonicalLoopInfo. Instead, we call EmitStmt and any
295	/// OMPEmitOMPCanonicalLoop called by it will add its CanonicalLoopInfo to
296	/// this stack when done. Entering a new loop requires clearing this list; it
297	/// either means we start parsing a new loop nest (in which case the previous
298	/// loop nest goes out of scope) or a second loop in the same level in which
299	/// case it would be ambiguous into which of the two (or more) loops the loop
300	/// nest would extend.
301	SmallVector<llvm::CanonicalLoopInfo *, 4> OMPLoopNestStack;
302
303	/// Stack to track the Logical Operator recursion nest for MC/DC.
304	SmallVector<const BinaryOperator *, 16> MCDCLogOpStack;
305
306	/// Stack to track the controlled convergence tokens.
307	SmallVector<llvm::ConvergenceControlInst *, 4> ConvergenceTokenStack;
308
309	/// Number of nested loop to be consumed by the last surrounding
310	/// loop-associated directive.
311	int ExpectedOMPLoopDepth = 0;
312
313	// CodeGen lambda for loops and support for ordered clause
314	typedef llvm::function_ref<void(CodeGenFunction &, const OMPLoopDirective &,
315	JumpDest)>
316	CodeGenLoopTy;
317	typedef llvm::function_ref<void(CodeGenFunction &, SourceLocation,
318	const unsigned, const bool)>
319	CodeGenOrderedTy;
320
321	// Codegen lambda for loop bounds in worksharing loop constructs
322	typedef llvm::function_ref<std::pair<LValue, LValue>(
323	CodeGenFunction &, const OMPExecutableDirective &S)>
324	CodeGenLoopBoundsTy;
325
326	// Codegen lambda for loop bounds in dispatch-based loop implementation
327	typedef llvm::function_ref<std::pair<llvm::Value *, llvm::Value *>(
328	CodeGenFunction &, const OMPExecutableDirective &S, Address LB,
329	Address UB)>
330	CodeGenDispatchBoundsTy;
331
332	/// CGBuilder insert helper. This function is called after an
333	/// instruction is created using Builder.
334	void InsertHelper(llvm::Instruction *I, const llvm::Twine &Name,
335	llvm::BasicBlock::iterator InsertPt) const;
336
337	/// CurFuncDecl - Holds the Decl for the current outermost
338	/// non-closure context.
339	const Decl *CurFuncDecl = nullptr;
340	/// CurCodeDecl - This is the inner-most code context, which includes blocks.
341	const Decl *CurCodeDecl = nullptr;
342	const CGFunctionInfo *CurFnInfo = nullptr;
343	QualType FnRetTy;
344	llvm::Function *CurFn = nullptr;
345
346	/// Save Parameter Decl for coroutine.
347	llvm::SmallVector<const ParmVarDecl *, 4> FnArgs;
348
349	// Holds coroutine data if the current function is a coroutine. We use a
350	// wrapper to manage its lifetime, so that we don't have to define CGCoroData
351	// in this header.
352	struct CGCoroInfo {
353	std::unique_ptr<CGCoroData> Data;
354	bool InSuspendBlock = false;
355	CGCoroInfo();
356	~CGCoroInfo();
357	};
358	CGCoroInfo CurCoro;
359
360	bool isCoroutine() const { return CurCoro.Data != nullptr; }
361
362	bool inSuspendBlock() const {
363	return isCoroutine() && CurCoro.InSuspendBlock;
364	}
365
366	// Holds FramePtr for await_suspend wrapper generation,
367	// so that __builtin_coro_frame call can be lowered
368	// directly to value of its second argument
369	struct AwaitSuspendWrapperInfo {
370	llvm::Value *FramePtr = nullptr;
371	};
372	AwaitSuspendWrapperInfo CurAwaitSuspendWrapper;
373
374	// Generates wrapper function for `llvm.coro.await.suspend.*` intrinisics.
375	// It encapsulates SuspendExpr in a function, to separate it's body
376	// from the main coroutine to avoid miscompilations. Intrinisic
377	// is lowered to this function call in CoroSplit pass
378	// Function signature is:
379	// <type> __await_suspend_wrapper_<name>(ptr %awaiter, ptr %hdl)
380	// where type is one of (void, i1, ptr)
381	llvm::Function *generateAwaitSuspendWrapper(Twine const &CoroName,
382	Twine const &SuspendPointName,
383	CoroutineSuspendExpr const &S);
384
385	/// CurGD - The GlobalDecl for the current function being compiled.
386	GlobalDecl CurGD;
387
388	/// PrologueCleanupDepth - The cleanup depth enclosing all the
389	/// cleanups associated with the parameters.
390	EHScopeStack::stable_iterator PrologueCleanupDepth;
391
392	/// ReturnBlock - Unified return block.
393	JumpDest ReturnBlock;
394
395	/// ReturnValue - The temporary alloca to hold the return
396	/// value. This is invalid iff the function has no return value.
397	Address ReturnValue = Address::invalid();
398
399	/// ReturnValuePointer - The temporary alloca to hold a pointer to sret.
400	/// This is invalid if sret is not in use.
401	Address ReturnValuePointer = Address::invalid();
402
403	/// If a return statement is being visited, this holds the return statment's
404	/// result expression.
405	const Expr *RetExpr = nullptr;
406
407	/// Return true if a label was seen in the current scope.
408	bool hasLabelBeenSeenInCurrentScope() const {
409	if (CurLexicalScope)
410	return CurLexicalScope->hasLabels();
411	return !LabelMap.empty();
412	}
413
414	/// AllocaInsertPoint - This is an instruction in the entry block before which
415	/// we prefer to insert allocas.
416	llvm::AssertingVH<llvm::Instruction> AllocaInsertPt;
417
418	private:
419	/// PostAllocaInsertPt - This is a place in the prologue where code can be
420	/// inserted that will be dominated by all the static allocas. This helps
421	/// achieve two things:
422	/// 1. Contiguity of all static allocas (within the prologue) is maintained.
423	/// 2. All other prologue code (which are dominated by static allocas) do
424	/// appear in the source order immediately after all static allocas.
425	///
426	/// PostAllocaInsertPt will be lazily created when it is *really* required.
427	llvm::AssertingVH<llvm::Instruction> PostAllocaInsertPt = nullptr;
428
429	public:
430	/// Return PostAllocaInsertPt. If it is not yet created, then insert it
431	/// immediately after AllocaInsertPt.
432	llvm::Instruction *getPostAllocaInsertPoint() {
433	if (!PostAllocaInsertPt) {
434	assert(AllocaInsertPt &&
435	"Expected static alloca insertion point at function prologue");
436	assert(AllocaInsertPt->getParent()->isEntryBlock() &&
437	"EBB should be entry block of the current code gen function");
438	PostAllocaInsertPt = AllocaInsertPt->clone();
439	PostAllocaInsertPt->setName("postallocapt");
440	PostAllocaInsertPt->insertAfter(InsertPos: AllocaInsertPt->getIterator());
441	}
442
443	return PostAllocaInsertPt;
444	}
445
446	/// API for captured statement code generation.
447	class CGCapturedStmtInfo {
448	public:
449	explicit CGCapturedStmtInfo(CapturedRegionKind K = CR_Default)
450	: Kind(K), ThisValue(nullptr), CXXThisFieldDecl(nullptr) {}
451	explicit CGCapturedStmtInfo(const CapturedStmt &S,
452	CapturedRegionKind K = CR_Default)
453	: Kind(K), ThisValue(nullptr), CXXThisFieldDecl(nullptr) {
454
455	RecordDecl::field_iterator Field =
456	S.getCapturedRecordDecl()->field_begin();
457	for (CapturedStmt::const_capture_iterator I = S.capture_begin(),
458	E = S.capture_end();
459	I != E; ++I, ++Field) {
460	if (I->capturesThis())
461	CXXThisFieldDecl = *Field;
462	else if (I->capturesVariable())
463	CaptureFields[I->getCapturedVar()->getCanonicalDecl()] = *Field;
464	else if (I->capturesVariableByCopy())
465	CaptureFields[I->getCapturedVar()->getCanonicalDecl()] = *Field;
466	}
467	}
468
469	virtual ~CGCapturedStmtInfo();
470
471	CapturedRegionKind getKind() const { return Kind; }
472
473	virtual void setContextValue(llvm::Value *V) { ThisValue = V; }
474	// Retrieve the value of the context parameter.
475	virtual llvm::Value *getContextValue() const { return ThisValue; }
476
477	/// Lookup the captured field decl for a variable.
478	virtual const FieldDecl *lookup(const VarDecl *VD) const {
479	return CaptureFields.lookup(Val: VD->getCanonicalDecl());
480	}
481
482	bool isCXXThisExprCaptured() const { return getThisFieldDecl() != nullptr; }
483	virtual FieldDecl *getThisFieldDecl() const { return CXXThisFieldDecl; }
484
485	static bool classof(const CGCapturedStmtInfo *) { return true; }
486
487	/// Emit the captured statement body.
488	virtual void EmitBody(CodeGenFunction &CGF, const Stmt *S) {
489	CGF.incrementProfileCounter(S);
490	CGF.EmitStmt(S);
491	}
492
493	/// Get the name of the capture helper.
494	virtual StringRef getHelperName() const { return "__captured_stmt"; }
495
496	/// Get the CaptureFields
497	llvm::SmallDenseMap<const VarDecl *, FieldDecl *> getCaptureFields() {
498	return CaptureFields;
499	}
500
501	private:
502	/// The kind of captured statement being generated.
503	CapturedRegionKind Kind;
504
505	/// Keep the map between VarDecl and FieldDecl.
506	llvm::SmallDenseMap<const VarDecl *, FieldDecl *> CaptureFields;
507
508	/// The base address of the captured record, passed in as the first
509	/// argument of the parallel region function.
510	llvm::Value *ThisValue;
511
512	/// Captured 'this' type.
513	FieldDecl *CXXThisFieldDecl;
514	};
515	CGCapturedStmtInfo *CapturedStmtInfo = nullptr;
516
517	/// RAII for correct setting/restoring of CapturedStmtInfo.
518	class CGCapturedStmtRAII {
519	private:
520	CodeGenFunction &CGF;
521	CGCapturedStmtInfo *PrevCapturedStmtInfo;
522
523	public:
524	CGCapturedStmtRAII(CodeGenFunction &CGF,
525	CGCapturedStmtInfo *NewCapturedStmtInfo)
526	: CGF(CGF), PrevCapturedStmtInfo(CGF.CapturedStmtInfo) {
527	CGF.CapturedStmtInfo = NewCapturedStmtInfo;
528	}
529	~CGCapturedStmtRAII() { CGF.CapturedStmtInfo = PrevCapturedStmtInfo; }
530	};
531
532	/// An abstract representation of regular/ObjC call/message targets.
533	class AbstractCallee {
534	/// The function declaration of the callee.
535	const Decl *CalleeDecl;
536
537	public:
538	AbstractCallee() : CalleeDecl(nullptr) {}
539	AbstractCallee(const FunctionDecl *FD) : CalleeDecl(FD) {}
540	AbstractCallee(const ObjCMethodDecl *OMD) : CalleeDecl(OMD) {}
541	bool hasFunctionDecl() const {
542	return isa_and_nonnull<FunctionDecl>(Val: CalleeDecl);
543	}
544	const Decl *getDecl() const { return CalleeDecl; }
545	unsigned getNumParams() const {
546	if (const auto *FD = dyn_cast<FunctionDecl>(Val: CalleeDecl))
547	return FD->getNumParams();
548	return cast<ObjCMethodDecl>(Val: CalleeDecl)->param_size();
549	}
550	const ParmVarDecl *getParamDecl(unsigned I) const {
551	if (const auto *FD = dyn_cast<FunctionDecl>(Val: CalleeDecl))
552	return FD->getParamDecl(i: I);
553	return *(cast<ObjCMethodDecl>(Val: CalleeDecl)->param_begin() + I);
554	}
555	};
556
557	/// Sanitizers enabled for this function.
558	SanitizerSet SanOpts;
559
560	/// True if CodeGen currently emits code implementing sanitizer checks.
561	bool IsSanitizerScope = false;
562
563	/// RAII object to set/unset CodeGenFunction::IsSanitizerScope.
564	class SanitizerScope {
565	CodeGenFunction *CGF;
566
567	public:
568	SanitizerScope(CodeGenFunction *CGF);
569	~SanitizerScope();
570	};
571
572	/// In C++, whether we are code generating a thunk. This controls whether we
573	/// should emit cleanups.
574	bool CurFuncIsThunk = false;
575
576	/// In ARC, whether we should autorelease the return value.
577	bool AutoreleaseResult = false;
578
579	/// Whether we processed a Microsoft-style asm block during CodeGen. These can
580	/// potentially set the return value.
581	bool SawAsmBlock = false;
582
583	GlobalDecl CurSEHParent;
584
585	/// True if the current function is an outlined SEH helper. This can be a
586	/// finally block or filter expression.
587	bool IsOutlinedSEHHelper = false;
588
589	/// True if CodeGen currently emits code inside presereved access index
590	/// region.
591	bool IsInPreservedAIRegion = false;
592
593	/// True if the current statement has nomerge attribute.
594	bool InNoMergeAttributedStmt = false;
595
596	/// True if the current statement has noinline attribute.
597	bool InNoInlineAttributedStmt = false;
598
599	/// True if the current statement has always_inline attribute.
600	bool InAlwaysInlineAttributedStmt = false;
601
602	/// True if the current statement has noconvergent attribute.
603	bool InNoConvergentAttributedStmt = false;
604
605	/// HLSL Branch attribute.
606	HLSLControlFlowHintAttr::Spelling HLSLControlFlowAttr =
607	HLSLControlFlowHintAttr::SpellingNotCalculated;
608
609	// The CallExpr within the current statement that the musttail attribute
610	// applies to. nullptr if there is no 'musttail' on the current statement.
611	const CallExpr *MustTailCall = nullptr;
612
613	/// Returns true if a function must make progress, which means the
614	/// mustprogress attribute can be added.
615	bool checkIfFunctionMustProgress() {
616	if (CGM.getCodeGenOpts().getFiniteLoops() ==
617	CodeGenOptions::FiniteLoopsKind::Never)
618	return false;
619
620	// C++11 and later guarantees that a thread eventually will do one of the
621	// following (C++11 [intro.multithread]p24 and C++17 [intro.progress]p1):
622	// - terminate,
623	// - make a call to a library I/O function,
624	// - perform an access through a volatile glvalue, or
625	// - perform a synchronization operation or an atomic operation.
626	//
627	// Hence each function is 'mustprogress' in C++11 or later.
628	return getLangOpts().CPlusPlus11;
629	}
630
631	/// Returns true if a loop must make progress, which means the mustprogress
632	/// attribute can be added. \p HasConstantCond indicates whether the branch
633	/// condition is a known constant.
634	bool checkIfLoopMustProgress(const Expr *, bool HasEmptyBody);
635
636	const CodeGen::CGBlockInfo *BlockInfo = nullptr;
637	llvm::Value *BlockPointer = nullptr;
638
639	llvm::DenseMap<const ValueDecl *, FieldDecl *> LambdaCaptureFields;
640	FieldDecl *LambdaThisCaptureField = nullptr;
641
642	/// A mapping from NRVO variables to the flags used to indicate
643	/// when the NRVO has been applied to this variable.
644	llvm::DenseMap<const VarDecl *, llvm::Value *> NRVOFlags;
645
646	EHScopeStack EHStack;
647	llvm::SmallVector<char, 256> LifetimeExtendedCleanupStack;
648
649	// A stack of cleanups which were added to EHStack but have to be deactivated
650	// later before being popped or emitted. These are usually deactivated on
651	// exiting a `CleanupDeactivationScope` scope. For instance, after a
652	// full-expr.
653	//
654	// These are specially useful for correctly emitting cleanups while
655	// encountering branches out of expression (through stmt-expr or coroutine
656	// suspensions).
657	struct DeferredDeactivateCleanup {
658	EHScopeStack::stable_iterator Cleanup;
659	llvm::Instruction *DominatingIP;
660	};
661	llvm::SmallVector<DeferredDeactivateCleanup> DeferredDeactivationCleanupStack;
662
663	// Enters a new scope for capturing cleanups which are deferred to be
664	// deactivated, all of which will be deactivated once the scope is exited.
665	struct CleanupDeactivationScope {
666	CodeGenFunction &CGF;
667	size_t OldDeactivateCleanupStackSize;
668	bool Deactivated;
669	CleanupDeactivationScope(CodeGenFunction &CGF)
670	: CGF(CGF), OldDeactivateCleanupStackSize(
671	CGF.DeferredDeactivationCleanupStack.size()),
672	Deactivated(false) {}
673
674	void ForceDeactivate() {
675	assert(!Deactivated && "Deactivating already deactivated scope");
676	auto &Stack = CGF.DeferredDeactivationCleanupStack;
677	for (size_t I = Stack.size(); I > OldDeactivateCleanupStackSize; I--) {
678	CGF.DeactivateCleanupBlock(Cleanup: Stack[I - 1].Cleanup,
679	DominatingIP: Stack[I - 1].DominatingIP);
680	Stack[I - 1].DominatingIP->eraseFromParent();
681	}
682	Stack.resize(N: OldDeactivateCleanupStackSize);
683	Deactivated = true;
684	}
685
686	~CleanupDeactivationScope() {
687	if (Deactivated)
688	return;
689	ForceDeactivate();
690	}
691	};
692
693	llvm::SmallVector<const JumpDest *, 2> SEHTryEpilogueStack;
694
695	llvm::Instruction *CurrentFuncletPad = nullptr;
696
697	class CallLifetimeEnd final : public EHScopeStack::Cleanup {
698	bool isRedundantBeforeReturn() override { return true; }
699
700	llvm::Value *Addr;
701	llvm::Value *Size;
702
703	public:
704	CallLifetimeEnd(RawAddress addr, llvm::Value *size)
705	: Addr(addr.getPointer()), Size(size) {}
706
707	void Emit(CodeGenFunction &CGF, Flags flags) override {
708	CGF.EmitLifetimeEnd(Size, Addr);
709	}
710	};
711
712	// We are using objects of this 'cleanup' class to emit fake.use calls
713	// for -fextend-variable-liveness. They are placed at the end of a variable's
714	// scope analogous to lifetime markers.
715	class FakeUse final : public EHScopeStack::Cleanup {
716	Address Addr;
717
718	public:
719	FakeUse(Address addr) : Addr(addr) {}
720
721	void Emit(CodeGenFunction &CGF, Flags flags) override {
722	CGF.EmitFakeUse(Addr);
723	}
724	};
725
726	/// Header for data within LifetimeExtendedCleanupStack.
727	struct LifetimeExtendedCleanupHeader {
728	/// The size of the following cleanup object.
729	unsigned Size;
730	/// The kind of cleanup to push.
731	LLVM_PREFERRED_TYPE(CleanupKind)
732	unsigned Kind : 31;
733	/// Whether this is a conditional cleanup.
734	LLVM_PREFERRED_TYPE(bool)
735	unsigned IsConditional : 1;
736
737	size_t getSize() const { return Size; }
738	CleanupKind getKind() const { return (CleanupKind)Kind; }
739	bool isConditional() const { return IsConditional; }
740	};
741
742	/// i32s containing the indexes of the cleanup destinations.
743	RawAddress NormalCleanupDest = RawAddress::invalid();
744
745	unsigned NextCleanupDestIndex = 1;
746
747	/// EHResumeBlock - Unified block containing a call to llvm.eh.resume.
748	llvm::BasicBlock *EHResumeBlock = nullptr;
749
750	/// The exception slot. All landing pads write the current exception pointer
751	/// into this alloca.
752	llvm::Value *ExceptionSlot = nullptr;
753
754	/// The selector slot. Under the MandatoryCleanup model, all landing pads
755	/// write the current selector value into this alloca.
756	llvm::AllocaInst *EHSelectorSlot = nullptr;
757
758	/// A stack of exception code slots. Entering an __except block pushes a slot
759	/// on the stack and leaving pops one. The __exception_code() intrinsic loads
760	/// a value from the top of the stack.
761	SmallVector<Address, 1> SEHCodeSlotStack;
762
763	/// Value returned by __exception_info intrinsic.
764	llvm::Value *SEHInfo = nullptr;
765
766	/// Emits a landing pad for the current EH stack.
767	llvm::BasicBlock *EmitLandingPad();
768
769	llvm::BasicBlock *getInvokeDestImpl();
770
771	/// Parent loop-based directive for scan directive.
772	const OMPExecutableDirective *OMPParentLoopDirectiveForScan = nullptr;
773	llvm::BasicBlock *OMPBeforeScanBlock = nullptr;
774	llvm::BasicBlock *OMPAfterScanBlock = nullptr;
775	llvm::BasicBlock *OMPScanExitBlock = nullptr;
776	llvm::BasicBlock *OMPScanDispatch = nullptr;
777	bool OMPFirstScanLoop = false;
778
779	/// Manages parent directive for scan directives.
780	class ParentLoopDirectiveForScanRegion {
781	CodeGenFunction &CGF;
782	const OMPExecutableDirective *ParentLoopDirectiveForScan;
783
784	public:
785	ParentLoopDirectiveForScanRegion(
786	CodeGenFunction &CGF,
787	const OMPExecutableDirective &ParentLoopDirectiveForScan)
788	: CGF(CGF),
789	ParentLoopDirectiveForScan(CGF.OMPParentLoopDirectiveForScan) {
790	CGF.OMPParentLoopDirectiveForScan = &ParentLoopDirectiveForScan;
791	}
792	~ParentLoopDirectiveForScanRegion() {
793	CGF.OMPParentLoopDirectiveForScan = ParentLoopDirectiveForScan;
794	}
795	};
796
797	template <class T>
798	typename DominatingValue<T>::saved_type saveValueInCond(T value) {
799	return DominatingValue<T>::save(*this, value);
800	}
801
802	class CGFPOptionsRAII {
803	public:
804	CGFPOptionsRAII(CodeGenFunction &CGF, FPOptions FPFeatures);
805	CGFPOptionsRAII(CodeGenFunction &CGF, const Expr *E);
806	~CGFPOptionsRAII();
807
808	private:
809	void ConstructorHelper(FPOptions FPFeatures);
810	CodeGenFunction &CGF;
811	FPOptions OldFPFeatures;
812	llvm::fp::ExceptionBehavior OldExcept;
813	llvm::RoundingMode OldRounding;
814	std::optional<CGBuilderTy::FastMathFlagGuard> FMFGuard;
815	};
816	FPOptions CurFPFeatures;
817
818	class CGAtomicOptionsRAII {
819	public:
820	CGAtomicOptionsRAII(CodeGenModule &CGM_, AtomicOptions AO)
821	: CGM(CGM_), SavedAtomicOpts(CGM.getAtomicOpts()) {
822	CGM.setAtomicOpts(AO);
823	}
824	CGAtomicOptionsRAII(CodeGenModule &CGM_, const AtomicAttr *AA)
825	: CGM(CGM_), SavedAtomicOpts(CGM.getAtomicOpts()) {
826	if (!AA)
827	return;
828	AtomicOptions AO = SavedAtomicOpts;
829	for (auto Option : AA->atomicOptions()) {
830	switch (Option) {
831	case AtomicAttr::remote_memory:
832	AO.remote_memory = true;
833	break;
834	case AtomicAttr::no_remote_memory:
835	AO.remote_memory = false;
836	break;
837	case AtomicAttr::fine_grained_memory:
838	AO.fine_grained_memory = true;
839	break;
840	case AtomicAttr::no_fine_grained_memory:
841	AO.fine_grained_memory = false;
842	break;
843	case AtomicAttr::ignore_denormal_mode:
844	AO.ignore_denormal_mode = true;
845	break;
846	case AtomicAttr::no_ignore_denormal_mode:
847	AO.ignore_denormal_mode = false;
848	break;
849	}
850	}
851	CGM.setAtomicOpts(AO);
852	}
853
854	CGAtomicOptionsRAII(const CGAtomicOptionsRAII &) = delete;
855	CGAtomicOptionsRAII &operator=(const CGAtomicOptionsRAII &) = delete;
856	~CGAtomicOptionsRAII() { CGM.setAtomicOpts(SavedAtomicOpts); }
857
858	private:
859	CodeGenModule &CGM;
860	AtomicOptions SavedAtomicOpts;
861	};
862
863	public:
864	/// ObjCEHValueStack - Stack of Objective-C exception values, used for
865	/// rethrows.
866	SmallVector<llvm::Value *, 8> ObjCEHValueStack;
867
868	/// A class controlling the emission of a finally block.
869	class FinallyInfo {
870	/// Where the catchall's edge through the cleanup should go.
871	JumpDest RethrowDest;
872
873	/// A function to call to enter the catch.
874	llvm::FunctionCallee BeginCatchFn;
875
876	/// An i1 variable indicating whether or not the @finally is
877	/// running for an exception.
878	llvm::AllocaInst *ForEHVar = nullptr;
879
880	/// An i8* variable into which the exception pointer to rethrow
881	/// has been saved.
882	llvm::AllocaInst *SavedExnVar = nullptr;
883
884	public:
885	void enter(CodeGenFunction &CGF, const Stmt *Finally,
886	llvm::FunctionCallee beginCatchFn,
887	llvm::FunctionCallee endCatchFn, llvm::FunctionCallee rethrowFn);
888	void exit(CodeGenFunction &CGF);
889	};
890
891	/// Returns true inside SEH __try blocks.
892	bool isSEHTryScope() const { return !SEHTryEpilogueStack.empty(); }
893
894	/// Returns true while emitting a cleanuppad.
895	bool isCleanupPadScope() const {
896	return CurrentFuncletPad && isa<llvm::CleanupPadInst>(Val: CurrentFuncletPad);
897	}
898
899	/// pushFullExprCleanup - Push a cleanup to be run at the end of the
900	/// current full-expression. Safe against the possibility that
901	/// we're currently inside a conditionally-evaluated expression.
902	template <class T, class... As>
903	void pushFullExprCleanup(CleanupKind kind, As... A) {
904	// If we're not in a conditional branch, or if none of the
905	// arguments requires saving, then use the unconditional cleanup.
906	if (!isInConditionalBranch())
907	return EHStack.pushCleanup<T>(kind, A...);
908
909	// Stash values in a tuple so we can guarantee the order of saves.
910	typedef std::tuple<typename DominatingValue<As>::saved_type...> SavedTuple;
911	SavedTuple Saved{saveValueInCond(A)...};
912
913	typedef EHScopeStack::ConditionalCleanup<T, As...> CleanupType;
914	EHStack.pushCleanupTuple<CleanupType>(kind, Saved);
915	initFullExprCleanup();
916	}
917
918	/// Queue a cleanup to be pushed after finishing the current full-expression,
919	/// potentially with an active flag.
920	template <class T, class... As>
921	void pushCleanupAfterFullExpr(CleanupKind Kind, As... A) {
922	if (!isInConditionalBranch())
923	return pushCleanupAfterFullExprWithActiveFlag<T>(
924	Kind, RawAddress::invalid(), A...);
925
926	RawAddress ActiveFlag = createCleanupActiveFlag();
927	assert(!DominatingValue<Address>::needsSaving(ActiveFlag) &&
928	"cleanup active flag should never need saving");
929
930	typedef std::tuple<typename DominatingValue<As>::saved_type...> SavedTuple;
931	SavedTuple Saved{saveValueInCond(A)...};
932
933	typedef EHScopeStack::ConditionalCleanup<T, As...> CleanupType;
934	pushCleanupAfterFullExprWithActiveFlag<CleanupType>(Kind, ActiveFlag,
935	Saved);
936	}
937
938	template <class T, class... As>
939	void pushCleanupAfterFullExprWithActiveFlag(CleanupKind Kind,
940	RawAddress ActiveFlag, As... A) {
941	LifetimeExtendedCleanupHeader Header = {.Size: sizeof(T), .Kind: Kind,
942	.IsConditional: ActiveFlag.isValid()};
943
944	size_t OldSize = LifetimeExtendedCleanupStack.size();
945	LifetimeExtendedCleanupStack.resize(
946	N: LifetimeExtendedCleanupStack.size() + sizeof(Header) + Header.Size +
947	(Header.IsConditional ? sizeof(ActiveFlag) : 0));
948
949	static_assert(sizeof(Header) % alignof(T) == 0,
950	"Cleanup will be allocated on misaligned address");
951	char *Buffer = &LifetimeExtendedCleanupStack[OldSize];
952	new (Buffer) LifetimeExtendedCleanupHeader(Header);
953	new (Buffer + sizeof(Header)) T(A...);
954	if (Header.IsConditional)
955	new (Buffer + sizeof(Header) + sizeof(T)) RawAddress(ActiveFlag);
956	}
957
958	// Push a cleanup onto EHStack and deactivate it later. It is usually
959	// deactivated when exiting a `CleanupDeactivationScope` (for example: after a
960	// full expression).
961	template <class T, class... As>
962	void pushCleanupAndDeferDeactivation(CleanupKind Kind, As... A) {
963	// Placeholder dominating IP for this cleanup.
964	llvm::Instruction *DominatingIP =
965	Builder.CreateFlagLoad(Addr: llvm::Constant::getNullValue(Ty: Int8PtrTy));
966	EHStack.pushCleanup<T>(Kind, A...);
967	DeferredDeactivationCleanupStack.push_back(
968	Elt: {.Cleanup: EHStack.stable_begin(), .DominatingIP: DominatingIP});
969	}
970
971	/// Set up the last cleanup that was pushed as a conditional
972	/// full-expression cleanup.
973	void initFullExprCleanup() {
974	initFullExprCleanupWithFlag(ActiveFlag: createCleanupActiveFlag());
975	}
976
977	void initFullExprCleanupWithFlag(RawAddress ActiveFlag);
978	RawAddress createCleanupActiveFlag();
979
980	/// PushDestructorCleanup - Push a cleanup to call the
981	/// complete-object destructor of an object of the given type at the
982	/// given address. Does nothing if T is not a C++ class type with a
983	/// non-trivial destructor.
984	void PushDestructorCleanup(QualType T, Address Addr);
985
986	/// PushDestructorCleanup - Push a cleanup to call the
987	/// complete-object variant of the given destructor on the object at
988	/// the given address.
989	void PushDestructorCleanup(const CXXDestructorDecl *Dtor, QualType T,
990	Address Addr);
991
992	/// PopCleanupBlock - Will pop the cleanup entry on the stack and
993	/// process all branch fixups.
994	void PopCleanupBlock(bool FallThroughIsBranchThrough = false,
995	bool ForDeactivation = false);
996
997	/// DeactivateCleanupBlock - Deactivates the given cleanup block.
998	/// The block cannot be reactivated. Pops it if it's the top of the
999	/// stack.
1000	///
1001	/// \param DominatingIP - An instruction which is known to
1002	/// dominate the current IP (if set) and which lies along
1003	/// all paths of execution between the current IP and the
1004	/// the point at which the cleanup comes into scope.
1005	void DeactivateCleanupBlock(EHScopeStack::stable_iterator Cleanup,
1006	llvm::Instruction *DominatingIP);
1007
1008	/// ActivateCleanupBlock - Activates an initially-inactive cleanup.
1009	/// Cannot be used to resurrect a deactivated cleanup.
1010	///
1011	/// \param DominatingIP - An instruction which is known to
1012	/// dominate the current IP (if set) and which lies along
1013	/// all paths of execution between the current IP and the
1014	/// the point at which the cleanup comes into scope.
1015	void ActivateCleanupBlock(EHScopeStack::stable_iterator Cleanup,
1016	llvm::Instruction *DominatingIP);
1017
1018	/// Enters a new scope for capturing cleanups, all of which
1019	/// will be executed once the scope is exited.
1020	class RunCleanupsScope {
1021	EHScopeStack::stable_iterator CleanupStackDepth, OldCleanupScopeDepth;
1022	size_t LifetimeExtendedCleanupStackSize;
1023	CleanupDeactivationScope DeactivateCleanups;
1024	bool OldDidCallStackSave;
1025
1026	protected:
1027	bool PerformCleanup;
1028
1029	private:
1030	RunCleanupsScope(const RunCleanupsScope &) = delete;
1031	void operator=(const RunCleanupsScope &) = delete;
1032
1033	protected:
1034	CodeGenFunction &CGF;
1035
1036	public:
1037	/// Enter a new cleanup scope.
1038	explicit RunCleanupsScope(CodeGenFunction &CGF)
1039	: DeactivateCleanups(CGF), PerformCleanup(true), CGF(CGF) {
1040	CleanupStackDepth = CGF.EHStack.stable_begin();
1041	LifetimeExtendedCleanupStackSize =
1042	CGF.LifetimeExtendedCleanupStack.size();
1043	OldDidCallStackSave = CGF.DidCallStackSave;
1044	CGF.DidCallStackSave = false;
1045	OldCleanupScopeDepth = CGF.CurrentCleanupScopeDepth;
1046	CGF.CurrentCleanupScopeDepth = CleanupStackDepth;
1047	}
1048
1049	/// Exit this cleanup scope, emitting any accumulated cleanups.
1050	~RunCleanupsScope() {
1051	if (PerformCleanup)
1052	ForceCleanup();
1053	}
1054
1055	/// Determine whether this scope requires any cleanups.
1056	bool requiresCleanups() const {
1057	return CGF.EHStack.stable_begin() != CleanupStackDepth;
1058	}
1059
1060	/// Force the emission of cleanups now, instead of waiting
1061	/// until this object is destroyed.
1062	/// \param ValuesToReload - A list of values that need to be available at
1063	/// the insertion point after cleanup emission. If cleanup emission created
1064	/// a shared cleanup block, these value pointers will be rewritten.
1065	/// Otherwise, they not will be modified.
1066	void
1067	ForceCleanup(std::initializer_list<llvm::Value **> ValuesToReload = {}) {
1068	assert(PerformCleanup && "Already forced cleanup");
1069	CGF.DidCallStackSave = OldDidCallStackSave;
1070	DeactivateCleanups.ForceDeactivate();
1071	CGF.PopCleanupBlocks(OldCleanupStackSize: CleanupStackDepth, OldLifetimeExtendedStackSize: LifetimeExtendedCleanupStackSize,
1072	ValuesToReload);
1073	PerformCleanup = false;
1074	CGF.CurrentCleanupScopeDepth = OldCleanupScopeDepth;
1075	}
1076	};
1077
1078	// Cleanup stack depth of the RunCleanupsScope that was pushed most recently.
1079	EHScopeStack::stable_iterator CurrentCleanupScopeDepth =
1080	EHScopeStack::stable_end();
1081
1082	class LexicalScope : public RunCleanupsScope {
1083	SourceRange Range;
1084	SmallVector<const LabelDecl *, 4> Labels;
1085	LexicalScope *ParentScope;
1086
1087	LexicalScope(const LexicalScope &) = delete;
1088	void operator=(const LexicalScope &) = delete;
1089
1090	public:
1091	/// Enter a new cleanup scope.
1092	explicit LexicalScope(CodeGenFunction &CGF, SourceRange Range);
1093
1094	void addLabel(const LabelDecl *label) {
1095	assert(PerformCleanup && "adding label to dead scope?");
1096	Labels.push_back(Elt: label);
1097	}
1098
1099	/// Exit this cleanup scope, emitting any accumulated
1100	/// cleanups.
1101	~LexicalScope();
1102
1103	/// Force the emission of cleanups now, instead of waiting
1104	/// until this object is destroyed.
1105	void ForceCleanup() {
1106	CGF.CurLexicalScope = ParentScope;
1107	RunCleanupsScope::ForceCleanup();
1108
1109	if (!Labels.empty())
1110	rescopeLabels();
1111	}
1112
1113	bool hasLabels() const { return !Labels.empty(); }
1114
1115	void rescopeLabels();
1116	};
1117
1118	typedef llvm::DenseMap<const Decl *, Address> DeclMapTy;
1119
1120	/// The class used to assign some variables some temporarily addresses.
1121	class OMPMapVars {
1122	DeclMapTy SavedLocals;
1123	DeclMapTy SavedTempAddresses;
1124	OMPMapVars(const OMPMapVars &) = delete;
1125	void operator=(const OMPMapVars &) = delete;
1126
1127	public:
1128	explicit OMPMapVars() = default;
1129	~OMPMapVars() {
1130	assert(SavedLocals.empty() && "Did not restored original addresses.");
1131	};
1132
1133	/// Sets the address of the variable \p LocalVD to be \p TempAddr in
1134	/// function \p CGF.
1135	/// \return true if at least one variable was set already, false otherwise.
1136	bool setVarAddr(CodeGenFunction &CGF, const VarDecl *LocalVD,
1137	Address TempAddr) {
1138	LocalVD = LocalVD->getCanonicalDecl();
1139	// Only save it once.
1140	if (SavedLocals.count(LocalVD))
1141	return false;
1142
1143	// Copy the existing local entry to SavedLocals.
1144	auto it = CGF.LocalDeclMap.find(LocalVD);
1145	if (it != CGF.LocalDeclMap.end())
1146	SavedLocals.try_emplace(LocalVD, it->second);
1147	else
1148	SavedLocals.try_emplace(LocalVD, Address::invalid());
1149
1150	// Generate the private entry.
1151	QualType VarTy = LocalVD->getType();
1152	if (VarTy->isReferenceType()) {
1153	Address Temp = CGF.CreateMemTemp(T: VarTy);
1154	CGF.Builder.CreateStore(Val: TempAddr.emitRawPointer(CGF), Addr: Temp);
1155	TempAddr = Temp;
1156	}
1157	SavedTempAddresses.try_emplace(LocalVD, TempAddr);
1158
1159	return true;
1160	}
1161
1162	/// Applies new addresses to the list of the variables.
1163	/// \return true if at least one variable is using new address, false
1164	/// otherwise.
1165	bool apply(CodeGenFunction &CGF) {
1166	copyInto(Src: SavedTempAddresses, Dest&: CGF.LocalDeclMap);
1167	SavedTempAddresses.clear();
1168	return !SavedLocals.empty();
1169	}
1170
1171	/// Restores original addresses of the variables.
1172	void restore(CodeGenFunction &CGF) {
1173	if (!SavedLocals.empty()) {
1174	copyInto(Src: SavedLocals, Dest&: CGF.LocalDeclMap);
1175	SavedLocals.clear();
1176	}
1177	}
1178
1179	private:
1180	/// Copy all the entries in the source map over the corresponding
1181	/// entries in the destination, which must exist.
1182	static void copyInto(const DeclMapTy &Src, DeclMapTy &Dest) {
1183	for (auto &[Decl, Addr] : Src) {
1184	if (!Addr.isValid())
1185	Dest.erase(Val: Decl);
1186	else
1187	Dest.insert_or_assign(Key: Decl, Val: Addr);
1188	}
1189	}
1190	};
1191
1192	/// The scope used to remap some variables as private in the OpenMP loop body
1193	/// (or other captured region emitted without outlining), and to restore old
1194	/// vars back on exit.
1195	class OMPPrivateScope : public RunCleanupsScope {
1196	OMPMapVars MappedVars;
1197	OMPPrivateScope(const OMPPrivateScope &) = delete;
1198	void operator=(const OMPPrivateScope &) = delete;
1199
1200	public:
1201	/// Enter a new OpenMP private scope.
1202	explicit OMPPrivateScope(CodeGenFunction &CGF) : RunCleanupsScope(CGF) {}
1203
1204	/// Registers \p LocalVD variable as a private with \p Addr as the address
1205	/// of the corresponding private variable. \p
1206	/// PrivateGen is the address of the generated private variable.
1207	/// \return true if the variable is registered as private, false if it has
1208	/// been privatized already.
1209	bool addPrivate(const VarDecl *LocalVD, Address Addr) {
1210	assert(PerformCleanup && "adding private to dead scope");
1211	return MappedVars.setVarAddr(CGF, LocalVD, TempAddr: Addr);
1212	}
1213
1214	/// Privatizes local variables previously registered as private.
1215	/// Registration is separate from the actual privatization to allow
1216	/// initializers use values of the original variables, not the private one.
1217	/// This is important, for example, if the private variable is a class
1218	/// variable initialized by a constructor that references other private
1219	/// variables. But at initialization original variables must be used, not
1220	/// private copies.
1221	/// \return true if at least one variable was privatized, false otherwise.
1222	bool Privatize() { return MappedVars.apply(CGF); }
1223
1224	void ForceCleanup() {
1225	RunCleanupsScope::ForceCleanup();
1226	restoreMap();
1227	}
1228
1229	/// Exit scope - all the mapped variables are restored.
1230	~OMPPrivateScope() {
1231	if (PerformCleanup)
1232	ForceCleanup();
1233	}
1234
1235	/// Checks if the global variable is captured in current function.
1236	bool isGlobalVarCaptured(const VarDecl *VD) const {
1237	VD = VD->getCanonicalDecl();
1238	return !VD->isLocalVarDeclOrParm() && CGF.LocalDeclMap.count(VD) > 0;
1239	}
1240
1241	/// Restore all mapped variables w/o clean up. This is usefully when we want
1242	/// to reference the original variables but don't want the clean up because
1243	/// that could emit lifetime end too early, causing backend issue #56913.
1244	void restoreMap() { MappedVars.restore(CGF); }
1245	};
1246
1247	/// Save/restore original map of previously emitted local vars in case when we
1248	/// need to duplicate emission of the same code several times in the same
1249	/// function for OpenMP code.
1250	class OMPLocalDeclMapRAII {
1251	CodeGenFunction &CGF;
1252	DeclMapTy SavedMap;
1253
1254	public:
1255	OMPLocalDeclMapRAII(CodeGenFunction &CGF)
1256	: CGF(CGF), SavedMap(CGF.LocalDeclMap) {}
1257	~OMPLocalDeclMapRAII() { SavedMap.swap(RHS&: CGF.LocalDeclMap); }
1258	};
1259
1260	/// Takes the old cleanup stack size and emits the cleanup blocks
1261	/// that have been added.
1262	void
1263	PopCleanupBlocks(EHScopeStack::stable_iterator OldCleanupStackSize,
1264	std::initializer_list<llvm::Value **> ValuesToReload = {});
1265
1266	/// Takes the old cleanup stack size and emits the cleanup blocks
1267	/// that have been added, then adds all lifetime-extended cleanups from
1268	/// the given position to the stack.
1269	void
1270	PopCleanupBlocks(EHScopeStack::stable_iterator OldCleanupStackSize,
1271	size_t OldLifetimeExtendedStackSize,
1272	std::initializer_list<llvm::Value **> ValuesToReload = {});
1273
1274	void ResolveBranchFixups(llvm::BasicBlock *Target);
1275
1276	/// The given basic block lies in the current EH scope, but may be a
1277	/// target of a potentially scope-crossing jump; get a stable handle
1278	/// to which we can perform this jump later.
1279	JumpDest getJumpDestInCurrentScope(llvm::BasicBlock *Target) {
1280	return JumpDest(Target, EHStack.getInnermostNormalCleanup(),
1281	NextCleanupDestIndex++);
1282	}
1283
1284	/// The given basic block lies in the current EH scope, but may be a
1285	/// target of a potentially scope-crossing jump; get a stable handle
1286	/// to which we can perform this jump later.
1287	JumpDest getJumpDestInCurrentScope(StringRef Name = StringRef()) {
1288	return getJumpDestInCurrentScope(Target: createBasicBlock(name: Name));
1289	}
1290
1291	/// EmitBranchThroughCleanup - Emit a branch from the current insert
1292	/// block through the normal cleanup handling code (if any) and then
1293	/// on to \arg Dest.
1294	void EmitBranchThroughCleanup(JumpDest Dest);
1295
1296	/// isObviouslyBranchWithoutCleanups - Return true if a branch to the
1297	/// specified destination obviously has no cleanups to run. 'false' is always
1298	/// a conservatively correct answer for this method.
1299	bool isObviouslyBranchWithoutCleanups(JumpDest Dest) const;
1300
1301	/// popCatchScope - Pops the catch scope at the top of the EHScope
1302	/// stack, emitting any required code (other than the catch handlers
1303	/// themselves).
1304	void popCatchScope();
1305
1306	llvm::BasicBlock *getEHResumeBlock(bool isCleanup);
1307	llvm::BasicBlock *getEHDispatchBlock(EHScopeStack::stable_iterator scope);
1308	llvm::BasicBlock *
1309	getFuncletEHDispatchBlock(EHScopeStack::stable_iterator scope);
1310
1311	/// An object to manage conditionally-evaluated expressions.
1312	class ConditionalEvaluation {
1313	llvm::BasicBlock *StartBB;
1314
1315	public:
1316	ConditionalEvaluation(CodeGenFunction &CGF)
1317	: StartBB(CGF.Builder.GetInsertBlock()) {}
1318
1319	void begin(CodeGenFunction &CGF) {
1320	assert(CGF.OutermostConditional != this);
1321	if (!CGF.OutermostConditional)
1322	CGF.OutermostConditional = this;
1323	}
1324
1325	void end(CodeGenFunction &CGF) {
1326	assert(CGF.OutermostConditional != nullptr);
1327	if (CGF.OutermostConditional == this)
1328	CGF.OutermostConditional = nullptr;
1329	}
1330
1331	/// Returns a block which will be executed prior to each
1332	/// evaluation of the conditional code.
1333	llvm::BasicBlock *getStartingBlock() const { return StartBB; }
1334	};
1335
1336	/// isInConditionalBranch - Return true if we're currently emitting
1337	/// one branch or the other of a conditional expression.
1338	bool isInConditionalBranch() const { return OutermostConditional != nullptr; }
1339
1340	void setBeforeOutermostConditional(llvm::Value *value, Address addr,
1341	CodeGenFunction &CGF) {
1342	assert(isInConditionalBranch());
1343	llvm::BasicBlock *block = OutermostConditional->getStartingBlock();
1344	auto store = new llvm::StoreInst(value, addr.emitRawPointer(CGF),
1345	block->back().getIterator());
1346	store->setAlignment(addr.getAlignment().getAsAlign());
1347	}
1348
1349	/// An RAII object to record that we're evaluating a statement
1350	/// expression.
1351	class StmtExprEvaluation {
1352	CodeGenFunction &CGF;
1353
1354	/// We have to save the outermost conditional: cleanups in a
1355	/// statement expression aren't conditional just because the
1356	/// StmtExpr is.
1357	ConditionalEvaluation *SavedOutermostConditional;
1358
1359	public:
1360	StmtExprEvaluation(CodeGenFunction &CGF)
1361	: CGF(CGF), SavedOutermostConditional(CGF.OutermostConditional) {
1362	CGF.OutermostConditional = nullptr;
1363	}
1364
1365	~StmtExprEvaluation() {
1366	CGF.OutermostConditional = SavedOutermostConditional;
1367	CGF.EnsureInsertPoint();
1368	}
1369	};
1370
1371	/// An object which temporarily prevents a value from being
1372	/// destroyed by aggressive peephole optimizations that assume that
1373	/// all uses of a value have been realized in the IR.
1374	class PeepholeProtection {
1375	llvm::Instruction *Inst = nullptr;
1376	friend class CodeGenFunction;
1377
1378	public:
1379	PeepholeProtection() = default;
1380	};
1381
1382	/// A non-RAII class containing all the information about a bound
1383	/// opaque value. OpaqueValueMapping, below, is a RAII wrapper for
1384	/// this which makes individual mappings very simple; using this
1385	/// class directly is useful when you have a variable number of
1386	/// opaque values or don't want the RAII functionality for some
1387	/// reason.
1388	class OpaqueValueMappingData {
1389	const OpaqueValueExpr *OpaqueValue;
1390	bool BoundLValue;
1391	CodeGenFunction::PeepholeProtection Protection;
1392
1393	OpaqueValueMappingData(const OpaqueValueExpr *ov, bool boundLValue)
1394	: OpaqueValue(ov), BoundLValue(boundLValue) {}
1395
1396	public:
1397	OpaqueValueMappingData() : OpaqueValue(nullptr) {}
1398
1399	static bool shouldBindAsLValue(const Expr *expr) {
1400	// gl-values should be bound as l-values for obvious reasons.
1401	// Records should be bound as l-values because IR generation
1402	// always keeps them in memory. Expressions of function type
1403	// act exactly like l-values but are formally required to be
1404	// r-values in C.
1405	return expr->isGLValue() || expr->getType()->isFunctionType() ||
1406	hasAggregateEvaluationKind(T: expr->getType());
1407	}
1408
1409	static OpaqueValueMappingData
1410	bind(CodeGenFunction &CGF, const OpaqueValueExpr *ov, const Expr *e) {
1411	if (shouldBindAsLValue(ov))
1412	return bind(CGF, ov, lv: CGF.EmitLValue(E: e));
1413	return bind(CGF, ov, rv: CGF.EmitAnyExpr(E: e));
1414	}
1415
1416	static OpaqueValueMappingData
1417	bind(CodeGenFunction &CGF, const OpaqueValueExpr *ov, const LValue &lv) {
1418	assert(shouldBindAsLValue(ov));
1419	CGF.OpaqueLValues.insert(KV: std::make_pair(x&: ov, y: lv));
1420	return OpaqueValueMappingData(ov, true);
1421	}
1422
1423	static OpaqueValueMappingData
1424	bind(CodeGenFunction &CGF, const OpaqueValueExpr *ov, const RValue &rv) {
1425	assert(!shouldBindAsLValue(ov));
1426	CGF.OpaqueRValues.insert(KV: std::make_pair(x&: ov, y: rv));
1427
1428	OpaqueValueMappingData data(ov, false);
1429
1430	// Work around an extremely aggressive peephole optimization in
1431	// EmitScalarConversion which assumes that all other uses of a
1432	// value are extant.
1433	data.Protection = CGF.protectFromPeepholes(rvalue: rv);
1434
1435	return data;
1436	}
1437
1438	bool isValid() const { return OpaqueValue != nullptr; }
1439	void clear() { OpaqueValue = nullptr; }
1440
1441	void unbind(CodeGenFunction &CGF) {
1442	assert(OpaqueValue && "no data to unbind!");
1443
1444	if (BoundLValue) {
1445	CGF.OpaqueLValues.erase(Val: OpaqueValue);
1446	} else {
1447	CGF.OpaqueRValues.erase(Val: OpaqueValue);
1448	CGF.unprotectFromPeepholes(protection: Protection);
1449	}
1450	}
1451	};
1452
1453	/// An RAII object to set (and then clear) a mapping for an OpaqueValueExpr.
1454	class OpaqueValueMapping {
1455	CodeGenFunction &CGF;
1456	OpaqueValueMappingData Data;
1457
1458	public:
1459	static bool shouldBindAsLValue(const Expr *expr) {
1460	return OpaqueValueMappingData::shouldBindAsLValue(expr);
1461	}
1462
1463	/// Build the opaque value mapping for the given conditional
1464	/// operator if it's the GNU ?: extension. This is a common
1465	/// enough pattern that the convenience operator is really
1466	/// helpful.
1467	///
1468	OpaqueValueMapping(CodeGenFunction &CGF,
1469	const AbstractConditionalOperator *op)
1470	: CGF(CGF) {
1471	if (isa<ConditionalOperator>(Val: op))
1472	// Leave Data empty.
1473	return;
1474
1475	const BinaryConditionalOperator *e = cast<BinaryConditionalOperator>(Val: op);
1476	Data = OpaqueValueMappingData::bind(CGF, ov: e->getOpaqueValue(),
1477	e: e->getCommon());
1478	}
1479
1480	/// Build the opaque value mapping for an OpaqueValueExpr whose source
1481	/// expression is set to the expression the OVE represents.
1482	OpaqueValueMapping(CodeGenFunction &CGF, const OpaqueValueExpr *OV)
1483	: CGF(CGF) {
1484	if (OV) {
1485	assert(OV->getSourceExpr() && "wrong form of OpaqueValueMapping used "
1486	"for OVE with no source expression");
1487	Data = OpaqueValueMappingData::bind(CGF, ov: OV, e: OV->getSourceExpr());
1488	}
1489	}
1490
1491	OpaqueValueMapping(CodeGenFunction &CGF, const OpaqueValueExpr *opaqueValue,
1492	LValue lvalue)
1493	: CGF(CGF),
1494	Data(OpaqueValueMappingData::bind(CGF, ov: opaqueValue, lv: lvalue)) {}
1495
1496	OpaqueValueMapping(CodeGenFunction &CGF, const OpaqueValueExpr *opaqueValue,
1497	RValue rvalue)
1498	: CGF(CGF),
1499	Data(OpaqueValueMappingData::bind(CGF, ov: opaqueValue, rv: rvalue)) {}
1500
1501	void pop() {
1502	Data.unbind(CGF);
1503	Data.clear();
1504	}
1505
1506	~OpaqueValueMapping() {
1507	if (Data.isValid())
1508	Data.unbind(CGF);
1509	}
1510	};
1511
1512	private:
1513	CGDebugInfo *DebugInfo;
1514	/// Used to create unique names for artificial VLA size debug info variables.
1515	unsigned VLAExprCounter = 0;
1516	bool DisableDebugInfo = false;
1517
1518	/// DidCallStackSave - Whether llvm.stacksave has been called. Used to avoid
1519	/// calling llvm.stacksave for multiple VLAs in the same scope.
1520	bool DidCallStackSave = false;
1521
1522	/// IndirectBranch - The first time an indirect goto is seen we create a block
1523	/// with an indirect branch. Every time we see the address of a label taken,
1524	/// we add the label to the indirect goto. Every subsequent indirect goto is
1525	/// codegen'd as a jump to the IndirectBranch's basic block.
1526	llvm::IndirectBrInst *IndirectBranch = nullptr;
1527
1528	/// LocalDeclMap - This keeps track of the LLVM allocas or globals for local C
1529	/// decls.
1530	DeclMapTy LocalDeclMap;
1531
1532	// Keep track of the cleanups for callee-destructed parameters pushed to the
1533	// cleanup stack so that they can be deactivated later.
1534	llvm::DenseMap<const ParmVarDecl *, EHScopeStack::stable_iterator>
1535	CalleeDestructedParamCleanups;
1536
1537	/// SizeArguments - If a ParmVarDecl had the pass_object_size attribute, this
1538	/// will contain a mapping from said ParmVarDecl to its implicit "object_size"
1539	/// parameter.
1540	llvm::SmallDenseMap<const ParmVarDecl *, const ImplicitParamDecl *, 2>
1541	SizeArguments;
1542
1543	/// Track escaped local variables with auto storage. Used during SEH
1544	/// outlining to produce a call to llvm.localescape.
1545	llvm::DenseMap<llvm::AllocaInst *, int> EscapedLocals;
1546
1547	/// LabelMap - This keeps track of the LLVM basic block for each C label.
1548	llvm::DenseMap<const LabelDecl *, JumpDest> LabelMap;
1549
1550	// BreakContinueStack - This keeps track of where break and continue
1551	// statements should jump to.
1552	struct BreakContinue {
1553	BreakContinue(JumpDest Break, JumpDest Continue)
1554	: BreakBlock(Break), ContinueBlock(Continue) {}
1555
1556	JumpDest BreakBlock;
1557	JumpDest ContinueBlock;
1558	};
1559	SmallVector<BreakContinue, 8> BreakContinueStack;
1560
1561	/// Handles cancellation exit points in OpenMP-related constructs.
1562	class OpenMPCancelExitStack {
1563	/// Tracks cancellation exit point and join point for cancel-related exit
1564	/// and normal exit.
1565	struct CancelExit {
1566	CancelExit() = default;
1567	CancelExit(OpenMPDirectiveKind Kind, JumpDest ExitBlock,
1568	JumpDest ContBlock)
1569	: Kind(Kind), ExitBlock(ExitBlock), ContBlock(ContBlock) {}
1570	OpenMPDirectiveKind Kind = llvm::omp::OMPD_unknown;
1571	/// true if the exit block has been emitted already by the special
1572	/// emitExit() call, false if the default codegen is used.
1573	bool HasBeenEmitted = false;
1574	JumpDest ExitBlock;
1575	JumpDest ContBlock;
1576	};
1577
1578	SmallVector<CancelExit, 8> Stack;
1579
1580	public:
1581	OpenMPCancelExitStack() : Stack(1) {}
1582	~OpenMPCancelExitStack() = default;
1583	/// Fetches the exit block for the current OpenMP construct.
1584	JumpDest getExitBlock() const { return Stack.back().ExitBlock; }
1585	/// Emits exit block with special codegen procedure specific for the related
1586	/// OpenMP construct + emits code for normal construct cleanup.
1587	void emitExit(CodeGenFunction &CGF, OpenMPDirectiveKind Kind,
1588	const llvm::function_ref<void(CodeGenFunction &)> CodeGen) {
1589	if (Stack.back().Kind == Kind && getExitBlock().isValid()) {
1590	assert(CGF.getOMPCancelDestination(Kind).isValid());
1591	assert(CGF.HaveInsertPoint());
1592	assert(!Stack.back().HasBeenEmitted);
1593	auto IP = CGF.Builder.saveAndClearIP();
1594	CGF.EmitBlock(BB: Stack.back().ExitBlock.getBlock());
1595	CodeGen(CGF);
1596	CGF.EmitBranch(Block: Stack.back().ContBlock.getBlock());
1597	CGF.Builder.restoreIP(IP);
1598	Stack.back().HasBeenEmitted = true;
1599	}
1600	CodeGen(CGF);
1601	}
1602	/// Enter the cancel supporting \a Kind construct.
1603	/// \param Kind OpenMP directive that supports cancel constructs.
1604	/// \param HasCancel true, if the construct has inner cancel directive,
1605	/// false otherwise.
1606	void enter(CodeGenFunction &CGF, OpenMPDirectiveKind Kind, bool HasCancel) {
1607	Stack.push_back(Elt: {Kind,
1608	HasCancel ? CGF.getJumpDestInCurrentScope(Name: "cancel.exit")
1609	: JumpDest(),
1610	HasCancel ? CGF.getJumpDestInCurrentScope(Name: "cancel.cont")
1611	: JumpDest()});
1612	}
1613	/// Emits default exit point for the cancel construct (if the special one
1614	/// has not be used) + join point for cancel/normal exits.
1615	void exit(CodeGenFunction &CGF) {
1616	if (getExitBlock().isValid()) {
1617	assert(CGF.getOMPCancelDestination(Stack.back().Kind).isValid());
1618	bool HaveIP = CGF.HaveInsertPoint();
1619	if (!Stack.back().HasBeenEmitted) {
1620	if (HaveIP)
1621	CGF.EmitBranchThroughCleanup(Dest: Stack.back().ContBlock);
1622	CGF.EmitBlock(BB: Stack.back().ExitBlock.getBlock());
1623	CGF.EmitBranchThroughCleanup(Dest: Stack.back().ContBlock);
1624	}
1625	CGF.EmitBlock(BB: Stack.back().ContBlock.getBlock());
1626	if (!HaveIP) {
1627	CGF.Builder.CreateUnreachable();
1628	CGF.Builder.ClearInsertionPoint();
1629	}
1630	}
1631	Stack.pop_back();
1632	}
1633	};
1634	OpenMPCancelExitStack OMPCancelStack;
1635
1636	/// Lower the Likelihood knowledge about the \p Cond via llvm.expect intrin.
1637	llvm::Value *emitCondLikelihoodViaExpectIntrinsic(llvm::Value *Cond,
1638	Stmt::Likelihood LH);
1639
1640	std::unique_ptr<CodeGenPGO> PGO;
1641
1642	/// Bitmap used by MC/DC to track condition outcomes of a boolean expression.
1643	Address MCDCCondBitmapAddr = Address::invalid();
1644
1645	/// Calculate branch weights appropriate for PGO data
1646	llvm::MDNode *createProfileWeights(uint64_t TrueCount,
1647	uint64_t FalseCount) const;
1648	llvm::MDNode *createProfileWeights(ArrayRef<uint64_t> Weights) const;
1649	llvm::MDNode *createProfileWeightsForLoop(const Stmt *Cond,
1650	uint64_t LoopCount) const;
1651
1652	public:
1653	std::pair<bool, bool> getIsCounterPair(const Stmt *S) const;
1654	void markStmtAsUsed(bool Skipped, const Stmt *S);
1655	void markStmtMaybeUsed(const Stmt *S);
1656
1657	/// Increment the profiler's counter for the given statement by \p StepV.
1658	/// If \p StepV is null, the default increment is 1.
1659	void incrementProfileCounter(const Stmt *S, llvm::Value *StepV = nullptr);
1660
1661	bool isMCDCCoverageEnabled() const {
1662	return (CGM.getCodeGenOpts().hasProfileClangInstr() &&
1663	CGM.getCodeGenOpts().MCDCCoverage &&
1664	!CurFn->hasFnAttribute(llvm::Attribute::NoProfile));
1665	}
1666
1667	/// Allocate a temp value on the stack that MCDC can use to track condition
1668	/// results.
1669	void maybeCreateMCDCCondBitmap();
1670
1671	bool isBinaryLogicalOp(const Expr *E) const {
1672	const BinaryOperator *BOp = dyn_cast<BinaryOperator>(Val: E->IgnoreParens());
1673	return (BOp && BOp->isLogicalOp());
1674	}
1675
1676	/// Zero-init the MCDC temp value.
1677	void maybeResetMCDCCondBitmap(const Expr *E);
1678
1679	/// Increment the profiler's counter for the given expression by \p StepV.
1680	/// If \p StepV is null, the default increment is 1.
1681	void maybeUpdateMCDCTestVectorBitmap(const Expr *E);
1682
1683	/// Update the MCDC temp value with the condition's evaluated result.
1684	void maybeUpdateMCDCCondBitmap(const Expr *E, llvm::Value *Val);
1685
1686	/// Get the profiler's count for the given statement.
1687	uint64_t getProfileCount(const Stmt *S);
1688
1689	/// Set the profiler's current count.
1690	void setCurrentProfileCount(uint64_t Count);
1691
1692	/// Get the profiler's current count. This is generally the count for the most
1693	/// recently incremented counter.
1694	uint64_t getCurrentProfileCount();
1695
1696	/// See CGDebugInfo::addInstToCurrentSourceAtom.
1697	void addInstToCurrentSourceAtom(llvm::Instruction *KeyInstruction,
1698	llvm::Value *Backup);
1699
1700	/// See CGDebugInfo::addInstToSpecificSourceAtom.
1701	void addInstToSpecificSourceAtom(llvm::Instruction *KeyInstruction,
1702	llvm::Value *Backup, uint64_t Atom);
1703
1704	/// Add \p KeyInstruction and an optional \p Backup instruction to a new atom
1705	/// group (See ApplyAtomGroup for more info).
1706	void addInstToNewSourceAtom(llvm::Instruction *KeyInstruction,
1707	llvm::Value *Backup);
1708
1709	private:
1710	/// SwitchInsn - This is nearest current switch instruction. It is null if
1711	/// current context is not in a switch.
1712	llvm::SwitchInst *SwitchInsn = nullptr;
1713	/// The branch weights of SwitchInsn when doing instrumentation based PGO.
1714	SmallVector<uint64_t, 16> *SwitchWeights = nullptr;
1715
1716	/// The likelihood attributes of the SwitchCase.
1717	SmallVector<Stmt::Likelihood, 16> *SwitchLikelihood = nullptr;
1718
1719	/// CaseRangeBlock - This block holds if condition check for last case
1720	/// statement range in current switch instruction.
1721	llvm::BasicBlock *CaseRangeBlock = nullptr;
1722
1723	/// OpaqueLValues - Keeps track of the current set of opaque value
1724	/// expressions.
1725	llvm::DenseMap<const OpaqueValueExpr *, LValue> OpaqueLValues;
1726	llvm::DenseMap<const OpaqueValueExpr *, RValue> OpaqueRValues;
1727
1728	// VLASizeMap - This keeps track of the associated size for each VLA type.
1729	// We track this by the size expression rather than the type itself because
1730	// in certain situations, like a const qualifier applied to an VLA typedef,
1731	// multiple VLA types can share the same size expression.
1732	// FIXME: Maybe this could be a stack of maps that is pushed/popped as we
1733	// enter/leave scopes.
1734	llvm::DenseMap<const Expr *, llvm::Value *> VLASizeMap;
1735
1736	/// A block containing a single 'unreachable' instruction. Created
1737	/// lazily by getUnreachableBlock().
1738	llvm::BasicBlock *UnreachableBlock = nullptr;
1739
1740	/// Counts of the number return expressions in the function.
1741	unsigned NumReturnExprs = 0;
1742
1743	/// Count the number of simple (constant) return expressions in the function.
1744	unsigned NumSimpleReturnExprs = 0;
1745
1746	/// The last regular (non-return) debug location (breakpoint) in the function.
1747	SourceLocation LastStopPoint;
1748
1749	public:
1750	/// Source location information about the default argument or member
1751	/// initializer expression we're evaluating, if any.
1752	CurrentSourceLocExprScope CurSourceLocExprScope;
1753	using SourceLocExprScopeGuard =
1754	CurrentSourceLocExprScope::SourceLocExprScopeGuard;
1755
1756	/// A scope within which we are constructing the fields of an object which
1757	/// might use a CXXDefaultInitExpr. This stashes away a 'this' value to use
1758	/// if we need to evaluate a CXXDefaultInitExpr within the evaluation.
1759	class FieldConstructionScope {
1760	public:
1761	FieldConstructionScope(CodeGenFunction &CGF, Address This)
1762	: CGF(CGF), OldCXXDefaultInitExprThis(CGF.CXXDefaultInitExprThis) {
1763	CGF.CXXDefaultInitExprThis = This;
1764	}
1765	~FieldConstructionScope() {
1766	CGF.CXXDefaultInitExprThis = OldCXXDefaultInitExprThis;
1767	}
1768
1769	private:
1770	CodeGenFunction &CGF;
1771	Address OldCXXDefaultInitExprThis;
1772	};
1773
1774	/// The scope of a CXXDefaultInitExpr. Within this scope, the value of 'this'
1775	/// is overridden to be the object under construction.
1776	class CXXDefaultInitExprScope {
1777	public:
1778	CXXDefaultInitExprScope(CodeGenFunction &CGF, const CXXDefaultInitExpr *E)
1779	: CGF(CGF), OldCXXThisValue(CGF.CXXThisValue),
1780	OldCXXThisAlignment(CGF.CXXThisAlignment),
1781	SourceLocScope(E, CGF.CurSourceLocExprScope) {
1782	CGF.CXXThisValue = CGF.CXXDefaultInitExprThis.getBasePointer();
1783	CGF.CXXThisAlignment = CGF.CXXDefaultInitExprThis.getAlignment();
1784	}
1785	~CXXDefaultInitExprScope() {
1786	CGF.CXXThisValue = OldCXXThisValue;
1787	CGF.CXXThisAlignment = OldCXXThisAlignment;
1788	}
1789
1790	public:
1791	CodeGenFunction &CGF;
1792	llvm::Value *OldCXXThisValue;
1793	CharUnits OldCXXThisAlignment;
1794	SourceLocExprScopeGuard SourceLocScope;
1795	};
1796
1797	struct CXXDefaultArgExprScope : SourceLocExprScopeGuard {
1798	CXXDefaultArgExprScope(CodeGenFunction &CGF, const CXXDefaultArgExpr *E)
1799	: SourceLocExprScopeGuard(E, CGF.CurSourceLocExprScope) {}
1800	};
1801
1802	/// The scope of an ArrayInitLoopExpr. Within this scope, the value of the
1803	/// current loop index is overridden.
1804	class ArrayInitLoopExprScope {
1805	public:
1806	ArrayInitLoopExprScope(CodeGenFunction &CGF, llvm::Value *Index)
1807	: CGF(CGF), OldArrayInitIndex(CGF.ArrayInitIndex) {
1808	CGF.ArrayInitIndex = Index;
1809	}
1810	~ArrayInitLoopExprScope() { CGF.ArrayInitIndex = OldArrayInitIndex; }
1811
1812	private:
1813	CodeGenFunction &CGF;
1814	llvm::Value *OldArrayInitIndex;
1815	};
1816
1817	class InlinedInheritingConstructorScope {
1818	public:
1819	InlinedInheritingConstructorScope(CodeGenFunction &CGF, GlobalDecl GD)
1820	: CGF(CGF), OldCurGD(CGF.CurGD), OldCurFuncDecl(CGF.CurFuncDecl),
1821	OldCurCodeDecl(CGF.CurCodeDecl),
1822	OldCXXABIThisDecl(CGF.CXXABIThisDecl),
1823	OldCXXABIThisValue(CGF.CXXABIThisValue),
1824	OldCXXThisValue(CGF.CXXThisValue),
1825	OldCXXABIThisAlignment(CGF.CXXABIThisAlignment),
1826	OldCXXThisAlignment(CGF.CXXThisAlignment),
1827	OldReturnValue(CGF.ReturnValue), OldFnRetTy(CGF.FnRetTy),
1828	OldCXXInheritedCtorInitExprArgs(
1829	std::move(CGF.CXXInheritedCtorInitExprArgs)) {
1830	CGF.CurGD = GD;
1831	CGF.CurFuncDecl = CGF.CurCodeDecl =
1832	cast<CXXConstructorDecl>(Val: GD.getDecl());
1833	CGF.CXXABIThisDecl = nullptr;
1834	CGF.CXXABIThisValue = nullptr;
1835	CGF.CXXThisValue = nullptr;
1836	CGF.CXXABIThisAlignment = CharUnits();
1837	CGF.CXXThisAlignment = CharUnits();
1838	CGF.ReturnValue = Address::invalid();
1839	CGF.FnRetTy = QualType();
1840	CGF.CXXInheritedCtorInitExprArgs.clear();
1841	}
1842	~InlinedInheritingConstructorScope() {
1843	CGF.CurGD = OldCurGD;
1844	CGF.CurFuncDecl = OldCurFuncDecl;
1845	CGF.CurCodeDecl = OldCurCodeDecl;
1846	CGF.CXXABIThisDecl = OldCXXABIThisDecl;
1847	CGF.CXXABIThisValue = OldCXXABIThisValue;
1848	CGF.CXXThisValue = OldCXXThisValue;
1849	CGF.CXXABIThisAlignment = OldCXXABIThisAlignment;
1850	CGF.CXXThisAlignment = OldCXXThisAlignment;
1851	CGF.ReturnValue = OldReturnValue;
1852	CGF.FnRetTy = OldFnRetTy;
1853	CGF.CXXInheritedCtorInitExprArgs =
1854	std::move(OldCXXInheritedCtorInitExprArgs);
1855	}
1856
1857	private:
1858	CodeGenFunction &CGF;
1859	GlobalDecl OldCurGD;
1860	const Decl *OldCurFuncDecl;
1861	const Decl *OldCurCodeDecl;
1862	ImplicitParamDecl *OldCXXABIThisDecl;
1863	llvm::Value *OldCXXABIThisValue;
1864	llvm::Value *OldCXXThisValue;
1865	CharUnits OldCXXABIThisAlignment;
1866	CharUnits OldCXXThisAlignment;
1867	Address OldReturnValue;
1868	QualType OldFnRetTy;
1869	CallArgList OldCXXInheritedCtorInitExprArgs;
1870	};
1871
1872	// Helper class for the OpenMP IR Builder. Allows reusability of code used for
1873	// region body, and finalization codegen callbacks. This will class will also
1874	// contain privatization functions used by the privatization call backs
1875	//
1876	// TODO: this is temporary class for things that are being moved out of
1877	// CGOpenMPRuntime, new versions of current CodeGenFunction methods, or
1878	// utility function for use with the OMPBuilder. Once that move to use the
1879	// OMPBuilder is done, everything here will either become part of CodeGenFunc.
1880	// directly, or a new helper class that will contain functions used by both
1881	// this and the OMPBuilder
1882
1883	struct OMPBuilderCBHelpers {
1884
1885	OMPBuilderCBHelpers() = delete;
1886	OMPBuilderCBHelpers(const OMPBuilderCBHelpers &) = delete;
1887	OMPBuilderCBHelpers &operator=(const OMPBuilderCBHelpers &) = delete;
1888
1889	using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
1890
1891	/// Cleanup action for allocate support.
1892	class OMPAllocateCleanupTy final : public EHScopeStack::Cleanup {
1893
1894	private:
1895	llvm::CallInst *RTLFnCI;
1896
1897	public:
1898	OMPAllocateCleanupTy(llvm::CallInst *RLFnCI) : RTLFnCI(RLFnCI) {
1899	RLFnCI->removeFromParent();
1900	}
1901
1902	void Emit(CodeGenFunction &CGF, Flags /*flags*/) override {
1903	if (!CGF.HaveInsertPoint())
1904	return;
1905	CGF.Builder.Insert(I: RTLFnCI);
1906	}
1907	};
1908
1909	/// Returns address of the threadprivate variable for the current
1910	/// thread. This Also create any necessary OMP runtime calls.
1911	///
1912	/// \param VD VarDecl for Threadprivate variable.
1913	/// \param VDAddr Address of the Vardecl
1914	/// \param Loc The location where the barrier directive was encountered
1915	static Address getAddrOfThreadPrivate(CodeGenFunction &CGF,
1916	const VarDecl *VD, Address VDAddr,
1917	SourceLocation Loc);
1918
1919	/// Gets the OpenMP-specific address of the local variable /p VD.
1920	static Address getAddressOfLocalVariable(CodeGenFunction &CGF,
1921	const VarDecl *VD);
1922	/// Get the platform-specific name separator.
1923	/// \param Parts different parts of the final name that needs separation
1924	/// \param FirstSeparator First separator used between the initial two
1925	/// parts of the name.
1926	/// \param Separator separator used between all of the rest consecutinve
1927	/// parts of the name
1928	static std::string getNameWithSeparators(ArrayRef<StringRef> Parts,
1929	StringRef FirstSeparator = ".",
1930	StringRef Separator = ".");
1931	/// Emit the Finalization for an OMP region
1932	/// \param CGF The Codegen function this belongs to
1933	/// \param IP Insertion point for generating the finalization code.
1934	static void FinalizeOMPRegion(CodeGenFunction &CGF, InsertPointTy IP) {
1935	CGBuilderTy::InsertPointGuard IPG(CGF.Builder);
1936	assert(IP.getBlock()->end() != IP.getPoint() &&
1937	"OpenMP IR Builder should cause terminated block!");
1938
1939	llvm::BasicBlock *IPBB = IP.getBlock();
1940	llvm::BasicBlock *DestBB = IPBB->getUniqueSuccessor();
1941	assert(DestBB && "Finalization block should have one successor!");
1942
1943	// erase and replace with cleanup branch.
1944	IPBB->getTerminator()->eraseFromParent();
1945	CGF.Builder.SetInsertPoint(IPBB);
1946	CodeGenFunction::JumpDest Dest = CGF.getJumpDestInCurrentScope(Target: DestBB);
1947	CGF.EmitBranchThroughCleanup(Dest);
1948	}
1949
1950	/// Emit the body of an OMP region
1951	/// \param CGF The Codegen function this belongs to
1952	/// \param RegionBodyStmt The body statement for the OpenMP region being
1953	/// generated
1954	/// \param AllocaIP Where to insert alloca instructions
1955	/// \param CodeGenIP Where to insert the region code
1956	/// \param RegionName Name to be used for new blocks
1957	static void EmitOMPInlinedRegionBody(CodeGenFunction &CGF,
1958	const Stmt *RegionBodyStmt,
1959	InsertPointTy AllocaIP,
1960	InsertPointTy CodeGenIP,
1961	Twine RegionName);
1962
1963	static void EmitCaptureStmt(CodeGenFunction &CGF, InsertPointTy CodeGenIP,
1964	llvm::BasicBlock &FiniBB, llvm::Function *Fn,
1965	ArrayRef<llvm::Value *> Args) {
1966	llvm::BasicBlock *CodeGenIPBB = CodeGenIP.getBlock();
1967	if (llvm::Instruction *CodeGenIPBBTI = CodeGenIPBB->getTerminator())
1968	CodeGenIPBBTI->eraseFromParent();
1969
1970	CGF.Builder.SetInsertPoint(CodeGenIPBB);
1971
1972	if (Fn->doesNotThrow())
1973	CGF.EmitNounwindRuntimeCall(callee: Fn, args: Args);
1974	else
1975	CGF.EmitRuntimeCall(callee: Fn, args: Args);
1976
1977	if (CGF.Builder.saveIP().isSet())
1978	CGF.Builder.CreateBr(Dest: &FiniBB);
1979	}
1980
1981	/// Emit the body of an OMP region that will be outlined in
1982	/// OpenMPIRBuilder::finalize().
1983	/// \param CGF The Codegen function this belongs to
1984	/// \param RegionBodyStmt The body statement for the OpenMP region being
1985	/// generated
1986	/// \param AllocaIP Where to insert alloca instructions
1987	/// \param CodeGenIP Where to insert the region code
1988	/// \param RegionName Name to be used for new blocks
1989	static void EmitOMPOutlinedRegionBody(CodeGenFunction &CGF,
1990	const Stmt *RegionBodyStmt,
1991	InsertPointTy AllocaIP,
1992	InsertPointTy CodeGenIP,
1993	Twine RegionName);
1994
1995	/// RAII for preserving necessary info during Outlined region body codegen.
1996	class OutlinedRegionBodyRAII {
1997
1998	llvm::AssertingVH<llvm::Instruction> OldAllocaIP;
1999	CodeGenFunction::JumpDest OldReturnBlock;
2000	CodeGenFunction &CGF;
2001
2002	public:
2003	OutlinedRegionBodyRAII(CodeGenFunction &cgf, InsertPointTy &AllocaIP,
2004	llvm::BasicBlock &RetBB)
2005	: CGF(cgf) {
2006	assert(AllocaIP.isSet() &&
2007	"Must specify Insertion point for allocas of outlined function");
2008	OldAllocaIP = CGF.AllocaInsertPt;
2009	CGF.AllocaInsertPt = &*AllocaIP.getPoint();
2010
2011	OldReturnBlock = CGF.ReturnBlock;
2012	CGF.ReturnBlock = CGF.getJumpDestInCurrentScope(Target: &RetBB);
2013	}
2014
2015	~OutlinedRegionBodyRAII() {
2016	CGF.AllocaInsertPt = OldAllocaIP;
2017	CGF.ReturnBlock = OldReturnBlock;
2018	}
2019	};
2020
2021	/// RAII for preserving necessary info during inlined region body codegen.
2022	class InlinedRegionBodyRAII {
2023
2024	llvm::AssertingVH<llvm::Instruction> OldAllocaIP;
2025	CodeGenFunction &CGF;
2026
2027	public:
2028	InlinedRegionBodyRAII(CodeGenFunction &cgf, InsertPointTy &AllocaIP,
2029	llvm::BasicBlock &FiniBB)
2030	: CGF(cgf) {
2031	// Alloca insertion block should be in the entry block of the containing
2032	// function so it expects an empty AllocaIP in which case will reuse the
2033	// old alloca insertion point, or a new AllocaIP in the same block as
2034	// the old one
2035	assert((!AllocaIP.isSet() ||
2036	CGF.AllocaInsertPt->getParent() == AllocaIP.getBlock()) &&
2037	"Insertion point should be in the entry block of containing "
2038	"function!");
2039	OldAllocaIP = CGF.AllocaInsertPt;
2040	if (AllocaIP.isSet())
2041	CGF.AllocaInsertPt = &*AllocaIP.getPoint();
2042
2043	// TODO: Remove the call, after making sure the counter is not used by
2044	// the EHStack.
2045	// Since this is an inlined region, it should not modify the
2046	// ReturnBlock, and should reuse the one for the enclosing outlined
2047	// region. So, the JumpDest being return by the function is discarded
2048	(void)CGF.getJumpDestInCurrentScope(Target: &FiniBB);
2049	}
2050
2051	~InlinedRegionBodyRAII() { CGF.AllocaInsertPt = OldAllocaIP; }
2052	};
2053	};
2054
2055	private:
2056	/// CXXThisDecl - When generating code for a C++ member function,
2057	/// this will hold the implicit 'this' declaration.
2058	ImplicitParamDecl *CXXABIThisDecl = nullptr;
2059	llvm::Value *CXXABIThisValue = nullptr;
2060	llvm::Value *CXXThisValue = nullptr;
2061	CharUnits CXXABIThisAlignment;
2062	CharUnits CXXThisAlignment;
2063
2064	/// The value of 'this' to use when evaluating CXXDefaultInitExprs within
2065	/// this expression.
2066	Address CXXDefaultInitExprThis = Address::invalid();
2067
2068	/// The current array initialization index when evaluating an
2069	/// ArrayInitIndexExpr within an ArrayInitLoopExpr.
2070	llvm::Value *ArrayInitIndex = nullptr;
2071
2072	/// The values of function arguments to use when evaluating
2073	/// CXXInheritedCtorInitExprs within this context.
2074	CallArgList CXXInheritedCtorInitExprArgs;
2075
2076	/// CXXStructorImplicitParamDecl - When generating code for a constructor or
2077	/// destructor, this will hold the implicit argument (e.g. VTT).
2078	ImplicitParamDecl *CXXStructorImplicitParamDecl = nullptr;
2079	llvm::Value *CXXStructorImplicitParamValue = nullptr;
2080
2081	/// OutermostConditional - Points to the outermost active
2082	/// conditional control. This is used so that we know if a
2083	/// temporary should be destroyed conditionally.
2084	ConditionalEvaluation *OutermostConditional = nullptr;
2085
2086	/// The current lexical scope.
2087	LexicalScope *CurLexicalScope = nullptr;
2088
2089	/// The current source location that should be used for exception
2090	/// handling code.
2091	SourceLocation CurEHLocation;
2092
2093	/// BlockByrefInfos - For each __block variable, contains
2094	/// information about the layout of the variable.
2095	llvm::DenseMap<const ValueDecl *, BlockByrefInfo> BlockByrefInfos;
2096
2097	/// Used by -fsanitize=nullability-return to determine whether the return
2098	/// value can be checked.
2099	llvm::Value *RetValNullabilityPrecondition = nullptr;
2100
2101	/// Check if -fsanitize=nullability-return instrumentation is required for
2102	/// this function.
2103	bool requiresReturnValueNullabilityCheck() const {
2104	return RetValNullabilityPrecondition;
2105	}
2106
2107	/// Used to store precise source locations for return statements by the
2108	/// runtime return value checks.
2109	Address ReturnLocation = Address::invalid();
2110
2111	/// Check if the return value of this function requires sanitization.
2112	bool requiresReturnValueCheck() const;
2113
2114	bool isInAllocaArgument(CGCXXABI &ABI, QualType Ty);
2115	bool hasInAllocaArg(const CXXMethodDecl *MD);
2116
2117	llvm::BasicBlock *TerminateLandingPad = nullptr;
2118	llvm::BasicBlock *TerminateHandler = nullptr;
2119	llvm::SmallVector<llvm::BasicBlock *, 2> TrapBBs;
2120
2121	/// Terminate funclets keyed by parent funclet pad.
2122	llvm::MapVector<llvm::Value *, llvm::BasicBlock *> TerminateFunclets;
2123
2124	/// Largest vector width used in ths function. Will be used to create a
2125	/// function attribute.
2126	unsigned LargestVectorWidth = 0;
2127
2128	/// True if we need emit the life-time markers. This is initially set in
2129	/// the constructor, but could be overwritten to true if this is a coroutine.
2130	bool ShouldEmitLifetimeMarkers;
2131
2132	/// Add OpenCL kernel arg metadata and the kernel attribute metadata to
2133	/// the function metadata.
2134	void EmitKernelMetadata(const FunctionDecl *FD, llvm::Function *Fn);
2135
2136	public:
2137	CodeGenFunction(CodeGenModule &cgm, bool suppressNewContext = false);
2138	~CodeGenFunction();
2139
2140	CodeGenTypes &getTypes() const { return CGM.getTypes(); }
2141	ASTContext &getContext() const { return CGM.getContext(); }
2142	CGDebugInfo *getDebugInfo() {
2143	if (DisableDebugInfo)
2144	return nullptr;
2145	return DebugInfo;
2146	}
2147	void disableDebugInfo() { DisableDebugInfo = true; }
2148	void enableDebugInfo() { DisableDebugInfo = false; }
2149
2150	bool shouldUseFusedARCCalls() {
2151	return CGM.getCodeGenOpts().OptimizationLevel == 0;
2152	}
2153
2154	const LangOptions &getLangOpts() const { return CGM.getLangOpts(); }
2155
2156	/// Returns a pointer to the function's exception object and selector slot,
2157	/// which is assigned in every landing pad.
2158	Address getExceptionSlot();
2159	Address getEHSelectorSlot();
2160
2161	/// Returns the contents of the function's exception object and selector
2162	/// slots.
2163	llvm::Value *getExceptionFromSlot();
2164	llvm::Value *getSelectorFromSlot();
2165
2166	RawAddress getNormalCleanupDestSlot();
2167
2168	llvm::BasicBlock *getUnreachableBlock() {
2169	if (!UnreachableBlock) {
2170	UnreachableBlock = createBasicBlock(name: "unreachable");
2171	new llvm::UnreachableInst(getLLVMContext(), UnreachableBlock);
2172	}
2173	return UnreachableBlock;
2174	}
2175
2176	llvm::BasicBlock *getInvokeDest() {
2177	if (!EHStack.requiresLandingPad())
2178	return nullptr;
2179	return getInvokeDestImpl();
2180	}
2181
2182	bool currentFunctionUsesSEHTry() const { return !!CurSEHParent; }
2183
2184	const TargetInfo &getTarget() const { return Target; }
2185	llvm::LLVMContext &getLLVMContext() { return CGM.getLLVMContext(); }
2186	const TargetCodeGenInfo &getTargetHooks() const {
2187	return CGM.getTargetCodeGenInfo();
2188	}
2189
2190	//===--------------------------------------------------------------------===//
2191	// Cleanups
2192	//===--------------------------------------------------------------------===//
2193
2194	typedef void Destroyer(CodeGenFunction &CGF, Address addr, QualType ty);
2195
2196	void pushIrregularPartialArrayCleanup(llvm::Value *arrayBegin,
2197	Address arrayEndPointer,
2198	QualType elementType,
2199	CharUnits elementAlignment,
2200	Destroyer *destroyer);
2201	void pushRegularPartialArrayCleanup(llvm::Value *arrayBegin,
2202	llvm::Value *arrayEnd,
2203	QualType elementType,
2204	CharUnits elementAlignment,
2205	Destroyer *destroyer);
2206
2207	void pushDestroy(QualType::DestructionKind dtorKind, Address addr,
2208	QualType type);
2209	void pushEHDestroy(QualType::DestructionKind dtorKind, Address addr,
2210	QualType type);
2211	void pushDestroy(CleanupKind kind, Address addr, QualType type,
2212	Destroyer *destroyer, bool useEHCleanupForArray);
2213	void pushDestroyAndDeferDeactivation(QualType::DestructionKind dtorKind,
2214	Address addr, QualType type);
2215	void pushDestroyAndDeferDeactivation(CleanupKind cleanupKind, Address addr,
2216	QualType type, Destroyer *destroyer,
2217	bool useEHCleanupForArray);
2218	void pushLifetimeExtendedDestroy(CleanupKind kind, Address addr,
2219	QualType type, Destroyer *destroyer,
2220	bool useEHCleanupForArray);
2221	void pushLifetimeExtendedDestroy(QualType::DestructionKind dtorKind,
2222	Address addr, QualType type);
2223	void pushCallObjectDeleteCleanup(const FunctionDecl *OperatorDelete,
2224	llvm::Value *CompletePtr,
2225	QualType ElementType);
2226	void pushStackRestore(CleanupKind kind, Address SPMem);
2227	void pushKmpcAllocFree(CleanupKind Kind,
2228	std::pair<llvm::Value *, llvm::Value *> AddrSizePair);
2229	void emitDestroy(Address addr, QualType type, Destroyer *destroyer,
2230	bool useEHCleanupForArray);
2231	llvm::Function *generateDestroyHelper(Address addr, QualType type,
2232	Destroyer *destroyer,
2233	bool useEHCleanupForArray,
2234	const VarDecl *VD);
2235	void emitArrayDestroy(llvm::Value *begin, llvm::Value *end,
2236	QualType elementType, CharUnits elementAlign,
2237	Destroyer *destroyer, bool checkZeroLength,
2238	bool useEHCleanup);
2239
2240	Destroyer *getDestroyer(QualType::DestructionKind destructionKind);
2241
2242	/// Determines whether an EH cleanup is required to destroy a type
2243	/// with the given destruction kind.
2244	bool needsEHCleanup(QualType::DestructionKind kind) {
2245	switch (kind) {
2246	case QualType::DK_none:
2247	return false;
2248	case QualType::DK_cxx_destructor:
2249	case QualType::DK_objc_weak_lifetime:
2250	case QualType::DK_nontrivial_c_struct:
2251	return getLangOpts().Exceptions;
2252	case QualType::DK_objc_strong_lifetime:
2253	return getLangOpts().Exceptions &&
2254	CGM.getCodeGenOpts().ObjCAutoRefCountExceptions;
2255	}
2256	llvm_unreachable("bad destruction kind");
2257	}
2258
2259	CleanupKind getCleanupKind(QualType::DestructionKind kind) {
2260	return (needsEHCleanup(kind) ? NormalAndEHCleanup : NormalCleanup);
2261	}
2262
2263	//===--------------------------------------------------------------------===//
2264	// Objective-C
2265	//===--------------------------------------------------------------------===//
2266
2267	void GenerateObjCMethod(const ObjCMethodDecl *OMD);
2268
2269	void StartObjCMethod(const ObjCMethodDecl *MD, const ObjCContainerDecl *CD);
2270
2271	/// GenerateObjCGetter - Synthesize an Objective-C property getter function.
2272	void GenerateObjCGetter(ObjCImplementationDecl *IMP,
2273	const ObjCPropertyImplDecl *PID);
2274	void generateObjCGetterBody(const ObjCImplementationDecl *classImpl,
2275	const ObjCPropertyImplDecl *propImpl,
2276	const ObjCMethodDecl *GetterMothodDecl,
2277	llvm::Constant *AtomicHelperFn);
2278
2279	void GenerateObjCCtorDtorMethod(ObjCImplementationDecl *IMP,
2280	ObjCMethodDecl *MD, bool ctor);
2281
2282	/// GenerateObjCSetter - Synthesize an Objective-C property setter function
2283	/// for the given property.
2284	void GenerateObjCSetter(ObjCImplementationDecl *IMP,
2285	const ObjCPropertyImplDecl *PID);
2286	void generateObjCSetterBody(const ObjCImplementationDecl *classImpl,
2287	const ObjCPropertyImplDecl *propImpl,
2288	llvm::Constant *AtomicHelperFn);
2289
2290	//===--------------------------------------------------------------------===//
2291	// Block Bits
2292	//===--------------------------------------------------------------------===//
2293
2294	/// Emit block literal.
2295	/// \return an LLVM value which is a pointer to a struct which contains
2296	/// information about the block, including the block invoke function, the
2297	/// captured variables, etc.
2298	llvm::Value *EmitBlockLiteral(const BlockExpr *);
2299
2300	llvm::Function *GenerateBlockFunction(GlobalDecl GD, const CGBlockInfo &Info,
2301	const DeclMapTy &ldm,
2302	bool IsLambdaConversionToBlock,
2303	bool BuildGlobalBlock);
2304
2305	/// Check if \p T is a C++ class that has a destructor that can throw.
2306	static bool cxxDestructorCanThrow(QualType T);
2307
2308	llvm::Constant *GenerateCopyHelperFunction(const CGBlockInfo &blockInfo);
2309	llvm::Constant *GenerateDestroyHelperFunction(const CGBlockInfo &blockInfo);
2310	llvm::Constant *
2311	GenerateObjCAtomicSetterCopyHelperFunction(const ObjCPropertyImplDecl *PID);
2312	llvm::Constant *
2313	GenerateObjCAtomicGetterCopyHelperFunction(const ObjCPropertyImplDecl *PID);
2314	llvm::Value *EmitBlockCopyAndAutorelease(llvm::Value *Block, QualType Ty);
2315
2316	void BuildBlockRelease(llvm::Value *DeclPtr, BlockFieldFlags flags,
2317	bool CanThrow);
2318
2319	class AutoVarEmission;
2320
2321	void emitByrefStructureInit(const AutoVarEmission &emission);
2322
2323	/// Enter a cleanup to destroy a __block variable. Note that this
2324	/// cleanup should be a no-op if the variable hasn't left the stack
2325	/// yet; if a cleanup is required for the variable itself, that needs
2326	/// to be done externally.
2327	///
2328	/// \param Kind Cleanup kind.
2329	///
2330	/// \param Addr When \p LoadBlockVarAddr is false, the address of the __block
2331	/// structure that will be passed to _Block_object_dispose. When
2332	/// \p LoadBlockVarAddr is true, the address of the field of the block
2333	/// structure that holds the address of the __block structure.
2334	///
2335	/// \param Flags The flag that will be passed to _Block_object_dispose.
2336	///
2337	/// \param LoadBlockVarAddr Indicates whether we need to emit a load from
2338	/// \p Addr to get the address of the __block structure.
2339	void enterByrefCleanup(CleanupKind Kind, Address Addr, BlockFieldFlags Flags,
2340	bool LoadBlockVarAddr, bool CanThrow);
2341
2342	void setBlockContextParameter(const ImplicitParamDecl *D, unsigned argNum,
2343	llvm::Value *ptr);
2344
2345	Address LoadBlockStruct();
2346	Address GetAddrOfBlockDecl(const VarDecl *var);
2347
2348	/// BuildBlockByrefAddress - Computes the location of the
2349	/// data in a variable which is declared as __block.
2350	Address emitBlockByrefAddress(Address baseAddr, const VarDecl *V,
2351	bool followForward = true);
2352	Address emitBlockByrefAddress(Address baseAddr, const BlockByrefInfo &info,
2353	bool followForward, const llvm::Twine &name);
2354
2355	const BlockByrefInfo &getBlockByrefInfo(const VarDecl *var);
2356
2357	QualType BuildFunctionArgList(GlobalDecl GD, FunctionArgList &Args);
2358
2359	void GenerateCode(GlobalDecl GD, llvm::Function *Fn,
2360	const CGFunctionInfo &FnInfo);
2361
2362	/// Annotate the function with an attribute that disables TSan checking at
2363	/// runtime.
2364	void markAsIgnoreThreadCheckingAtRuntime(llvm::Function *Fn);
2365
2366	/// Emit code for the start of a function.
2367	/// \param Loc The location to be associated with the function.
2368	/// \param StartLoc The location of the function body.
2369	void StartFunction(GlobalDecl GD, QualType RetTy, llvm::Function *Fn,
2370	const CGFunctionInfo &FnInfo, const FunctionArgList &Args,
2371	SourceLocation Loc = SourceLocation(),
2372	SourceLocation StartLoc = SourceLocation());
2373
2374	static bool IsConstructorDelegationValid(const CXXConstructorDecl *Ctor);
2375
2376	void EmitConstructorBody(FunctionArgList &Args);
2377	void EmitDestructorBody(FunctionArgList &Args);
2378	void emitImplicitAssignmentOperatorBody(FunctionArgList &Args);
2379	void EmitFunctionBody(const Stmt *Body);
2380	void EmitBlockWithFallThrough(llvm::BasicBlock *BB, const Stmt *S);
2381
2382	void EmitForwardingCallToLambda(const CXXMethodDecl *LambdaCallOperator,
2383	CallArgList &CallArgs,
2384	const CGFunctionInfo *CallOpFnInfo = nullptr,
2385	llvm::Constant *CallOpFn = nullptr);
2386	void EmitLambdaBlockInvokeBody();
2387	void EmitLambdaStaticInvokeBody(const CXXMethodDecl *MD);
2388	void EmitLambdaDelegatingInvokeBody(const CXXMethodDecl *MD,
2389	CallArgList &CallArgs);
2390	void EmitLambdaInAllocaImplFn(const CXXMethodDecl *CallOp,
2391	const CGFunctionInfo **ImplFnInfo,
2392	llvm::Function **ImplFn);
2393	void EmitLambdaInAllocaCallOpBody(const CXXMethodDecl *MD);
2394	void EmitLambdaVLACapture(const VariableArrayType *VAT, LValue LV) {
2395	EmitStoreThroughLValue(Src: RValue::get(V: VLASizeMap[VAT->getSizeExpr()]), Dst: LV);
2396	}
2397	void EmitAsanPrologueOrEpilogue(bool Prologue);
2398
2399	/// Emit the unified return block, trying to avoid its emission when
2400	/// possible.
2401	/// \return The debug location of the user written return statement if the
2402	/// return block is avoided.
2403	llvm::DebugLoc EmitReturnBlock();
2404
2405	/// FinishFunction - Complete IR generation of the current function. It is
2406	/// legal to call this function even if there is no current insertion point.
2407	void FinishFunction(SourceLocation EndLoc = SourceLocation());
2408
2409	void StartThunk(llvm::Function *Fn, GlobalDecl GD,
2410	const CGFunctionInfo &FnInfo, bool IsUnprototyped);
2411
2412	void EmitCallAndReturnForThunk(llvm::FunctionCallee Callee,
2413	const ThunkInfo *Thunk, bool IsUnprototyped);
2414
2415	void FinishThunk();
2416
2417	/// Emit a musttail call for a thunk with a potentially adjusted this pointer.
2418	void EmitMustTailThunk(GlobalDecl GD, llvm::Value *AdjustedThisPtr,
2419	llvm::FunctionCallee Callee);
2420
2421	/// Generate a thunk for the given method.
2422	void generateThunk(llvm::Function *Fn, const CGFunctionInfo &FnInfo,
2423	GlobalDecl GD, const ThunkInfo &Thunk,
2424	bool IsUnprototyped);
2425
2426	llvm::Function *GenerateVarArgsThunk(llvm::Function *Fn,
2427	const CGFunctionInfo &FnInfo,
2428	GlobalDecl GD, const ThunkInfo &Thunk);
2429
2430	void EmitCtorPrologue(const CXXConstructorDecl *CD, CXXCtorType Type,
2431	FunctionArgList &Args);
2432
2433	void EmitInitializerForField(FieldDecl *Field, LValue LHS, Expr *Init);
2434
2435	/// Struct with all information about dynamic [sub]class needed to set vptr.
2436	struct VPtr {
2437	BaseSubobject Base;
2438	const CXXRecordDecl *NearestVBase;
2439	CharUnits OffsetFromNearestVBase;
2440	const CXXRecordDecl *VTableClass;
2441	};
2442
2443	/// Initialize the vtable pointer of the given subobject.
2444	void InitializeVTablePointer(const VPtr &vptr);
2445
2446	typedef llvm::SmallVector<VPtr, 4> VPtrsVector;
2447
2448	typedef llvm::SmallPtrSet<const CXXRecordDecl *, 4> VisitedVirtualBasesSetTy;
2449	VPtrsVector getVTablePointers(const CXXRecordDecl *VTableClass);
2450
2451	void getVTablePointers(BaseSubobject Base, const CXXRecordDecl *NearestVBase,
2452	CharUnits OffsetFromNearestVBase,
2453	bool BaseIsNonVirtualPrimaryBase,
2454	const CXXRecordDecl *VTableClass,
2455	VisitedVirtualBasesSetTy &VBases, VPtrsVector &vptrs);
2456
2457	void InitializeVTablePointers(const CXXRecordDecl *ClassDecl);
2458
2459	// VTableTrapMode - whether we guarantee that loading the
2460	// vtable is guaranteed to trap on authentication failure,
2461	// even if the resulting vtable pointer is unused.
2462	enum class VTableAuthMode {
2463	Authenticate,
2464	MustTrap,
2465	UnsafeUbsanStrip // Should only be used for Vptr UBSan check
2466	};
2467	/// GetVTablePtr - Return the Value of the vtable pointer member pointed
2468	/// to by This.
2469	llvm::Value *
2470	GetVTablePtr(Address This, llvm::Type *VTableTy,
2471	const CXXRecordDecl *VTableClass,
2472	VTableAuthMode AuthMode = VTableAuthMode::Authenticate);
2473
2474	enum CFITypeCheckKind {
2475	CFITCK_VCall,
2476	CFITCK_NVCall,
2477	CFITCK_DerivedCast,
2478	CFITCK_UnrelatedCast,
2479	CFITCK_ICall,
2480	CFITCK_NVMFCall,
2481	CFITCK_VMFCall,
2482	};
2483
2484	/// Derived is the presumed address of an object of type T after a
2485	/// cast. If T is a polymorphic class type, emit a check that the virtual
2486	/// table for Derived belongs to a class derived from T.
2487	void EmitVTablePtrCheckForCast(QualType T, Address Derived, bool MayBeNull,
2488	CFITypeCheckKind TCK, SourceLocation Loc);
2489
2490	/// EmitVTablePtrCheckForCall - Virtual method MD is being called via VTable.
2491	/// If vptr CFI is enabled, emit a check that VTable is valid.
2492	void EmitVTablePtrCheckForCall(const CXXRecordDecl *RD, llvm::Value *VTable,
2493	CFITypeCheckKind TCK, SourceLocation Loc);
2494
2495	/// EmitVTablePtrCheck - Emit a check that VTable is a valid virtual table for
2496	/// RD using llvm.type.test.
2497	void EmitVTablePtrCheck(const CXXRecordDecl *RD, llvm::Value *VTable,
2498	CFITypeCheckKind TCK, SourceLocation Loc);
2499
2500	/// If whole-program virtual table optimization is enabled, emit an assumption
2501	/// that VTable is a member of RD's type identifier. Or, if vptr CFI is
2502	/// enabled, emit a check that VTable is a member of RD's type identifier.
2503	void EmitTypeMetadataCodeForVCall(const CXXRecordDecl *RD,
2504	llvm::Value *VTable, SourceLocation Loc);
2505
2506	/// Returns whether we should perform a type checked load when loading a
2507	/// virtual function for virtual calls to members of RD. This is generally
2508	/// true when both vcall CFI and whole-program-vtables are enabled.
2509	bool ShouldEmitVTableTypeCheckedLoad(const CXXRecordDecl *RD);
2510
2511	/// Emit a type checked load from the given vtable.
2512	llvm::Value *EmitVTableTypeCheckedLoad(const CXXRecordDecl *RD,
2513	llvm::Value *VTable,
2514	llvm::Type *VTableTy,
2515	uint64_t VTableByteOffset);
2516
2517	/// EnterDtorCleanups - Enter the cleanups necessary to complete the
2518	/// given phase of destruction for a destructor. The end result
2519	/// should call destructors on members and base classes in reverse
2520	/// order of their construction.
2521	void EnterDtorCleanups(const CXXDestructorDecl *Dtor, CXXDtorType Type);
2522
2523	/// ShouldInstrumentFunction - Return true if the current function should be
2524	/// instrumented with __cyg_profile_func_* calls
2525	bool ShouldInstrumentFunction();
2526
2527	/// ShouldSkipSanitizerInstrumentation - Return true if the current function
2528	/// should not be instrumented with sanitizers.
2529	bool ShouldSkipSanitizerInstrumentation();
2530
2531	/// ShouldXRayInstrument - Return true if the current function should be
2532	/// instrumented with XRay nop sleds.
2533	bool ShouldXRayInstrumentFunction() const;
2534
2535	/// AlwaysEmitXRayCustomEvents - Return true if we must unconditionally emit
2536	/// XRay custom event handling calls.
2537	bool AlwaysEmitXRayCustomEvents() const;
2538
2539	/// AlwaysEmitXRayTypedEvents - Return true if clang must unconditionally emit
2540	/// XRay typed event handling calls.
2541	bool AlwaysEmitXRayTypedEvents() const;
2542
2543	/// Return a type hash constant for a function instrumented by
2544	/// -fsanitize=function.
2545	llvm::ConstantInt *getUBSanFunctionTypeHash(QualType T) const;
2546
2547	/// EmitFunctionProlog - Emit the target specific LLVM code to load the
2548	/// arguments for the given function. This is also responsible for naming the
2549	/// LLVM function arguments.
2550	void EmitFunctionProlog(const CGFunctionInfo &FI, llvm::Function *Fn,
2551	const FunctionArgList &Args);
2552
2553	/// EmitFunctionEpilog - Emit the target specific LLVM code to return the
2554	/// given temporary. Specify the source location atom group (Key Instructions
2555	/// debug info feature) for the `ret` using \p RetKeyInstructionsSourceAtom.
2556	/// If it's 0, the `ret` will get added to a new source atom group.
2557	void EmitFunctionEpilog(const CGFunctionInfo &FI, bool EmitRetDbgLoc,
2558	SourceLocation EndLoc,
2559	uint64_t RetKeyInstructionsSourceAtom);
2560
2561	/// Emit a test that checks if the return value \p RV is nonnull.
2562	void EmitReturnValueCheck(llvm::Value *RV);
2563
2564	/// EmitStartEHSpec - Emit the start of the exception spec.
2565	void EmitStartEHSpec(const Decl *D);
2566
2567	/// EmitEndEHSpec - Emit the end of the exception spec.
2568	void EmitEndEHSpec(const Decl *D);
2569
2570	/// getTerminateLandingPad - Return a landing pad that just calls terminate.
2571	llvm::BasicBlock *getTerminateLandingPad();
2572
2573	/// getTerminateLandingPad - Return a cleanup funclet that just calls
2574	/// terminate.
2575	llvm::BasicBlock *getTerminateFunclet();
2576
2577	/// getTerminateHandler - Return a handler (not a landing pad, just
2578	/// a catch handler) that just calls terminate. This is used when
2579	/// a terminate scope encloses a try.
2580	llvm::BasicBlock *getTerminateHandler();
2581
2582	llvm::Type *ConvertTypeForMem(QualType T);
2583	llvm::Type *ConvertType(QualType T);
2584	llvm::Type *convertTypeForLoadStore(QualType ASTTy,
2585	llvm::Type *LLVMTy = nullptr);
2586	llvm::Type *ConvertType(const TypeDecl *T) {
2587	return ConvertType(T: getContext().getTypeDeclType(Decl: T));
2588	}
2589
2590	/// LoadObjCSelf - Load the value of self. This function is only valid while
2591	/// generating code for an Objective-C method.
2592	llvm::Value *LoadObjCSelf();
2593
2594	/// TypeOfSelfObject - Return type of object that this self represents.
2595	QualType TypeOfSelfObject();
2596
2597	/// getEvaluationKind - Return the TypeEvaluationKind of QualType \c T.
2598	static TypeEvaluationKind getEvaluationKind(QualType T);
2599
2600	static bool hasScalarEvaluationKind(QualType T) {
2601	return getEvaluationKind(T) == TEK_Scalar;
2602	}
2603
2604	static bool hasAggregateEvaluationKind(QualType T) {
2605	return getEvaluationKind(T) == TEK_Aggregate;
2606	}
2607
2608	/// createBasicBlock - Create an LLVM basic block.
2609	llvm::BasicBlock *createBasicBlock(const Twine &name = "",
2610	llvm::Function *parent = nullptr,
2611	llvm::BasicBlock *before = nullptr) {
2612	return llvm::BasicBlock::Create(Context&: getLLVMContext(), Name: name, Parent: parent, InsertBefore: before);
2613	}
2614
2615	/// getBasicBlockForLabel - Return the LLVM basicblock that the specified
2616	/// label maps to.
2617	JumpDest getJumpDestForLabel(const LabelDecl *S);
2618
2619	/// SimplifyForwardingBlocks - If the given basic block is only a branch to
2620	/// another basic block, simplify it. This assumes that no other code could
2621	/// potentially reference the basic block.
2622	void SimplifyForwardingBlocks(llvm::BasicBlock *BB);
2623
2624	/// EmitBlock - Emit the given block \arg BB and set it as the insert point,
2625	/// adding a fall-through branch from the current insert block if
2626	/// necessary. It is legal to call this function even if there is no current
2627	/// insertion point.
2628	///
2629	/// IsFinished - If true, indicates that the caller has finished emitting
2630	/// branches to the given block and does not expect to emit code into it. This
2631	/// means the block can be ignored if it is unreachable.
2632	void EmitBlock(llvm::BasicBlock *BB, bool IsFinished = false);
2633
2634	/// EmitBlockAfterUses - Emit the given block somewhere hopefully
2635	/// near its uses, and leave the insertion point in it.
2636	void EmitBlockAfterUses(llvm::BasicBlock *BB);
2637
2638	/// EmitBranch - Emit a branch to the specified basic block from the current
2639	/// insert block, taking care to avoid creation of branches from dummy
2640	/// blocks. It is legal to call this function even if there is no current
2641	/// insertion point.
2642	///
2643	/// This function clears the current insertion point. The caller should follow
2644	/// calls to this function with calls to Emit*Block prior to generation new
2645	/// code.
2646	void EmitBranch(llvm::BasicBlock *Block);
2647
2648	/// HaveInsertPoint - True if an insertion point is defined. If not, this
2649	/// indicates that the current code being emitted is unreachable.
2650	bool HaveInsertPoint() const { return Builder.GetInsertBlock() != nullptr; }
2651
2652	/// EnsureInsertPoint - Ensure that an insertion point is defined so that
2653	/// emitted IR has a place to go. Note that by definition, if this function
2654	/// creates a block then that block is unreachable; callers may do better to
2655	/// detect when no insertion point is defined and simply skip IR generation.
2656	void EnsureInsertPoint() {
2657	if (!HaveInsertPoint())
2658	EmitBlock(BB: createBasicBlock());
2659	}
2660
2661	/// ErrorUnsupported - Print out an error that codegen doesn't support the
2662	/// specified stmt yet.
2663	void ErrorUnsupported(const Stmt *S, const char *Type);
2664
2665	//===--------------------------------------------------------------------===//
2666	// Helpers
2667	//===--------------------------------------------------------------------===//
2668
2669	Address mergeAddressesInConditionalExpr(Address LHS, Address RHS,
2670	llvm::BasicBlock *LHSBlock,
2671	llvm::BasicBlock *RHSBlock,
2672	llvm::BasicBlock *MergeBlock,
2673	QualType MergedType) {
2674	Builder.SetInsertPoint(MergeBlock);
2675	llvm::PHINode *PtrPhi = Builder.CreatePHI(Ty: LHS.getType(), NumReservedValues: 2, Name: "cond");
2676	PtrPhi->addIncoming(V: LHS.getBasePointer(), BB: LHSBlock);
2677	PtrPhi->addIncoming(V: RHS.getBasePointer(), BB: RHSBlock);
2678	LHS.replaceBasePointer(P: PtrPhi);
2679	LHS.setAlignment(std::min(a: LHS.getAlignment(), b: RHS.getAlignment()));
2680	return LHS;
2681	}
2682
2683	/// Construct an address with the natural alignment of T. If a pointer to T
2684	/// is expected to be signed, the pointer passed to this function must have
2685	/// been signed, and the returned Address will have the pointer authentication
2686	/// information needed to authenticate the signed pointer.
2687	Address makeNaturalAddressForPointer(
2688	llvm::Value *Ptr, QualType T, CharUnits Alignment = CharUnits::Zero(),
2689	bool ForPointeeType = false, LValueBaseInfo *BaseInfo = nullptr,
2690	TBAAAccessInfo *TBAAInfo = nullptr,
2691	KnownNonNull_t IsKnownNonNull = NotKnownNonNull) {
2692	if (Alignment.isZero())
2693	Alignment =
2694	CGM.getNaturalTypeAlignment(T, BaseInfo, TBAAInfo, forPointeeType: ForPointeeType);
2695	return Address(Ptr, ConvertTypeForMem(T), Alignment,
2696	CGM.getPointerAuthInfoForPointeeType(type: T), /*Offset=*/nullptr,
2697	IsKnownNonNull);
2698	}
2699
2700	LValue MakeAddrLValue(Address Addr, QualType T,
2701	AlignmentSource Source = AlignmentSource::Type) {
2702	return MakeAddrLValue(Addr, T, BaseInfo: LValueBaseInfo(Source),
2703	TBAAInfo: CGM.getTBAAAccessInfo(AccessType: T));
2704	}
2705
2706	LValue MakeAddrLValue(Address Addr, QualType T, LValueBaseInfo BaseInfo,
2707	TBAAAccessInfo TBAAInfo) {
2708	return LValue::MakeAddr(Addr, type: T, Context&: getContext(), BaseInfo, TBAAInfo);
2709	}
2710
2711	LValue MakeAddrLValue(llvm::Value *V, QualType T, CharUnits Alignment,
2712	AlignmentSource Source = AlignmentSource::Type) {
2713	return MakeAddrLValue(Addr: makeNaturalAddressForPointer(Ptr: V, T, Alignment), T,
2714	BaseInfo: LValueBaseInfo(Source), TBAAInfo: CGM.getTBAAAccessInfo(AccessType: T));
2715	}
2716
2717	/// Same as MakeAddrLValue above except that the pointer is known to be
2718	/// unsigned.
2719	LValue MakeRawAddrLValue(llvm::Value *V, QualType T, CharUnits Alignment,
2720	AlignmentSource Source = AlignmentSource::Type) {
2721	Address Addr(V, ConvertTypeForMem(T), Alignment);
2722	return LValue::MakeAddr(Addr, type: T, Context&: getContext(), BaseInfo: LValueBaseInfo(Source),
2723	TBAAInfo: CGM.getTBAAAccessInfo(AccessType: T));
2724	}
2725
2726	LValue
2727	MakeAddrLValueWithoutTBAA(Address Addr, QualType T,
2728	AlignmentSource Source = AlignmentSource::Type) {
2729	return LValue::MakeAddr(Addr, type: T, Context&: getContext(), BaseInfo: LValueBaseInfo(Source),
2730	TBAAInfo: TBAAAccessInfo());
2731	}
2732
2733	/// Given a value of type T* that may not be to a complete object, construct
2734	/// an l-value with the natural pointee alignment of T.
2735	LValue MakeNaturalAlignPointeeAddrLValue(llvm::Value *V, QualType T);
2736
2737	LValue
2738	MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T,
2739	KnownNonNull_t IsKnownNonNull = NotKnownNonNull);
2740
2741	/// Same as MakeNaturalAlignPointeeAddrLValue except that the pointer is known
2742	/// to be unsigned.
2743	LValue MakeNaturalAlignPointeeRawAddrLValue(llvm::Value *V, QualType T);
2744
2745	LValue MakeNaturalAlignRawAddrLValue(llvm::Value *V, QualType T);
2746
2747	Address EmitLoadOfReference(LValue RefLVal,
2748	LValueBaseInfo *PointeeBaseInfo = nullptr,
2749	TBAAAccessInfo *PointeeTBAAInfo = nullptr);
2750	LValue EmitLoadOfReferenceLValue(LValue RefLVal);
2751	LValue
2752	EmitLoadOfReferenceLValue(Address RefAddr, QualType RefTy,
2753	AlignmentSource Source = AlignmentSource::Type) {
2754	LValue RefLVal = MakeAddrLValue(Addr: RefAddr, T: RefTy, BaseInfo: LValueBaseInfo(Source),
2755	TBAAInfo: CGM.getTBAAAccessInfo(AccessType: RefTy));
2756	return EmitLoadOfReferenceLValue(RefLVal);
2757	}
2758
2759	/// Load a pointer with type \p PtrTy stored at address \p Ptr.
2760	/// Note that \p PtrTy is the type of the loaded pointer, not the addresses
2761	/// it is loaded from.
2762	Address EmitLoadOfPointer(Address Ptr, const PointerType *PtrTy,
2763	LValueBaseInfo *BaseInfo = nullptr,
2764	TBAAAccessInfo *TBAAInfo = nullptr);
2765	LValue EmitLoadOfPointerLValue(Address Ptr, const PointerType *PtrTy);
2766
2767	private:
2768	struct AllocaTracker {
2769	void Add(llvm::AllocaInst *I) { Allocas.push_back(Elt: I); }
2770	llvm::SmallVector<llvm::AllocaInst *> Take() { return std::move(Allocas); }
2771
2772	private:
2773	llvm::SmallVector<llvm::AllocaInst *> Allocas;
2774	};
2775	AllocaTracker *Allocas = nullptr;
2776
2777	/// CGDecl helper.
2778	void emitStoresForConstant(const VarDecl &D, Address Loc, bool isVolatile,
2779	llvm::Constant *constant, bool IsAutoInit);
2780	/// CGDecl helper.
2781	void emitStoresForZeroInit(const VarDecl &D, Address Loc, bool isVolatile);
2782	/// CGDecl helper.
2783	void emitStoresForPatternInit(const VarDecl &D, Address Loc, bool isVolatile);
2784	/// CGDecl helper.
2785	void emitStoresForInitAfterBZero(llvm::Constant *Init, Address Loc,
2786	bool isVolatile, bool IsAutoInit);
2787
2788	public:
2789	// Captures all the allocas created during the scope of its RAII object.
2790	struct AllocaTrackerRAII {
2791	AllocaTrackerRAII(CodeGenFunction &CGF)
2792	: CGF(CGF), OldTracker(CGF.Allocas) {
2793	CGF.Allocas = &Tracker;
2794	}
2795	~AllocaTrackerRAII() { CGF.Allocas = OldTracker; }
2796
2797	llvm::SmallVector<llvm::AllocaInst *> Take() { return Tracker.Take(); }
2798
2799	private:
2800	CodeGenFunction &CGF;
2801	AllocaTracker *OldTracker;
2802	AllocaTracker Tracker;
2803	};
2804
2805	/// CreateTempAlloca - This creates an alloca and inserts it into the entry
2806	/// block if \p ArraySize is nullptr, otherwise inserts it at the current
2807	/// insertion point of the builder. The caller is responsible for setting an
2808	/// appropriate alignment on
2809	/// the alloca.
2810	///
2811	/// \p ArraySize is the number of array elements to be allocated if it
2812	/// is not nullptr.
2813	///
2814	/// LangAS::Default is the address space of pointers to local variables and
2815	/// temporaries, as exposed in the source language. In certain
2816	/// configurations, this is not the same as the alloca address space, and a
2817	/// cast is needed to lift the pointer from the alloca AS into
2818	/// LangAS::Default. This can happen when the target uses a restricted
2819	/// address space for the stack but the source language requires
2820	/// LangAS::Default to be a generic address space. The latter condition is
2821	/// common for most programming languages; OpenCL is an exception in that
2822	/// LangAS::Default is the private address space, which naturally maps
2823	/// to the stack.
2824	///
2825	/// Because the address of a temporary is often exposed to the program in
2826	/// various ways, this function will perform the cast. The original alloca
2827	/// instruction is returned through \p Alloca if it is not nullptr.
2828	///
2829	/// The cast is not performaed in CreateTempAllocaWithoutCast. This is
2830	/// more efficient if the caller knows that the address will not be exposed.
2831	llvm::AllocaInst *CreateTempAlloca(llvm::Type *Ty, const Twine &Name = "tmp",
2832	llvm::Value *ArraySize = nullptr);
2833
2834	/// CreateTempAlloca - This creates a alloca and inserts it into the entry
2835	/// block. The alloca is casted to the address space of \p UseAddrSpace if
2836	/// necessary.
2837	RawAddress CreateTempAlloca(llvm::Type *Ty, LangAS UseAddrSpace,
2838	CharUnits align, const Twine &Name = "tmp",
2839	llvm::Value *ArraySize = nullptr,
2840	RawAddress *Alloca = nullptr);
2841
2842	/// CreateTempAlloca - This creates a alloca and inserts it into the entry
2843	/// block. The alloca is casted to default address space if necessary.
2844	///
2845	/// FIXME: This version should be removed, and context should provide the
2846	/// context use address space used instead of default.
2847	RawAddress CreateTempAlloca(llvm::Type *Ty, CharUnits align,
2848	const Twine &Name = "tmp",
2849	llvm::Value *ArraySize = nullptr,
2850	RawAddress *Alloca = nullptr) {
2851	return CreateTempAlloca(Ty, UseAddrSpace: LangAS::Default, align, Name, ArraySize,
2852	Alloca);
2853	}
2854
2855	RawAddress CreateTempAllocaWithoutCast(llvm::Type *Ty, CharUnits align,
2856	const Twine &Name = "tmp",
2857	llvm::Value *ArraySize = nullptr);
2858
2859	/// CreateDefaultAlignedTempAlloca - This creates an alloca with the
2860	/// default ABI alignment of the given LLVM type.
2861	///
2862	/// IMPORTANT NOTE: This is *not* generally the right alignment for
2863	/// any given AST type that happens to have been lowered to the
2864	/// given IR type. This should only ever be used for function-local,
2865	/// IR-driven manipulations like saving and restoring a value. Do
2866	/// not hand this address off to arbitrary IRGen routines, and especially
2867	/// do not pass it as an argument to a function that might expect a
2868	/// properly ABI-aligned value.
2869	RawAddress CreateDefaultAlignTempAlloca(llvm::Type *Ty,
2870	const Twine &Name = "tmp");
2871
2872	/// CreateIRTemp - Create a temporary IR object of the given type, with
2873	/// appropriate alignment. This routine should only be used when an temporary
2874	/// value needs to be stored into an alloca (for example, to avoid explicit
2875	/// PHI construction), but the type is the IR type, not the type appropriate
2876	/// for storing in memory.
2877	///
2878	/// That is, this is exactly equivalent to CreateMemTemp, but calling
2879	/// ConvertType instead of ConvertTypeForMem.
2880	RawAddress CreateIRTemp(QualType T, const Twine &Name = "tmp");
2881
2882	/// CreateMemTemp - Create a temporary memory object of the given type, with
2883	/// appropriate alignmen and cast it to the default address space. Returns
2884	/// the original alloca instruction by \p Alloca if it is not nullptr.
2885	RawAddress CreateMemTemp(QualType T, const Twine &Name = "tmp",
2886	RawAddress *Alloca = nullptr);
2887	RawAddress CreateMemTemp(QualType T, CharUnits Align,
2888	const Twine &Name = "tmp",
2889	RawAddress *Alloca = nullptr);
2890
2891	/// CreateMemTemp - Create a temporary memory object of the given type, with
2892	/// appropriate alignmen without casting it to the default address space.
2893	RawAddress CreateMemTempWithoutCast(QualType T, const Twine &Name = "tmp");
2894	RawAddress CreateMemTempWithoutCast(QualType T, CharUnits Align,
2895	const Twine &Name = "tmp");
2896
2897	/// CreateAggTemp - Create a temporary memory object for the given
2898	/// aggregate type.
2899	AggValueSlot CreateAggTemp(QualType T, const Twine &Name = "tmp",
2900	RawAddress *Alloca = nullptr) {
2901	return AggValueSlot::forAddr(
2902	addr: CreateMemTemp(T, Name, Alloca), quals: T.getQualifiers(),
2903	isDestructed: AggValueSlot::IsNotDestructed, needsGC: AggValueSlot::DoesNotNeedGCBarriers,
2904	isAliased: AggValueSlot::IsNotAliased, mayOverlap: AggValueSlot::DoesNotOverlap);
2905	}
2906
2907	/// EvaluateExprAsBool - Perform the usual unary conversions on the specified
2908	/// expression and compare the result against zero, returning an Int1Ty value.
2909	llvm::Value *EvaluateExprAsBool(const Expr *E);
2910
2911	/// Retrieve the implicit cast expression of the rhs in a binary operator
2912	/// expression by passing pointers to Value and QualType
2913	/// This is used for implicit bitfield conversion checks, which
2914	/// must compare with the value before potential truncation.
2915	llvm::Value *EmitWithOriginalRHSBitfieldAssignment(const BinaryOperator *E,
2916	llvm::Value **Previous,
2917	QualType *SrcType);
2918
2919	/// Emit a check that an [implicit] conversion of a bitfield. It is not UB,
2920	/// so we use the value after conversion.
2921	void EmitBitfieldConversionCheck(llvm::Value *Src, QualType SrcType,
2922	llvm::Value *Dst, QualType DstType,
2923	const CGBitFieldInfo &Info,
2924	SourceLocation Loc);
2925
2926	/// EmitIgnoredExpr - Emit an expression in a context which ignores the
2927	/// result.
2928	void EmitIgnoredExpr(const Expr *E);
2929
2930	/// EmitAnyExpr - Emit code to compute the specified expression which can have
2931	/// any type. The result is returned as an RValue struct. If this is an
2932	/// aggregate expression, the aggloc/agglocvolatile arguments indicate where
2933	/// the result should be returned.
2934	///
2935	/// \param ignoreResult True if the resulting value isn't used.
2936	RValue EmitAnyExpr(const Expr *E,
2937	AggValueSlot aggSlot = AggValueSlot::ignored(),
2938	bool ignoreResult = false);
2939
2940	// EmitVAListRef - Emit a "reference" to a va_list; this is either the address
2941	// or the value of the expression, depending on how va_list is defined.
2942	Address EmitVAListRef(const Expr *E);
2943
2944	/// Emit a "reference" to a __builtin_ms_va_list; this is
2945	/// always the value of the expression, because a __builtin_ms_va_list is a
2946	/// pointer to a char.
2947	Address EmitMSVAListRef(const Expr *E);
2948
2949	/// EmitAnyExprToTemp - Similarly to EmitAnyExpr(), however, the result will
2950	/// always be accessible even if no aggregate location is provided.
2951	RValue EmitAnyExprToTemp(const Expr *E);
2952
2953	/// EmitAnyExprToMem - Emits the code necessary to evaluate an
2954	/// arbitrary expression into the given memory location.
2955	void EmitAnyExprToMem(const Expr *E, Address Location, Qualifiers Quals,
2956	bool IsInitializer);
2957
2958	void EmitAnyExprToExn(const Expr *E, Address Addr);
2959
2960	/// EmitInitializationToLValue - Emit an initializer to an LValue.
2961	void EmitInitializationToLValue(
2962	const Expr *E, LValue LV,
2963	AggValueSlot::IsZeroed_t IsZeroed = AggValueSlot::IsNotZeroed);
2964
2965	/// EmitExprAsInit - Emits the code necessary to initialize a
2966	/// location in memory with the given initializer.
2967	void EmitExprAsInit(const Expr *init, const ValueDecl *D, LValue lvalue,
2968	bool capturedByInit);
2969
2970	/// hasVolatileMember - returns true if aggregate type has a volatile
2971	/// member.
2972	bool hasVolatileMember(QualType T) {
2973	if (const RecordType *RT = T->getAs<RecordType>()) {
2974	const RecordDecl *RD = cast<RecordDecl>(Val: RT->getDecl());
2975	return RD->hasVolatileMember();
2976	}
2977	return false;
2978	}
2979
2980	/// Determine whether a return value slot may overlap some other object.
2981	AggValueSlot::Overlap_t getOverlapForReturnValue() {
2982	// FIXME: Assuming no overlap here breaks guaranteed copy elision for base
2983	// class subobjects. These cases may need to be revisited depending on the
2984	// resolution of the relevant core issue.
2985	return AggValueSlot::DoesNotOverlap;
2986	}
2987
2988	/// Determine whether a field initialization may overlap some other object.
2989	AggValueSlot::Overlap_t getOverlapForFieldInit(const FieldDecl *FD);
2990
2991	/// Determine whether a base class initialization may overlap some other
2992	/// object.
2993	AggValueSlot::Overlap_t getOverlapForBaseInit(const CXXRecordDecl *RD,
2994	const CXXRecordDecl *BaseRD,
2995	bool IsVirtual);
2996
2997	/// Emit an aggregate assignment.
2998	void EmitAggregateAssign(LValue Dest, LValue Src, QualType EltTy) {
2999	ApplyAtomGroup Grp(getDebugInfo());
3000	bool IsVolatile = hasVolatileMember(T: EltTy);
3001	EmitAggregateCopy(Dest, Src, EltTy, MayOverlap: AggValueSlot::MayOverlap, isVolatile: IsVolatile);
3002	}
3003
3004	void EmitAggregateCopyCtor(LValue Dest, LValue Src,
3005	AggValueSlot::Overlap_t MayOverlap) {
3006	EmitAggregateCopy(Dest, Src, EltTy: Src.getType(), MayOverlap);
3007	}
3008
3009	/// EmitAggregateCopy - Emit an aggregate copy.
3010	///
3011	/// \param isVolatile \c true iff either the source or the destination is
3012	/// volatile.
3013	/// \param MayOverlap Whether the tail padding of the destination might be
3014	/// occupied by some other object. More efficient code can often be
3015	/// generated if not.
3016	void EmitAggregateCopy(LValue Dest, LValue Src, QualType EltTy,
3017	AggValueSlot::Overlap_t MayOverlap,
3018	bool isVolatile = false);
3019
3020	/// GetAddrOfLocalVar - Return the address of a local variable.
3021	Address GetAddrOfLocalVar(const VarDecl *VD) {
3022	auto it = LocalDeclMap.find(VD);
3023	assert(it != LocalDeclMap.end() &&
3024	"Invalid argument to GetAddrOfLocalVar(), no decl!");
3025	return it->second;
3026	}
3027
3028	/// Given an opaque value expression, return its LValue mapping if it exists,
3029	/// otherwise create one.
3030	LValue getOrCreateOpaqueLValueMapping(const OpaqueValueExpr *e);
3031
3032	/// Given an opaque value expression, return its RValue mapping if it exists,
3033	/// otherwise create one.
3034	RValue getOrCreateOpaqueRValueMapping(const OpaqueValueExpr *e);
3035
3036	/// isOpaqueValueEmitted - Return true if the opaque value expression has
3037	/// already been emitted.
3038	bool isOpaqueValueEmitted(const OpaqueValueExpr *E);
3039
3040	/// Get the index of the current ArrayInitLoopExpr, if any.
3041	llvm::Value *getArrayInitIndex() { return ArrayInitIndex; }
3042
3043	/// getAccessedFieldNo - Given an encoded value and a result number, return
3044	/// the input field number being accessed.
3045	static unsigned getAccessedFieldNo(unsigned Idx, const llvm::Constant *Elts);
3046
3047	llvm::BlockAddress *GetAddrOfLabel(const LabelDecl *L);
3048	llvm::BasicBlock *GetIndirectGotoBlock();
3049
3050	/// Check if \p E is a C++ "this" pointer wrapped in value-preserving casts.
3051	static bool IsWrappedCXXThis(const Expr *E);
3052
3053	/// EmitNullInitialization - Generate code to set a value of the given type to
3054	/// null, If the type contains data member pointers, they will be initialized
3055	/// to -1 in accordance with the Itanium C++ ABI.
3056	void EmitNullInitialization(Address DestPtr, QualType Ty);
3057
3058	/// Emits a call to an LLVM variable-argument intrinsic, either
3059	/// \c llvm.va_start or \c llvm.va_end.
3060	/// \param ArgValue A reference to the \c va_list as emitted by either
3061	/// \c EmitVAListRef or \c EmitMSVAListRef.
3062	/// \param IsStart If \c true, emits a call to \c llvm.va_start; otherwise,
3063	/// calls \c llvm.va_end.
3064	llvm::Value *EmitVAStartEnd(llvm::Value *ArgValue, bool IsStart);
3065
3066	/// Generate code to get an argument from the passed in pointer
3067	/// and update it accordingly.
3068	/// \param VE The \c VAArgExpr for which to generate code.
3069	/// \param VAListAddr Receives a reference to the \c va_list as emitted by
3070	/// either \c EmitVAListRef or \c EmitMSVAListRef.
3071	/// \returns A pointer to the argument.
3072	// FIXME: We should be able to get rid of this method and use the va_arg
3073	// instruction in LLVM instead once it works well enough.
3074	RValue EmitVAArg(VAArgExpr *VE, Address &VAListAddr,
3075	AggValueSlot Slot = AggValueSlot::ignored());
3076
3077	/// emitArrayLength - Compute the length of an array, even if it's a
3078	/// VLA, and drill down to the base element type.
3079	llvm::Value *emitArrayLength(const ArrayType *arrayType, QualType &baseType,
3080	Address &addr);
3081
3082	/// EmitVLASize - Capture all the sizes for the VLA expressions in
3083	/// the given variably-modified type and store them in the VLASizeMap.
3084	///
3085	/// This function can be called with a null (unreachable) insert point.
3086	void EmitVariablyModifiedType(QualType Ty);
3087
3088	struct VlaSizePair {
3089	llvm::Value *NumElts;
3090	QualType Type;
3091
3092	VlaSizePair(llvm::Value *NE, QualType T) : NumElts(NE), Type(T) {}
3093	};
3094
3095	/// Return the number of elements for a single dimension
3096	/// for the given array type.
3097	VlaSizePair getVLAElements1D(const VariableArrayType *vla);
3098	VlaSizePair getVLAElements1D(QualType vla);
3099
3100	/// Returns an LLVM value that corresponds to the size,
3101	/// in non-variably-sized elements, of a variable length array type,
3102	/// plus that largest non-variably-sized element type. Assumes that
3103	/// the type has already been emitted with EmitVariablyModifiedType.
3104	VlaSizePair getVLASize(const VariableArrayType *vla);
3105	VlaSizePair getVLASize(QualType vla);
3106
3107	/// LoadCXXThis - Load the value of 'this'. This function is only valid while
3108	/// generating code for an C++ member function.
3109	llvm::Value *LoadCXXThis() {
3110	assert(CXXThisValue && "no 'this' value for this function");
3111	return CXXThisValue;
3112	}
3113	Address LoadCXXThisAddress();
3114
3115	/// LoadCXXVTT - Load the VTT parameter to base constructors/destructors have
3116	/// virtual bases.
3117	// FIXME: Every place that calls LoadCXXVTT is something
3118	// that needs to be abstracted properly.
3119	llvm::Value *LoadCXXVTT() {
3120	assert(CXXStructorImplicitParamValue && "no VTT value for this function");
3121	return CXXStructorImplicitParamValue;
3122	}
3123
3124	/// GetAddressOfBaseOfCompleteClass - Convert the given pointer to a
3125	/// complete class to the given direct base.
3126	Address GetAddressOfDirectBaseInCompleteClass(Address Value,
3127	const CXXRecordDecl *Derived,
3128	const CXXRecordDecl *Base,
3129	bool BaseIsVirtual);
3130
3131	static bool ShouldNullCheckClassCastValue(const CastExpr *Cast);
3132
3133	/// GetAddressOfBaseClass - This function will add the necessary delta to the
3134	/// load of 'this' and returns address of the base class.
3135	Address GetAddressOfBaseClass(Address Value, const CXXRecordDecl *Derived,
3136	CastExpr::path_const_iterator PathBegin,
3137	CastExpr::path_const_iterator PathEnd,
3138	bool NullCheckValue, SourceLocation Loc);
3139
3140	Address GetAddressOfDerivedClass(Address Value, const CXXRecordDecl *Derived,
3141	CastExpr::path_const_iterator PathBegin,
3142	CastExpr::path_const_iterator PathEnd,
3143	bool NullCheckValue);
3144
3145	/// GetVTTParameter - Return the VTT parameter that should be passed to a
3146	/// base constructor/destructor with virtual bases.
3147	/// FIXME: VTTs are Itanium ABI-specific, so the definition should move
3148	/// to ItaniumCXXABI.cpp together with all the references to VTT.
3149	llvm::Value *GetVTTParameter(GlobalDecl GD, bool ForVirtualBase,
3150	bool Delegating);
3151
3152	void EmitDelegateCXXConstructorCall(const CXXConstructorDecl *Ctor,
3153	CXXCtorType CtorType,
3154	const FunctionArgList &Args,
3155	SourceLocation Loc);
3156	// It's important not to confuse this and the previous function. Delegating
3157	// constructors are the C++0x feature. The constructor delegate optimization
3158	// is used to reduce duplication in the base and complete consturctors where
3159	// they are substantially the same.
3160	void EmitDelegatingCXXConstructorCall(const CXXConstructorDecl *Ctor,
3161	const FunctionArgList &Args);
3162
3163	/// Emit a call to an inheriting constructor (that is, one that invokes a
3164	/// constructor inherited from a base class) by inlining its definition. This
3165	/// is necessary if the ABI does not support forwarding the arguments to the
3166	/// base class constructor (because they're variadic or similar).
3167	void EmitInlinedInheritingCXXConstructorCall(const CXXConstructorDecl *Ctor,
3168	CXXCtorType CtorType,
3169	bool ForVirtualBase,
3170	bool Delegating,
3171	CallArgList &Args);
3172
3173	/// Emit a call to a constructor inherited from a base class, passing the
3174	/// current constructor's arguments along unmodified (without even making
3175	/// a copy).
3176	void EmitInheritedCXXConstructorCall(const CXXConstructorDecl *D,
3177	bool ForVirtualBase, Address This,
3178	bool InheritedFromVBase,
3179	const CXXInheritedCtorInitExpr *E);
3180
3181	void EmitCXXConstructorCall(const CXXConstructorDecl *D, CXXCtorType Type,
3182	bool ForVirtualBase, bool Delegating,
3183	AggValueSlot ThisAVS, const CXXConstructExpr *E);
3184
3185	void EmitCXXConstructorCall(const CXXConstructorDecl *D, CXXCtorType Type,
3186	bool ForVirtualBase, bool Delegating,
3187	Address This, CallArgList &Args,
3188	AggValueSlot::Overlap_t Overlap,
3189	SourceLocation Loc, bool NewPointerIsChecked,
3190	llvm::CallBase **CallOrInvoke = nullptr);
3191
3192	/// Emit assumption load for all bases. Requires to be called only on
3193	/// most-derived class and not under construction of the object.
3194	void EmitVTableAssumptionLoads(const CXXRecordDecl *ClassDecl, Address This);
3195
3196	/// Emit assumption that vptr load == global vtable.
3197	void EmitVTableAssumptionLoad(const VPtr &vptr, Address This);
3198
3199	void EmitSynthesizedCXXCopyCtorCall(const CXXConstructorDecl *D, Address This,
3200	Address Src, const CXXConstructExpr *E);
3201
3202	void EmitCXXAggrConstructorCall(const CXXConstructorDecl *D,
3203	const ArrayType *ArrayTy, Address ArrayPtr,
3204	const CXXConstructExpr *E,
3205	bool NewPointerIsChecked,
3206	bool ZeroInitialization = false);
3207
3208	void EmitCXXAggrConstructorCall(const CXXConstructorDecl *D,
3209	llvm::Value *NumElements, Address ArrayPtr,
3210	const CXXConstructExpr *E,
3211	bool NewPointerIsChecked,
3212	bool ZeroInitialization = false);
3213
3214	static Destroyer destroyCXXObject;
3215
3216	void EmitCXXDestructorCall(const CXXDestructorDecl *D, CXXDtorType Type,
3217	bool ForVirtualBase, bool Delegating, Address This,
3218	QualType ThisTy);
3219
3220	void EmitNewArrayInitializer(const CXXNewExpr *E, QualType elementType,
3221	llvm::Type *ElementTy, Address NewPtr,
3222	llvm::Value *NumElements,
3223	llvm::Value *AllocSizeWithoutCookie);
3224
3225	void EmitCXXTemporary(const CXXTemporary *Temporary, QualType TempType,
3226	Address Ptr);
3227
3228	void EmitSehCppScopeBegin();
3229	void EmitSehCppScopeEnd();
3230	void EmitSehTryScopeBegin();
3231	void EmitSehTryScopeEnd();
3232
3233	llvm::Value *EmitLifetimeStart(llvm::TypeSize Size, llvm::Value *Addr);
3234	void EmitLifetimeEnd(llvm::Value *Size, llvm::Value *Addr);
3235
3236	llvm::Value *EmitCXXNewExpr(const CXXNewExpr *E);
3237	void EmitCXXDeleteExpr(const CXXDeleteExpr *E);
3238
3239	void EmitDeleteCall(const FunctionDecl *DeleteFD, llvm::Value *Ptr,
3240	QualType DeleteTy, llvm::Value *NumElements = nullptr,
3241	CharUnits CookieSize = CharUnits());
3242
3243	RValue EmitBuiltinNewDeleteCall(const FunctionProtoType *Type,
3244	const CallExpr *TheCallExpr, bool IsDelete);
3245
3246	llvm::Value *EmitCXXTypeidExpr(const CXXTypeidExpr *E);
3247	llvm::Value *EmitDynamicCast(Address V, const CXXDynamicCastExpr *DCE);
3248	Address EmitCXXUuidofExpr(const CXXUuidofExpr *E);
3249
3250	/// Situations in which we might emit a check for the suitability of a
3251	/// pointer or glvalue. Needs to be kept in sync with ubsan_handlers.cpp in
3252	/// compiler-rt.
3253	enum TypeCheckKind {
3254	/// Checking the operand of a load. Must be suitably sized and aligned.
3255	TCK_Load,
3256	/// Checking the destination of a store. Must be suitably sized and aligned.
3257	TCK_Store,
3258	/// Checking the bound value in a reference binding. Must be suitably sized
3259	/// and aligned, but is not required to refer to an object (until the
3260	/// reference is used), per core issue 453.
3261	TCK_ReferenceBinding,
3262	/// Checking the object expression in a non-static data member access. Must
3263	/// be an object within its lifetime.
3264	TCK_MemberAccess,
3265	/// Checking the 'this' pointer for a call to a non-static member function.
3266	/// Must be an object within its lifetime.
3267	TCK_MemberCall,
3268	/// Checking the 'this' pointer for a constructor call.
3269	TCK_ConstructorCall,
3270	/// Checking the operand of a static_cast to a derived pointer type. Must be
3271	/// null or an object within its lifetime.
3272	TCK_DowncastPointer,
3273	/// Checking the operand of a static_cast to a derived reference type. Must
3274	/// be an object within its lifetime.
3275	TCK_DowncastReference,
3276	/// Checking the operand of a cast to a base object. Must be suitably sized
3277	/// and aligned.
3278	TCK_Upcast,
3279	/// Checking the operand of a cast to a virtual base object. Must be an
3280	/// object within its lifetime.
3281	TCK_UpcastToVirtualBase,
3282	/// Checking the value assigned to a _Nonnull pointer. Must not be null.
3283	TCK_NonnullAssign,
3284	/// Checking the operand of a dynamic_cast or a typeid expression. Must be
3285	/// null or an object within its lifetime.
3286	TCK_DynamicOperation
3287	};
3288
3289	/// Determine whether the pointer type check \p TCK permits null pointers.
3290	static bool isNullPointerAllowed(TypeCheckKind TCK);
3291
3292	/// Determine whether the pointer type check \p TCK requires a vptr check.
3293	static bool isVptrCheckRequired(TypeCheckKind TCK, QualType Ty);
3294
3295	/// Whether any type-checking sanitizers are enabled. If \c false,
3296	/// calls to EmitTypeCheck can be skipped.
3297	bool sanitizePerformTypeCheck() const;
3298
3299	void EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc, LValue LV,
3300	QualType Type, SanitizerSet SkippedChecks = SanitizerSet(),
3301	llvm::Value *ArraySize = nullptr) {
3302	if (!sanitizePerformTypeCheck())
3303	return;
3304	EmitTypeCheck(TCK, Loc, V: LV.emitRawPointer(CGF&: *this), Type, Alignment: LV.getAlignment(),
3305	SkippedChecks, ArraySize);
3306	}
3307
3308	void EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc, Address Addr,
3309	QualType Type, CharUnits Alignment = CharUnits::Zero(),
3310	SanitizerSet SkippedChecks = SanitizerSet(),
3311	llvm::Value *ArraySize = nullptr) {
3312	if (!sanitizePerformTypeCheck())
3313	return;
3314	EmitTypeCheck(TCK, Loc, V: Addr.emitRawPointer(CGF&: *this), Type, Alignment,
3315	SkippedChecks, ArraySize);
3316	}
3317
3318	/// Emit a check that \p V is the address of storage of the
3319	/// appropriate size and alignment for an object of type \p Type
3320	/// (or if ArraySize is provided, for an array of that bound).
3321	void EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc, llvm::Value *V,
3322	QualType Type, CharUnits Alignment = CharUnits::Zero(),
3323	SanitizerSet SkippedChecks = SanitizerSet(),
3324	llvm::Value *ArraySize = nullptr);
3325
3326	/// Emit a check that \p Base points into an array object, which
3327	/// we can access at index \p Index. \p Accessed should be \c false if we
3328	/// this expression is used as an lvalue, for instance in "&Arr[Idx]".
3329	void EmitBoundsCheck(const Expr *E, const Expr *Base, llvm::Value *Index,
3330	QualType IndexType, bool Accessed);
3331	void EmitBoundsCheckImpl(const Expr *E, llvm::Value *Bound,
3332	llvm::Value *Index, QualType IndexType,
3333	QualType IndexedType, bool Accessed);
3334
3335	/// Returns debug info, with additional annotation if
3336	/// CGM.getCodeGenOpts().SanitizeAnnotateDebugInfo[Ordinal] is enabled for
3337	/// any of the ordinals.
3338	llvm::DILocation *
3339	SanitizerAnnotateDebugInfo(ArrayRef<SanitizerKind::SanitizerOrdinal> Ordinals,
3340	SanitizerHandler Handler);
3341
3342	llvm::Value *GetCountedByFieldExprGEP(const Expr *Base, const FieldDecl *FD,
3343	const FieldDecl *CountDecl);
3344
3345	/// Build an expression accessing the "counted_by" field.
3346	llvm::Value *EmitLoadOfCountedByField(const Expr *Base, const FieldDecl *FD,
3347	const FieldDecl *CountDecl);
3348
3349	// Emit bounds checking for flexible array and pointer members with the
3350	// counted_by attribute.
3351	void EmitCountedByBoundsChecking(const Expr *E, llvm::Value *Idx,
3352	Address Addr, QualType IdxTy,
3353	QualType ArrayTy, bool Accessed,
3354	bool FlexibleArray);
3355
3356	llvm::Value *EmitScalarPrePostIncDec(const UnaryOperator *E, LValue LV,
3357	bool isInc, bool isPre);
3358	ComplexPairTy EmitComplexPrePostIncDec(const UnaryOperator *E, LValue LV,
3359	bool isInc, bool isPre);
3360
3361	/// Converts Location to a DebugLoc, if debug information is enabled.
3362	llvm::DebugLoc SourceLocToDebugLoc(SourceLocation Location);
3363
3364	/// Get the record field index as represented in debug info.
3365	unsigned getDebugInfoFIndex(const RecordDecl *Rec, unsigned FieldIndex);
3366
3367	//===--------------------------------------------------------------------===//
3368	// Declaration Emission
3369	//===--------------------------------------------------------------------===//
3370
3371	/// EmitDecl - Emit a declaration.
3372	///
3373	/// This function can be called with a null (unreachable) insert point.
3374	void EmitDecl(const Decl &D, bool EvaluateConditionDecl = false);
3375
3376	/// EmitVarDecl - Emit a local variable declaration.
3377	///
3378	/// This function can be called with a null (unreachable) insert point.
3379	void EmitVarDecl(const VarDecl &D);
3380
3381	void EmitScalarInit(const Expr *init, const ValueDecl *D, LValue lvalue,
3382	bool capturedByInit);
3383
3384	typedef void SpecialInitFn(CodeGenFunction &Init, const VarDecl &D,
3385	llvm::Value *Address);
3386
3387	/// Determine whether the given initializer is trivial in the sense
3388	/// that it requires no code to be generated.
3389	bool isTrivialInitializer(const Expr *Init);
3390
3391	/// EmitAutoVarDecl - Emit an auto variable declaration.
3392	///
3393	/// This function can be called with a null (unreachable) insert point.
3394	void EmitAutoVarDecl(const VarDecl &D);
3395
3396	class AutoVarEmission {
3397	friend class CodeGenFunction;
3398
3399	const VarDecl *Variable;
3400
3401	/// The address of the alloca for languages with explicit address space
3402	/// (e.g. OpenCL) or alloca casted to generic pointer for address space
3403	/// agnostic languages (e.g. C++). Invalid if the variable was emitted
3404	/// as a global constant.
3405	Address Addr;
3406
3407	llvm::Value *NRVOFlag;
3408
3409	/// True if the variable is a __block variable that is captured by an
3410	/// escaping block.
3411	bool IsEscapingByRef;
3412
3413	/// True if the variable is of aggregate type and has a constant
3414	/// initializer.
3415	bool IsConstantAggregate;
3416
3417	/// Non-null if we should use lifetime annotations.
3418	llvm::Value *SizeForLifetimeMarkers;
3419
3420	/// Address with original alloca instruction. Invalid if the variable was
3421	/// emitted as a global constant.
3422	RawAddress AllocaAddr;
3423
3424	struct Invalid {};
3425	AutoVarEmission(Invalid)
3426	: Variable(nullptr), Addr(Address::invalid()),
3427	AllocaAddr(RawAddress::invalid()) {}
3428
3429	AutoVarEmission(const VarDecl &variable)
3430	: Variable(&variable), Addr(Address::invalid()), NRVOFlag(nullptr),
3431	IsEscapingByRef(false), IsConstantAggregate(false),
3432	SizeForLifetimeMarkers(nullptr), AllocaAddr(RawAddress::invalid()) {}
3433
3434	bool wasEmittedAsGlobal() const { return !Addr.isValid(); }
3435
3436	public:
3437	static AutoVarEmission invalid() { return AutoVarEmission(Invalid()); }
3438
3439	bool useLifetimeMarkers() const {
3440	return SizeForLifetimeMarkers != nullptr;
3441	}
3442	llvm::Value *getSizeForLifetimeMarkers() const {
3443	assert(useLifetimeMarkers());
3444	return SizeForLifetimeMarkers;
3445	}
3446
3447	/// Returns the raw, allocated address, which is not necessarily
3448	/// the address of the object itself. It is casted to default
3449	/// address space for address space agnostic languages.
3450	Address getAllocatedAddress() const { return Addr; }
3451
3452	/// Returns the address for the original alloca instruction.
3453	RawAddress getOriginalAllocatedAddress() const { return AllocaAddr; }
3454
3455	/// Returns the address of the object within this declaration.
3456	/// Note that this does not chase the forwarding pointer for
3457	/// __block decls.
3458	Address getObjectAddress(CodeGenFunction &CGF) const {
3459	if (!IsEscapingByRef)
3460	return Addr;
3461
3462	return CGF.emitBlockByrefAddress(baseAddr: Addr, V: Variable, /*forward*/ followForward: false);
3463	}
3464	};
3465	AutoVarEmission EmitAutoVarAlloca(const VarDecl &var);
3466	void EmitAutoVarInit(const AutoVarEmission &emission);
3467	void EmitAutoVarCleanups(const AutoVarEmission &emission);
3468	void emitAutoVarTypeCleanup(const AutoVarEmission &emission,
3469	QualType::DestructionKind dtorKind);
3470
3471	void MaybeEmitDeferredVarDeclInit(const VarDecl *var);
3472
3473	/// Emits the alloca and debug information for the size expressions for each
3474	/// dimension of an array. It registers the association of its (1-dimensional)
3475	/// QualTypes and size expression's debug node, so that CGDebugInfo can
3476	/// reference this node when creating the DISubrange object to describe the
3477	/// array types.
3478	void EmitAndRegisterVariableArrayDimensions(CGDebugInfo *DI, const VarDecl &D,
3479	bool EmitDebugInfo);
3480
3481	void EmitStaticVarDecl(const VarDecl &D,
3482	llvm::GlobalValue::LinkageTypes Linkage);
3483
3484	class ParamValue {
3485	union {
3486	Address Addr;
3487	llvm::Value *Value;
3488	};
3489
3490	bool IsIndirect;
3491
3492	ParamValue(llvm::Value *V) : Value(V), IsIndirect(false) {}
3493	ParamValue(Address A) : Addr(A), IsIndirect(true) {}
3494
3495	public:
3496	static ParamValue forDirect(llvm::Value *value) {
3497	return ParamValue(value);
3498	}
3499	static ParamValue forIndirect(Address addr) {
3500	assert(!addr.getAlignment().isZero());
3501	return ParamValue(addr);
3502	}
3503
3504	bool isIndirect() const { return IsIndirect; }
3505	llvm::Value *getAnyValue() const {
3506	if (!isIndirect())
3507	return Value;
3508	assert(!Addr.hasOffset() && "unexpected offset");
3509	return Addr.getBasePointer();
3510	}
3511
3512	llvm::Value *getDirectValue() const {
3513	assert(!isIndirect());
3514	return Value;
3515	}
3516
3517	Address getIndirectAddress() const {
3518	assert(isIndirect());
3519	return Addr;
3520	}
3521	};
3522
3523	/// EmitParmDecl - Emit a ParmVarDecl or an ImplicitParamDecl.
3524	void EmitParmDecl(const VarDecl &D, ParamValue Arg, unsigned ArgNo);
3525
3526	/// protectFromPeepholes - Protect a value that we're intending to
3527	/// store to the side, but which will probably be used later, from
3528	/// aggressive peepholing optimizations that might delete it.
3529	///
3530	/// Pass the result to unprotectFromPeepholes to declare that
3531	/// protection is no longer required.
3532	///
3533	/// There's no particular reason why this shouldn't apply to
3534	/// l-values, it's just that no existing peepholes work on pointers.
3535	PeepholeProtection protectFromPeepholes(RValue rvalue);
3536	void unprotectFromPeepholes(PeepholeProtection protection);
3537
3538	void emitAlignmentAssumptionCheck(llvm::Value *Ptr, QualType Ty,
3539	SourceLocation Loc,
3540	SourceLocation AssumptionLoc,
3541	llvm::Value *Alignment,
3542	llvm::Value *OffsetValue,
3543	llvm::Value *TheCheck,
3544	llvm::Instruction *Assumption);
3545
3546	void emitAlignmentAssumption(llvm::Value *PtrValue, QualType Ty,
3547	SourceLocation Loc, SourceLocation AssumptionLoc,
3548	llvm::Value *Alignment,
3549	llvm::Value *OffsetValue = nullptr);
3550
3551	void emitAlignmentAssumption(llvm::Value *PtrValue, const Expr *E,
3552	SourceLocation AssumptionLoc,
3553	llvm::Value *Alignment,
3554	llvm::Value *OffsetValue = nullptr);
3555
3556	//===--------------------------------------------------------------------===//
3557	// Statement Emission
3558	//===--------------------------------------------------------------------===//
3559
3560	/// EmitStopPoint - Emit a debug stoppoint if we are emitting debug info.
3561	void EmitStopPoint(const Stmt *S);
3562
3563	/// EmitStmt - Emit the code for the statement \arg S. It is legal to call
3564	/// this function even if there is no current insertion point.
3565	///
3566	/// This function may clear the current insertion point; callers should use
3567	/// EnsureInsertPoint if they wish to subsequently generate code without first
3568	/// calling EmitBlock, EmitBranch, or EmitStmt.
3569	void EmitStmt(const Stmt *S, ArrayRef<const Attr *> Attrs = {});
3570
3571	/// EmitSimpleStmt - Try to emit a "simple" statement which does not
3572	/// necessarily require an insertion point or debug information; typically
3573	/// because the statement amounts to a jump or a container of other
3574	/// statements.
3575	///
3576	/// \return True if the statement was handled.
3577	bool EmitSimpleStmt(const Stmt *S, ArrayRef<const Attr *> Attrs);
3578
3579	Address EmitCompoundStmt(const CompoundStmt &S, bool GetLast = false,
3580	AggValueSlot AVS = AggValueSlot::ignored());
3581	Address
3582	EmitCompoundStmtWithoutScope(const CompoundStmt &S, bool GetLast = false,
3583	AggValueSlot AVS = AggValueSlot::ignored());
3584
3585	/// EmitLabel - Emit the block for the given label. It is legal to call this
3586	/// function even if there is no current insertion point.
3587	void EmitLabel(const LabelDecl *D); // helper for EmitLabelStmt.
3588
3589	void EmitLabelStmt(const LabelStmt &S);
3590	void EmitAttributedStmt(const AttributedStmt &S);
3591	void EmitGotoStmt(const GotoStmt &S);
3592	void EmitIndirectGotoStmt(const IndirectGotoStmt &S);
3593	void EmitIfStmt(const IfStmt &S);
3594
3595	void EmitWhileStmt(const WhileStmt &S, ArrayRef<const Attr *> Attrs = {});
3596	void EmitDoStmt(const DoStmt &S, ArrayRef<const Attr *> Attrs = {});
3597	void EmitForStmt(const ForStmt &S, ArrayRef<const Attr *> Attrs = {});
3598	void EmitReturnStmt(const ReturnStmt &S);
3599	void EmitDeclStmt(const DeclStmt &S);
3600	void EmitBreakStmt(const BreakStmt &S);
3601	void EmitContinueStmt(const ContinueStmt &S);
3602	void EmitSwitchStmt(const SwitchStmt &S);
3603	void EmitDefaultStmt(const DefaultStmt &S, ArrayRef<const Attr *> Attrs);
3604	void EmitCaseStmt(const CaseStmt &S, ArrayRef<const Attr *> Attrs);
3605	void EmitCaseStmtRange(const CaseStmt &S, ArrayRef<const Attr *> Attrs);
3606	void EmitAsmStmt(const AsmStmt &S);
3607
3608	void EmitObjCForCollectionStmt(const ObjCForCollectionStmt &S);
3609	void EmitObjCAtTryStmt(const ObjCAtTryStmt &S);
3610	void EmitObjCAtThrowStmt(const ObjCAtThrowStmt &S);
3611	void EmitObjCAtSynchronizedStmt(const ObjCAtSynchronizedStmt &S);
3612	void EmitObjCAutoreleasePoolStmt(const ObjCAutoreleasePoolStmt &S);
3613
3614	void EmitCoroutineBody(const CoroutineBodyStmt &S);
3615	void EmitCoreturnStmt(const CoreturnStmt &S);
3616	RValue EmitCoawaitExpr(const CoawaitExpr &E,
3617	AggValueSlot aggSlot = AggValueSlot::ignored(),
3618	bool ignoreResult = false);
3619	LValue EmitCoawaitLValue(const CoawaitExpr *E);
3620	RValue EmitCoyieldExpr(const CoyieldExpr &E,
3621	AggValueSlot aggSlot = AggValueSlot::ignored(),
3622	bool ignoreResult = false);
3623	LValue EmitCoyieldLValue(const CoyieldExpr *E);
3624	RValue EmitCoroutineIntrinsic(const CallExpr *E, unsigned int IID);
3625
3626	void EnterCXXTryStmt(const CXXTryStmt &S, bool IsFnTryBlock = false);
3627	void ExitCXXTryStmt(const CXXTryStmt &S, bool IsFnTryBlock = false);
3628
3629	void EmitCXXTryStmt(const CXXTryStmt &S);
3630	void EmitSEHTryStmt(const SEHTryStmt &S);
3631	void EmitSEHLeaveStmt(const SEHLeaveStmt &S);
3632	void EnterSEHTryStmt(const SEHTryStmt &S);
3633	void ExitSEHTryStmt(const SEHTryStmt &S);
3634	void VolatilizeTryBlocks(llvm::BasicBlock *BB,
3635	llvm::SmallPtrSet<llvm::BasicBlock *, 10> &V);
3636
3637	void pushSEHCleanup(CleanupKind kind, llvm::Function *FinallyFunc);
3638	void startOutlinedSEHHelper(CodeGenFunction &ParentCGF, bool IsFilter,
3639	const Stmt *OutlinedStmt);
3640
3641	llvm::Function *GenerateSEHFilterFunction(CodeGenFunction &ParentCGF,
3642	const SEHExceptStmt &Except);
3643
3644	llvm::Function *GenerateSEHFinallyFunction(CodeGenFunction &ParentCGF,
3645	const SEHFinallyStmt &Finally);
3646
3647	void EmitSEHExceptionCodeSave(CodeGenFunction &ParentCGF,
3648	llvm::Value *ParentFP, llvm::Value *EntryEBP);
3649	llvm::Value *EmitSEHExceptionCode();
3650	llvm::Value *EmitSEHExceptionInfo();
3651	llvm::Value *EmitSEHAbnormalTermination();
3652
3653	/// Emit simple code for OpenMP directives in Simd-only mode.
3654	void EmitSimpleOMPExecutableDirective(const OMPExecutableDirective &D);
3655
3656	/// Scan the outlined statement for captures from the parent function. For
3657	/// each capture, mark the capture as escaped and emit a call to
3658	/// llvm.localrecover. Insert the localrecover result into the LocalDeclMap.
3659	void EmitCapturedLocals(CodeGenFunction &ParentCGF, const Stmt *OutlinedStmt,
3660	bool IsFilter);
3661
3662	/// Recovers the address of a local in a parent function. ParentVar is the
3663	/// address of the variable used in the immediate parent function. It can
3664	/// either be an alloca or a call to llvm.localrecover if there are nested
3665	/// outlined functions. ParentFP is the frame pointer of the outermost parent
3666	/// frame.
3667	Address recoverAddrOfEscapedLocal(CodeGenFunction &ParentCGF,
3668	Address ParentVar, llvm::Value *ParentFP);
3669
3670	void EmitCXXForRangeStmt(const CXXForRangeStmt &S,
3671	ArrayRef<const Attr *> Attrs = {});
3672
3673	/// Controls insertion of cancellation exit blocks in worksharing constructs.
3674	class OMPCancelStackRAII {
3675	CodeGenFunction &CGF;
3676
3677	public:
3678	OMPCancelStackRAII(CodeGenFunction &CGF, OpenMPDirectiveKind Kind,
3679	bool HasCancel)
3680	: CGF(CGF) {
3681	CGF.OMPCancelStack.enter(CGF, Kind, HasCancel);
3682	}
3683	~OMPCancelStackRAII() { CGF.OMPCancelStack.exit(CGF); }
3684	};
3685
3686	/// Returns calculated size of the specified type.
3687	llvm::Value *getTypeSize(QualType Ty);
3688	LValue InitCapturedStruct(const CapturedStmt &S);
3689	llvm::Function *EmitCapturedStmt(const CapturedStmt &S, CapturedRegionKind K);
3690	llvm::Function *GenerateCapturedStmtFunction(const CapturedStmt &S);
3691	Address GenerateCapturedStmtArgument(const CapturedStmt &S);
3692	llvm::Function *GenerateOpenMPCapturedStmtFunction(const CapturedStmt &S,
3693	SourceLocation Loc);
3694	void GenerateOpenMPCapturedVars(const CapturedStmt &S,
3695	SmallVectorImpl<llvm::Value *> &CapturedVars);
3696	void emitOMPSimpleStore(LValue LVal, RValue RVal, QualType RValTy,
3697	SourceLocation Loc);
3698	/// Perform element by element copying of arrays with type \a
3699	/// OriginalType from \a SrcAddr to \a DestAddr using copying procedure
3700	/// generated by \a CopyGen.
3701	///
3702	/// \param DestAddr Address of the destination array.
3703	/// \param SrcAddr Address of the source array.
3704	/// \param OriginalType Type of destination and source arrays.
3705	/// \param CopyGen Copying procedure that copies value of single array element
3706	/// to another single array element.
3707	void EmitOMPAggregateAssign(
3708	Address DestAddr, Address SrcAddr, QualType OriginalType,
3709	const llvm::function_ref<void(Address, Address)> CopyGen);
3710	/// Emit proper copying of data from one variable to another.
3711	///
3712	/// \param OriginalType Original type of the copied variables.
3713	/// \param DestAddr Destination address.
3714	/// \param SrcAddr Source address.
3715	/// \param DestVD Destination variable used in \a CopyExpr (for arrays, has
3716	/// type of the base array element).
3717	/// \param SrcVD Source variable used in \a CopyExpr (for arrays, has type of
3718	/// the base array element).
3719	/// \param Copy Actual copygin expression for copying data from \a SrcVD to \a
3720	/// DestVD.
3721	void EmitOMPCopy(QualType OriginalType, Address DestAddr, Address SrcAddr,
3722	const VarDecl *DestVD, const VarDecl *SrcVD,
3723	const Expr *Copy);
3724	/// Emit atomic update code for constructs: \a X = \a X \a BO \a E or
3725	/// \a X = \a E \a BO \a E.
3726	///
3727	/// \param X Value to be updated.
3728	/// \param E Update value.
3729	/// \param BO Binary operation for update operation.
3730	/// \param IsXLHSInRHSPart true if \a X is LHS in RHS part of the update
3731	/// expression, false otherwise.
3732	/// \param AO Atomic ordering of the generated atomic instructions.
3733	/// \param CommonGen Code generator for complex expressions that cannot be
3734	/// expressed through atomicrmw instruction.
3735	/// \returns <true, OldAtomicValue> if simple 'atomicrmw' instruction was
3736	/// generated, <false, RValue::get(nullptr)> otherwise.
3737	std::pair<bool, RValue> EmitOMPAtomicSimpleUpdateExpr(
3738	LValue X, RValue E, BinaryOperatorKind BO, bool IsXLHSInRHSPart,
3739	llvm::AtomicOrdering AO, SourceLocation Loc,
3740	const llvm::function_ref<RValue(RValue)> CommonGen);
3741	bool EmitOMPFirstprivateClause(const OMPExecutableDirective &D,
3742	OMPPrivateScope &PrivateScope);
3743	void EmitOMPPrivateClause(const OMPExecutableDirective &D,
3744	OMPPrivateScope &PrivateScope);
3745	void EmitOMPUseDevicePtrClause(
3746	const OMPUseDevicePtrClause &C, OMPPrivateScope &PrivateScope,
3747	const llvm::DenseMap<const ValueDecl *, llvm::Value *>
3748	CaptureDeviceAddrMap);
3749	void EmitOMPUseDeviceAddrClause(
3750	const OMPUseDeviceAddrClause &C, OMPPrivateScope &PrivateScope,
3751	const llvm::DenseMap<const ValueDecl *, llvm::Value *>
3752	CaptureDeviceAddrMap);
3753	/// Emit code for copyin clause in \a D directive. The next code is
3754	/// generated at the start of outlined functions for directives:
3755	/// \code
3756	/// threadprivate_var1 = master_threadprivate_var1;
3757	/// operator=(threadprivate_var2, master_threadprivate_var2);
3758	/// ...
3759	/// __kmpc_barrier(&loc, global_tid);
3760	/// \endcode
3761	///
3762	/// \param D OpenMP directive possibly with 'copyin' clause(s).
3763	/// \returns true if at least one copyin variable is found, false otherwise.
3764	bool EmitOMPCopyinClause(const OMPExecutableDirective &D);
3765	/// Emit initial code for lastprivate variables. If some variable is
3766	/// not also firstprivate, then the default initialization is used. Otherwise
3767	/// initialization of this variable is performed by EmitOMPFirstprivateClause
3768	/// method.
3769	///
3770	/// \param D Directive that may have 'lastprivate' directives.
3771	/// \param PrivateScope Private scope for capturing lastprivate variables for
3772	/// proper codegen in internal captured statement.
3773	///
3774	/// \returns true if there is at least one lastprivate variable, false
3775	/// otherwise.
3776	bool EmitOMPLastprivateClauseInit(const OMPExecutableDirective &D,
3777	OMPPrivateScope &PrivateScope);
3778	/// Emit final copying of lastprivate values to original variables at
3779	/// the end of the worksharing or simd directive.
3780	///
3781	/// \param D Directive that has at least one 'lastprivate' directives.
3782	/// \param IsLastIterCond Boolean condition that must be set to 'i1 true' if
3783	/// it is the last iteration of the loop code in associated directive, or to
3784	/// 'i1 false' otherwise. If this item is nullptr, no final check is required.
3785	void EmitOMPLastprivateClauseFinal(const OMPExecutableDirective &D,
3786	bool NoFinals,
3787	llvm::Value *IsLastIterCond = nullptr);
3788	/// Emit initial code for linear clauses.
3789	void EmitOMPLinearClause(const OMPLoopDirective &D,
3790	CodeGenFunction::OMPPrivateScope &PrivateScope);
3791	/// Emit final code for linear clauses.
3792	/// \param CondGen Optional conditional code for final part of codegen for
3793	/// linear clause.
3794	void EmitOMPLinearClauseFinal(
3795	const OMPLoopDirective &D,
3796	const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen);
3797	/// Emit initial code for reduction variables. Creates reduction copies
3798	/// and initializes them with the values according to OpenMP standard.
3799	///
3800	/// \param D Directive (possibly) with the 'reduction' clause.
3801	/// \param PrivateScope Private scope for capturing reduction variables for
3802	/// proper codegen in internal captured statement.
3803	///
3804	void EmitOMPReductionClauseInit(const OMPExecutableDirective &D,
3805	OMPPrivateScope &PrivateScope,
3806	bool ForInscan = false);
3807	/// Emit final update of reduction values to original variables at
3808	/// the end of the directive.
3809	///
3810	/// \param D Directive that has at least one 'reduction' directives.
3811	/// \param ReductionKind The kind of reduction to perform.
3812	void EmitOMPReductionClauseFinal(const OMPExecutableDirective &D,
3813	const OpenMPDirectiveKind ReductionKind);
3814	/// Emit initial code for linear variables. Creates private copies
3815	/// and initializes them with the values according to OpenMP standard.
3816	///
3817	/// \param D Directive (possibly) with the 'linear' clause.
3818	/// \return true if at least one linear variable is found that should be
3819	/// initialized with the value of the original variable, false otherwise.
3820	bool EmitOMPLinearClauseInit(const OMPLoopDirective &D);
3821
3822	typedef const llvm::function_ref<void(CodeGenFunction & /*CGF*/,
3823	llvm::Function * /*OutlinedFn*/,
3824	const OMPTaskDataTy & /*Data*/)>
3825	TaskGenTy;
3826	void EmitOMPTaskBasedDirective(const OMPExecutableDirective &S,
3827	const OpenMPDirectiveKind CapturedRegion,
3828	const RegionCodeGenTy &BodyGen,
3829	const TaskGenTy &TaskGen, OMPTaskDataTy &Data);
3830	struct OMPTargetDataInfo {
3831	Address BasePointersArray = Address::invalid();
3832	Address PointersArray = Address::invalid();
3833	Address SizesArray = Address::invalid();
3834	Address MappersArray = Address::invalid();
3835	unsigned NumberOfTargetItems = 0;
3836	explicit OMPTargetDataInfo() = default;
3837	OMPTargetDataInfo(Address BasePointersArray, Address PointersArray,
3838	Address SizesArray, Address MappersArray,
3839	unsigned NumberOfTargetItems)
3840	: BasePointersArray(BasePointersArray), PointersArray(PointersArray),
3841	SizesArray(SizesArray), MappersArray(MappersArray),
3842	NumberOfTargetItems(NumberOfTargetItems) {}
3843	};
3844	void EmitOMPTargetTaskBasedDirective(const OMPExecutableDirective &S,
3845	const RegionCodeGenTy &BodyGen,
3846	OMPTargetDataInfo &InputInfo);
3847	void processInReduction(const OMPExecutableDirective &S, OMPTaskDataTy &Data,
3848	CodeGenFunction &CGF, const CapturedStmt *CS,
3849	OMPPrivateScope &Scope);
3850	void EmitOMPMetaDirective(const OMPMetaDirective &S);
3851	void EmitOMPParallelDirective(const OMPParallelDirective &S);
3852	void EmitOMPSimdDirective(const OMPSimdDirective &S);
3853	void EmitOMPTileDirective(const OMPTileDirective &S);
3854	void EmitOMPStripeDirective(const OMPStripeDirective &S);
3855	void EmitOMPUnrollDirective(const OMPUnrollDirective &S);
3856	void EmitOMPReverseDirective(const OMPReverseDirective &S);
3857	void EmitOMPInterchangeDirective(const OMPInterchangeDirective &S);
3858	void EmitOMPForDirective(const OMPForDirective &S);
3859	void EmitOMPForSimdDirective(const OMPForSimdDirective &S);
3860	void EmitOMPScopeDirective(const OMPScopeDirective &S);
3861	void EmitOMPSectionsDirective(const OMPSectionsDirective &S);
3862	void EmitOMPSectionDirective(const OMPSectionDirective &S);
3863	void EmitOMPSingleDirective(const OMPSingleDirective &S);
3864	void EmitOMPMasterDirective(const OMPMasterDirective &S);
3865	void EmitOMPMaskedDirective(const OMPMaskedDirective &S);
3866	void EmitOMPCriticalDirective(const OMPCriticalDirective &S);
3867	void EmitOMPParallelForDirective(const OMPParallelForDirective &S);
3868	void EmitOMPParallelForSimdDirective(const OMPParallelForSimdDirective &S);
3869	void EmitOMPParallelSectionsDirective(const OMPParallelSectionsDirective &S);
3870	void EmitOMPParallelMasterDirective(const OMPParallelMasterDirective &S);
3871	void EmitOMPTaskDirective(const OMPTaskDirective &S);
3872	void EmitOMPTaskyieldDirective(const OMPTaskyieldDirective &S);
3873	void EmitOMPErrorDirective(const OMPErrorDirective &S);
3874	void EmitOMPBarrierDirective(const OMPBarrierDirective &S);
3875	void EmitOMPTaskwaitDirective(const OMPTaskwaitDirective &S);
3876	void EmitOMPTaskgroupDirective(const OMPTaskgroupDirective &S);
3877	void EmitOMPFlushDirective(const OMPFlushDirective &S);
3878	void EmitOMPDepobjDirective(const OMPDepobjDirective &S);
3879	void EmitOMPScanDirective(const OMPScanDirective &S);
3880	void EmitOMPOrderedDirective(const OMPOrderedDirective &S);
3881	void EmitOMPAtomicDirective(const OMPAtomicDirective &S);
3882	void EmitOMPTargetDirective(const OMPTargetDirective &S);
3883	void EmitOMPTargetDataDirective(const OMPTargetDataDirective &S);
3884	void EmitOMPTargetEnterDataDirective(const OMPTargetEnterDataDirective &S);
3885	void EmitOMPTargetExitDataDirective(const OMPTargetExitDataDirective &S);
3886	void EmitOMPTargetUpdateDirective(const OMPTargetUpdateDirective &S);
3887	void EmitOMPTargetParallelDirective(const OMPTargetParallelDirective &S);
3888	void
3889	EmitOMPTargetParallelForDirective(const OMPTargetParallelForDirective &S);
3890	void EmitOMPTeamsDirective(const OMPTeamsDirective &S);
3891	void
3892	EmitOMPCancellationPointDirective(const OMPCancellationPointDirective &S);
3893	void EmitOMPCancelDirective(const OMPCancelDirective &S);
3894	void EmitOMPTaskLoopBasedDirective(const OMPLoopDirective &S);
3895	void EmitOMPTaskLoopDirective(const OMPTaskLoopDirective &S);
3896	void EmitOMPTaskLoopSimdDirective(const OMPTaskLoopSimdDirective &S);
3897	void EmitOMPMasterTaskLoopDirective(const OMPMasterTaskLoopDirective &S);
3898	void EmitOMPMaskedTaskLoopDirective(const OMPMaskedTaskLoopDirective &S);
3899	void
3900	EmitOMPMasterTaskLoopSimdDirective(const OMPMasterTaskLoopSimdDirective &S);
3901	void
3902	EmitOMPMaskedTaskLoopSimdDirective(const OMPMaskedTaskLoopSimdDirective &S);
3903	void EmitOMPParallelMasterTaskLoopDirective(
3904	const OMPParallelMasterTaskLoopDirective &S);
3905	void EmitOMPParallelMaskedTaskLoopDirective(
3906	const OMPParallelMaskedTaskLoopDirective &S);
3907	void EmitOMPParallelMasterTaskLoopSimdDirective(
3908	const OMPParallelMasterTaskLoopSimdDirective &S);
3909	void EmitOMPParallelMaskedTaskLoopSimdDirective(
3910	const OMPParallelMaskedTaskLoopSimdDirective &S);
3911	void EmitOMPDistributeDirective(const OMPDistributeDirective &S);
3912	void EmitOMPDistributeParallelForDirective(
3913	const OMPDistributeParallelForDirective &S);
3914	void EmitOMPDistributeParallelForSimdDirective(
3915	const OMPDistributeParallelForSimdDirective &S);
3916	void EmitOMPDistributeSimdDirective(const OMPDistributeSimdDirective &S);
3917	void EmitOMPTargetParallelForSimdDirective(
3918	const OMPTargetParallelForSimdDirective &S);
3919	void EmitOMPTargetSimdDirective(const OMPTargetSimdDirective &S);
3920	void EmitOMPTeamsDistributeDirective(const OMPTeamsDistributeDirective &S);
3921	void
3922	EmitOMPTeamsDistributeSimdDirective(const OMPTeamsDistributeSimdDirective &S);
3923	void EmitOMPTeamsDistributeParallelForSimdDirective(
3924	const OMPTeamsDistributeParallelForSimdDirective &S);
3925	void EmitOMPTeamsDistributeParallelForDirective(
3926	const OMPTeamsDistributeParallelForDirective &S);
3927	void EmitOMPTargetTeamsDirective(const OMPTargetTeamsDirective &S);
3928	void EmitOMPTargetTeamsDistributeDirective(
3929	const OMPTargetTeamsDistributeDirective &S);
3930	void EmitOMPTargetTeamsDistributeParallelForDirective(
3931	const OMPTargetTeamsDistributeParallelForDirective &S);
3932	void EmitOMPTargetTeamsDistributeParallelForSimdDirective(
3933	const OMPTargetTeamsDistributeParallelForSimdDirective &S);
3934	void EmitOMPTargetTeamsDistributeSimdDirective(
3935	const OMPTargetTeamsDistributeSimdDirective &S);
3936	void EmitOMPGenericLoopDirective(const OMPGenericLoopDirective &S);
3937	void EmitOMPParallelGenericLoopDirective(const OMPLoopDirective &S);
3938	void EmitOMPTargetParallelGenericLoopDirective(
3939	const OMPTargetParallelGenericLoopDirective &S);
3940	void EmitOMPTargetTeamsGenericLoopDirective(
3941	const OMPTargetTeamsGenericLoopDirective &S);
3942	void EmitOMPTeamsGenericLoopDirective(const OMPTeamsGenericLoopDirective &S);
3943	void EmitOMPInteropDirective(const OMPInteropDirective &S);
3944	void EmitOMPParallelMaskedDirective(const OMPParallelMaskedDirective &S);
3945	void EmitOMPAssumeDirective(const OMPAssumeDirective &S);
3946
3947	/// Emit device code for the target directive.
3948	static void EmitOMPTargetDeviceFunction(CodeGenModule &CGM,
3949	StringRef ParentName,
3950	const OMPTargetDirective &S);
3951	static void
3952	EmitOMPTargetParallelDeviceFunction(CodeGenModule &CGM, StringRef ParentName,
3953	const OMPTargetParallelDirective &S);
3954	/// Emit device code for the target parallel for directive.
3955	static void EmitOMPTargetParallelForDeviceFunction(
3956	CodeGenModule &CGM, StringRef ParentName,
3957	const OMPTargetParallelForDirective &S);
3958	/// Emit device code for the target parallel for simd directive.
3959	static void EmitOMPTargetParallelForSimdDeviceFunction(
3960	CodeGenModule &CGM, StringRef ParentName,
3961	const OMPTargetParallelForSimdDirective &S);
3962	/// Emit device code for the target teams directive.
3963	static void
3964	EmitOMPTargetTeamsDeviceFunction(CodeGenModule &CGM, StringRef ParentName,
3965	const OMPTargetTeamsDirective &S);
3966	/// Emit device code for the target teams distribute directive.
3967	static void EmitOMPTargetTeamsDistributeDeviceFunction(
3968	CodeGenModule &CGM, StringRef ParentName,
3969	const OMPTargetTeamsDistributeDirective &S);
3970	/// Emit device code for the target teams distribute simd directive.
3971	static void EmitOMPTargetTeamsDistributeSimdDeviceFunction(
3972	CodeGenModule &CGM, StringRef ParentName,
3973	const OMPTargetTeamsDistributeSimdDirective &S);
3974	/// Emit device code for the target simd directive.
3975	static void EmitOMPTargetSimdDeviceFunction(CodeGenModule &CGM,
3976	StringRef ParentName,
3977	const OMPTargetSimdDirective &S);
3978	/// Emit device code for the target teams distribute parallel for simd
3979	/// directive.
3980	static void EmitOMPTargetTeamsDistributeParallelForSimdDeviceFunction(
3981	CodeGenModule &CGM, StringRef ParentName,
3982	const OMPTargetTeamsDistributeParallelForSimdDirective &S);
3983
3984	/// Emit device code for the target teams loop directive.
3985	static void EmitOMPTargetTeamsGenericLoopDeviceFunction(
3986	CodeGenModule &CGM, StringRef ParentName,
3987	const OMPTargetTeamsGenericLoopDirective &S);
3988
3989	/// Emit device code for the target parallel loop directive.
3990	static void EmitOMPTargetParallelGenericLoopDeviceFunction(
3991	CodeGenModule &CGM, StringRef ParentName,
3992	const OMPTargetParallelGenericLoopDirective &S);
3993
3994	static void EmitOMPTargetTeamsDistributeParallelForDeviceFunction(
3995	CodeGenModule &CGM, StringRef ParentName,
3996	const OMPTargetTeamsDistributeParallelForDirective &S);
3997
3998	/// Emit the Stmt \p S and return its topmost canonical loop, if any.
3999	/// TODO: The \p Depth paramter is not yet implemented and must be 1. In the
4000	/// future it is meant to be the number of loops expected in the loop nests
4001	/// (usually specified by the "collapse" clause) that are collapsed to a
4002	/// single loop by this function.
4003	llvm::CanonicalLoopInfo *EmitOMPCollapsedCanonicalLoopNest(const Stmt *S,
4004	int Depth);
4005
4006	/// Emit an OMPCanonicalLoop using the OpenMPIRBuilder.
4007	void EmitOMPCanonicalLoop(const OMPCanonicalLoop *S);
4008
4009	/// Emit inner loop of the worksharing/simd construct.
4010	///
4011	/// \param S Directive, for which the inner loop must be emitted.
4012	/// \param RequiresCleanup true, if directive has some associated private
4013	/// variables.
4014	/// \param LoopCond Bollean condition for loop continuation.
4015	/// \param IncExpr Increment expression for loop control variable.
4016	/// \param BodyGen Generator for the inner body of the inner loop.
4017	/// \param PostIncGen Genrator for post-increment code (required for ordered
4018	/// loop directvies).
4019	void EmitOMPInnerLoop(
4020	const OMPExecutableDirective &S, bool RequiresCleanup,
4021	const Expr *LoopCond, const Expr *IncExpr,
4022	const llvm::function_ref<void(CodeGenFunction &)> BodyGen,
4023	const llvm::function_ref<void(CodeGenFunction &)> PostIncGen);
4024
4025	JumpDest getOMPCancelDestination(OpenMPDirectiveKind Kind);
4026	/// Emit initial code for loop counters of loop-based directives.
4027	void EmitOMPPrivateLoopCounters(const OMPLoopDirective &S,
4028	OMPPrivateScope &LoopScope);
4029
4030	/// Helper for the OpenMP loop directives.
4031	void EmitOMPLoopBody(const OMPLoopDirective &D, JumpDest LoopExit);
4032
4033	/// Emit code for the worksharing loop-based directive.
4034	/// \return true, if this construct has any lastprivate clause, false -
4035	/// otherwise.
4036	bool EmitOMPWorksharingLoop(const OMPLoopDirective &S, Expr *EUB,
4037	const CodeGenLoopBoundsTy &CodeGenLoopBounds,
4038	const CodeGenDispatchBoundsTy &CGDispatchBounds);
4039
4040	/// Emit code for the distribute loop-based directive.
4041	void EmitOMPDistributeLoop(const OMPLoopDirective &S,
4042	const CodeGenLoopTy &CodeGenLoop, Expr *IncExpr);
4043
4044	/// Helpers for the OpenMP loop directives.
4045	void EmitOMPSimdInit(const OMPLoopDirective &D);
4046	void EmitOMPSimdFinal(
4047	const OMPLoopDirective &D,
4048	const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen);
4049
4050	/// Emits the lvalue for the expression with possibly captured variable.
4051	LValue EmitOMPSharedLValue(const Expr *E);
4052
4053	private:
4054	/// Helpers for blocks.
4055	llvm::Value *EmitBlockLiteral(const CGBlockInfo &Info);
4056
4057	/// struct with the values to be passed to the OpenMP loop-related functions
4058	struct OMPLoopArguments {
4059	/// loop lower bound
4060	Address LB = Address::invalid();
4061	/// loop upper bound
4062	Address UB = Address::invalid();
4063	/// loop stride
4064	Address ST = Address::invalid();
4065	/// isLastIteration argument for runtime functions
4066	Address IL = Address::invalid();
4067	/// Chunk value generated by sema
4068	llvm::Value *Chunk = nullptr;
4069	/// EnsureUpperBound
4070	Expr *EUB = nullptr;
4071	/// IncrementExpression
4072	Expr *IncExpr = nullptr;
4073	/// Loop initialization
4074	Expr *Init = nullptr;
4075	/// Loop exit condition
4076	Expr *Cond = nullptr;
4077	/// Update of LB after a whole chunk has been executed
4078	Expr *NextLB = nullptr;
4079	/// Update of UB after a whole chunk has been executed
4080	Expr *NextUB = nullptr;
4081	/// Distinguish between the for distribute and sections
4082	OpenMPDirectiveKind DKind = llvm::omp::OMPD_unknown;
4083	OMPLoopArguments() = default;
4084	OMPLoopArguments(Address LB, Address UB, Address ST, Address IL,
4085	llvm::Value *Chunk = nullptr, Expr *EUB = nullptr,
4086	Expr *IncExpr = nullptr, Expr *Init = nullptr,
4087	Expr *Cond = nullptr, Expr *NextLB = nullptr,
4088	Expr *NextUB = nullptr)
4089	: LB(LB), UB(UB), ST(ST), IL(IL), Chunk(Chunk), EUB(EUB),
4090	IncExpr(IncExpr), Init(Init), Cond(Cond), NextLB(NextLB),
4091	NextUB(NextUB) {}
4092	};
4093	void EmitOMPOuterLoop(bool DynamicOrOrdered, bool IsMonotonic,
4094	const OMPLoopDirective &S, OMPPrivateScope &LoopScope,
4095	const OMPLoopArguments &LoopArgs,
4096	const CodeGenLoopTy &CodeGenLoop,
4097	const CodeGenOrderedTy &CodeGenOrdered);
4098	void EmitOMPForOuterLoop(const OpenMPScheduleTy &ScheduleKind,
4099	bool IsMonotonic, const OMPLoopDirective &S,
4100	OMPPrivateScope &LoopScope, bool Ordered,
4101	const OMPLoopArguments &LoopArgs,
4102	const CodeGenDispatchBoundsTy &CGDispatchBounds);
4103	void EmitOMPDistributeOuterLoop(OpenMPDistScheduleClauseKind ScheduleKind,
4104	const OMPLoopDirective &S,
4105	OMPPrivateScope &LoopScope,
4106	const OMPLoopArguments &LoopArgs,
4107	const CodeGenLoopTy &CodeGenLoopContent);
4108	/// Emit code for sections directive.
4109	void EmitSections(const OMPExecutableDirective &S);
4110
4111	public:
4112	//===--------------------------------------------------------------------===//
4113	// OpenACC Emission
4114	//===--------------------------------------------------------------------===//
4115	void EmitOpenACCComputeConstruct(const OpenACCComputeConstruct &S) {
4116	// TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4117	// simply emitting its structured block, but in the future we will implement
4118	// some sort of IR.
4119	EmitStmt(S: S.getStructuredBlock());
4120	}
4121
4122	void EmitOpenACCLoopConstruct(const OpenACCLoopConstruct &S) {
4123	// TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4124	// simply emitting its loop, but in the future we will implement
4125	// some sort of IR.
4126	EmitStmt(S: S.getLoop());
4127	}
4128
4129	void EmitOpenACCCombinedConstruct(const OpenACCCombinedConstruct &S) {
4130	// TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4131	// simply emitting its loop, but in the future we will implement
4132	// some sort of IR.
4133	EmitStmt(S: S.getLoop());
4134	}
4135
4136	void EmitOpenACCDataConstruct(const OpenACCDataConstruct &S) {
4137	// TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4138	// simply emitting its structured block, but in the future we will implement
4139	// some sort of IR.
4140	EmitStmt(S: S.getStructuredBlock());
4141	}
4142
4143	void EmitOpenACCEnterDataConstruct(const OpenACCEnterDataConstruct &S) {
4144	// TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4145	// but in the future we will implement some sort of IR.
4146	}
4147
4148	void EmitOpenACCExitDataConstruct(const OpenACCExitDataConstruct &S) {
4149	// TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4150	// but in the future we will implement some sort of IR.
4151	}
4152
4153	void EmitOpenACCHostDataConstruct(const OpenACCHostDataConstruct &S) {
4154	// TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4155	// simply emitting its structured block, but in the future we will implement
4156	// some sort of IR.
4157	EmitStmt(S: S.getStructuredBlock());
4158	}
4159
4160	void EmitOpenACCWaitConstruct(const OpenACCWaitConstruct &S) {
4161	// TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4162	// but in the future we will implement some sort of IR.
4163	}
4164
4165	void EmitOpenACCInitConstruct(const OpenACCInitConstruct &S) {
4166	// TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4167	// but in the future we will implement some sort of IR.
4168	}
4169
4170	void EmitOpenACCShutdownConstruct(const OpenACCShutdownConstruct &S) {
4171	// TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4172	// but in the future we will implement some sort of IR.
4173	}
4174
4175	void EmitOpenACCSetConstruct(const OpenACCSetConstruct &S) {
4176	// TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4177	// but in the future we will implement some sort of IR.
4178	}
4179
4180	void EmitOpenACCUpdateConstruct(const OpenACCUpdateConstruct &S) {
4181	// TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4182	// but in the future we will implement some sort of IR.
4183	}
4184
4185	void EmitOpenACCAtomicConstruct(const OpenACCAtomicConstruct &S) {
4186	// TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4187	// simply emitting its associated stmt, but in the future we will implement
4188	// some sort of IR.
4189	EmitStmt(S: S.getAssociatedStmt());
4190	}
4191	void EmitOpenACCCacheConstruct(const OpenACCCacheConstruct &S) {
4192	// TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4193	// but in the future we will implement some sort of IR.
4194	}
4195
4196	//===--------------------------------------------------------------------===//
4197	// LValue Expression Emission
4198	//===--------------------------------------------------------------------===//
4199
4200	/// Create a check that a scalar RValue is non-null.
4201	llvm::Value *EmitNonNullRValueCheck(RValue RV, QualType T);
4202
4203	/// GetUndefRValue - Get an appropriate 'undef' rvalue for the given type.
4204	RValue GetUndefRValue(QualType Ty);
4205
4206	/// EmitUnsupportedRValue - Emit a dummy r-value using the type of E
4207	/// and issue an ErrorUnsupported style diagnostic (using the
4208	/// provided Name).
4209	RValue EmitUnsupportedRValue(const Expr *E, const char *Name);
4210
4211	/// EmitUnsupportedLValue - Emit a dummy l-value using the type of E and issue
4212	/// an ErrorUnsupported style diagnostic (using the provided Name).
4213	LValue EmitUnsupportedLValue(const Expr *E, const char *Name);
4214
4215	/// EmitLValue - Emit code to compute a designator that specifies the location
4216	/// of the expression.
4217	///
4218	/// This can return one of two things: a simple address or a bitfield
4219	/// reference. In either case, the LLVM Value* in the LValue structure is
4220	/// guaranteed to be an LLVM pointer type.
4221	///
4222	/// If this returns a bitfield reference, nothing about the pointee type of
4223	/// the LLVM value is known: For example, it may not be a pointer to an
4224	/// integer.
4225	///
4226	/// If this returns a normal address, and if the lvalue's C type is fixed
4227	/// size, this method guarantees that the returned pointer type will point to
4228	/// an LLVM type of the same size of the lvalue's type. If the lvalue has a
4229	/// variable length type, this is not possible.
4230	///
4231	LValue EmitLValue(const Expr *E,
4232	KnownNonNull_t IsKnownNonNull = NotKnownNonNull);
4233
4234	private:
4235	LValue EmitLValueHelper(const Expr *E, KnownNonNull_t IsKnownNonNull);
4236
4237	public:
4238	/// Same as EmitLValue but additionally we generate checking code to
4239	/// guard against undefined behavior. This is only suitable when we know
4240	/// that the address will be used to access the object.
4241	LValue EmitCheckedLValue(const Expr *E, TypeCheckKind TCK);
4242
4243	RValue convertTempToRValue(Address addr, QualType type, SourceLocation Loc);
4244
4245	void EmitAtomicInit(Expr *E, LValue lvalue);
4246
4247	bool LValueIsSuitableForInlineAtomic(LValue Src);
4248
4249	RValue EmitAtomicLoad(LValue LV, SourceLocation SL,
4250	AggValueSlot Slot = AggValueSlot::ignored());
4251
4252	RValue EmitAtomicLoad(LValue lvalue, SourceLocation loc,
4253	llvm::AtomicOrdering AO, bool IsVolatile = false,
4254	AggValueSlot slot = AggValueSlot::ignored());
4255
4256	void EmitAtomicStore(RValue rvalue, LValue lvalue, bool isInit);
4257
4258	void EmitAtomicStore(RValue rvalue, LValue lvalue, llvm::AtomicOrdering AO,
4259	bool IsVolatile, bool isInit);
4260
4261	std::pair<RValue, llvm::Value *> EmitAtomicCompareExchange(
4262	LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc,
4263	llvm::AtomicOrdering Success =
4264	llvm::AtomicOrdering::SequentiallyConsistent,
4265	llvm::AtomicOrdering Failure =
4266	llvm::AtomicOrdering::SequentiallyConsistent,
4267	bool IsWeak = false, AggValueSlot Slot = AggValueSlot::ignored());
4268
4269	/// Emit an atomicrmw instruction, and applying relevant metadata when
4270	/// applicable.
4271	llvm::AtomicRMWInst *emitAtomicRMWInst(
4272	llvm::AtomicRMWInst::BinOp Op, Address Addr, llvm::Value *Val,
4273	llvm::AtomicOrdering Order = llvm::AtomicOrdering::SequentiallyConsistent,
4274	llvm::SyncScope::ID SSID = llvm::SyncScope::System,
4275	const AtomicExpr *AE = nullptr);
4276
4277	void EmitAtomicUpdate(LValue LVal, llvm::AtomicOrdering AO,
4278	const llvm::function_ref<RValue(RValue)> &UpdateOp,
4279	bool IsVolatile);
4280
4281	/// EmitToMemory - Change a scalar value from its value
4282	/// representation to its in-memory representation.
4283	llvm::Value *EmitToMemory(llvm::Value *Value, QualType Ty);
4284
4285	/// EmitFromMemory - Change a scalar value from its memory
4286	/// representation to its value representation.
4287	llvm::Value *EmitFromMemory(llvm::Value *Value, QualType Ty);
4288
4289	/// Check if the scalar \p Value is within the valid range for the given
4290	/// type \p Ty.
4291	///
4292	/// Returns true if a check is needed (even if the range is unknown).
4293	bool EmitScalarRangeCheck(llvm::Value *Value, QualType Ty,
4294	SourceLocation Loc);
4295
4296	/// EmitLoadOfScalar - Load a scalar value from an address, taking
4297	/// care to appropriately convert from the memory representation to
4298	/// the LLVM value representation.
4299	llvm::Value *EmitLoadOfScalar(Address Addr, bool Volatile, QualType Ty,
4300	SourceLocation Loc,
4301	AlignmentSource Source = AlignmentSource::Type,
4302	bool isNontemporal = false) {
4303	return EmitLoadOfScalar(Addr, Volatile, Ty, Loc, BaseInfo: LValueBaseInfo(Source),
4304	TBAAInfo: CGM.getTBAAAccessInfo(AccessType: Ty), isNontemporal);
4305	}
4306
4307	llvm::Value *EmitLoadOfScalar(Address Addr, bool Volatile, QualType Ty,
4308	SourceLocation Loc, LValueBaseInfo BaseInfo,
4309	TBAAAccessInfo TBAAInfo,
4310	bool isNontemporal = false);
4311
4312	/// EmitLoadOfScalar - Load a scalar value from an address, taking
4313	/// care to appropriately convert from the memory representation to
4314	/// the LLVM value representation. The l-value must be a simple
4315	/// l-value.
4316	llvm::Value *EmitLoadOfScalar(LValue lvalue, SourceLocation Loc);
4317
4318	/// EmitStoreOfScalar - Store a scalar value to an address, taking
4319	/// care to appropriately convert from the memory representation to
4320	/// the LLVM value representation.
4321	void EmitStoreOfScalar(llvm::Value *Value, Address Addr, bool Volatile,
4322	QualType Ty,
4323	AlignmentSource Source = AlignmentSource::Type,
4324	bool isInit = false, bool isNontemporal = false) {
4325	EmitStoreOfScalar(Value, Addr, Volatile, Ty, BaseInfo: LValueBaseInfo(Source),
4326	TBAAInfo: CGM.getTBAAAccessInfo(AccessType: Ty), isInit, isNontemporal);
4327	}
4328
4329	void EmitStoreOfScalar(llvm::Value *Value, Address Addr, bool Volatile,
4330	QualType Ty, LValueBaseInfo BaseInfo,
4331	TBAAAccessInfo TBAAInfo, bool isInit = false,
4332	bool isNontemporal = false);
4333
4334	/// EmitStoreOfScalar - Store a scalar value to an address, taking
4335	/// care to appropriately convert from the memory representation to
4336	/// the LLVM value representation. The l-value must be a simple
4337	/// l-value. The isInit flag indicates whether this is an initialization.
4338	/// If so, atomic qualifiers are ignored and the store is always non-atomic.
4339	void EmitStoreOfScalar(llvm::Value *value, LValue lvalue,
4340	bool isInit = false);
4341
4342	/// EmitLoadOfLValue - Given an expression that represents a value lvalue,
4343	/// this method emits the address of the lvalue, then loads the result as an
4344	/// rvalue, returning the rvalue.
4345	RValue EmitLoadOfLValue(LValue V, SourceLocation Loc);
4346	RValue EmitLoadOfExtVectorElementLValue(LValue V);
4347	RValue EmitLoadOfBitfieldLValue(LValue LV, SourceLocation Loc);
4348	RValue EmitLoadOfGlobalRegLValue(LValue LV);
4349
4350	/// Like EmitLoadOfLValue but also handles complex and aggregate types.
4351	RValue EmitLoadOfAnyValue(LValue V,
4352	AggValueSlot Slot = AggValueSlot::ignored(),
4353	SourceLocation Loc = {});
4354
4355	/// EmitStoreThroughLValue - Store the specified rvalue into the specified
4356	/// lvalue, where both are guaranteed to the have the same type, and that type
4357	/// is 'Ty'.
4358	void EmitStoreThroughLValue(RValue Src, LValue Dst, bool isInit = false);
4359	void EmitStoreThroughExtVectorComponentLValue(RValue Src, LValue Dst);
4360	void EmitStoreThroughGlobalRegLValue(RValue Src, LValue Dst);
4361
4362	/// EmitStoreThroughBitfieldLValue - Store Src into Dst with same constraints
4363	/// as EmitStoreThroughLValue.
4364	///
4365	/// \param Result [out] - If non-null, this will be set to a Value* for the
4366	/// bit-field contents after the store, appropriate for use as the result of
4367	/// an assignment to the bit-field.
4368	void EmitStoreThroughBitfieldLValue(RValue Src, LValue Dst,
4369	llvm::Value **Result = nullptr);
4370
4371	/// Emit an l-value for an assignment (simple or compound) of complex type.
4372	LValue EmitComplexAssignmentLValue(const BinaryOperator *E);
4373	LValue EmitComplexCompoundAssignmentLValue(const CompoundAssignOperator *E);
4374	LValue EmitScalarCompoundAssignWithComplex(const CompoundAssignOperator *E,
4375	llvm::Value *&Result);
4376
4377	// Note: only available for agg return types
4378	LValue EmitBinaryOperatorLValue(const BinaryOperator *E);
4379	LValue EmitCompoundAssignmentLValue(const CompoundAssignOperator *E);
4380	// Note: only available for agg return types
4381	LValue EmitCallExprLValue(const CallExpr *E,
4382	llvm::CallBase **CallOrInvoke = nullptr);
4383	// Note: only available for agg return types
4384	LValue EmitVAArgExprLValue(const VAArgExpr *E);
4385	LValue EmitDeclRefLValue(const DeclRefExpr *E);
4386	LValue EmitStringLiteralLValue(const StringLiteral *E);
4387	LValue EmitObjCEncodeExprLValue(const ObjCEncodeExpr *E);
4388	LValue EmitPredefinedLValue(const PredefinedExpr *E);
4389	LValue EmitUnaryOpLValue(const UnaryOperator *E);
4390	LValue EmitArraySubscriptExpr(const ArraySubscriptExpr *E,
4391	bool Accessed = false);
4392	llvm::Value *EmitMatrixIndexExpr(const Expr *E);
4393	LValue EmitMatrixSubscriptExpr(const MatrixSubscriptExpr *E);
4394	LValue EmitArraySectionExpr(const ArraySectionExpr *E,
4395	bool IsLowerBound = true);
4396	LValue EmitExtVectorElementExpr(const ExtVectorElementExpr *E);
4397	LValue EmitMemberExpr(const MemberExpr *E);
4398	LValue EmitObjCIsaExpr(const ObjCIsaExpr *E);
4399	LValue EmitCompoundLiteralLValue(const CompoundLiteralExpr *E);
4400	LValue EmitInitListLValue(const InitListExpr *E);
4401	void EmitIgnoredConditionalOperator(const AbstractConditionalOperator *E);
4402	LValue EmitConditionalOperatorLValue(const AbstractConditionalOperator *E);
4403	LValue EmitCastLValue(const CastExpr *E);
4404	LValue EmitMaterializeTemporaryExpr(const MaterializeTemporaryExpr *E);
4405	LValue EmitOpaqueValueLValue(const OpaqueValueExpr *e);
4406	LValue EmitHLSLArrayAssignLValue(const BinaryOperator *E);
4407
4408	std::pair<LValue, LValue> EmitHLSLOutArgLValues(const HLSLOutArgExpr *E,
4409	QualType Ty);
4410	LValue EmitHLSLOutArgExpr(const HLSLOutArgExpr *E, CallArgList &Args,
4411	QualType Ty);
4412
4413	Address EmitExtVectorElementLValue(LValue V);
4414
4415	RValue EmitRValueForField(LValue LV, const FieldDecl *FD, SourceLocation Loc);
4416
4417	Address EmitArrayToPointerDecay(const Expr *Array,
4418	LValueBaseInfo *BaseInfo = nullptr,
4419	TBAAAccessInfo *TBAAInfo = nullptr);
4420
4421	class ConstantEmission {
4422	llvm::PointerIntPair<llvm::Constant *, 1, bool> ValueAndIsReference;
4423	ConstantEmission(llvm::Constant *C, bool isReference)
4424	: ValueAndIsReference(C, isReference) {}
4425
4426	public:
4427	ConstantEmission() {}
4428	static ConstantEmission forReference(llvm::Constant *C) {
4429	return ConstantEmission(C, true);
4430	}
4431	static ConstantEmission forValue(llvm::Constant *C) {
4432	return ConstantEmission(C, false);
4433	}
4434
4435	explicit operator bool() const {
4436	return ValueAndIsReference.getOpaqueValue() != nullptr;
4437	}
4438
4439	bool isReference() const { return ValueAndIsReference.getInt(); }
4440	LValue getReferenceLValue(CodeGenFunction &CGF, const Expr *RefExpr) const {
4441	assert(isReference());
4442	return CGF.MakeNaturalAlignAddrLValue(V: ValueAndIsReference.getPointer(),
4443	T: RefExpr->getType());
4444	}
4445
4446	llvm::Constant *getValue() const {
4447	assert(!isReference());
4448	return ValueAndIsReference.getPointer();
4449	}
4450	};
4451
4452	ConstantEmission tryEmitAsConstant(const DeclRefExpr *RefExpr);
4453	ConstantEmission tryEmitAsConstant(const MemberExpr *ME);
4454	llvm::Value *emitScalarConstant(const ConstantEmission &Constant, Expr *E);
4455
4456	RValue EmitPseudoObjectRValue(const PseudoObjectExpr *e,
4457	AggValueSlot slot = AggValueSlot::ignored());
4458	LValue EmitPseudoObjectLValue(const PseudoObjectExpr *e);
4459
4460	void FlattenAccessAndType(
4461	Address Addr, QualType AddrTy,
4462	SmallVectorImpl<std::pair<Address, llvm::Value *>> &AccessList,
4463	SmallVectorImpl<QualType> &FlatTypes);
4464
4465	llvm::Value *EmitIvarOffset(const ObjCInterfaceDecl *Interface,
4466	const ObjCIvarDecl *Ivar);
4467	llvm::Value *EmitIvarOffsetAsPointerDiff(const ObjCInterfaceDecl *Interface,
4468	const ObjCIvarDecl *Ivar);
4469	LValue EmitLValueForField(LValue Base, const FieldDecl *Field,
4470	bool IsInBounds = true);
4471	LValue EmitLValueForLambdaField(const FieldDecl *Field);
4472	LValue EmitLValueForLambdaField(const FieldDecl *Field,
4473	llvm::Value *ThisValue);
4474
4475	/// EmitLValueForFieldInitialization - Like EmitLValueForField, except that
4476	/// if the Field is a reference, this will return the address of the reference
4477	/// and not the address of the value stored in the reference.
4478	LValue EmitLValueForFieldInitialization(LValue Base, const FieldDecl *Field);
4479
4480	LValue EmitLValueForIvar(QualType ObjectTy, llvm::Value *Base,
4481	const ObjCIvarDecl *Ivar, unsigned CVRQualifiers);
4482
4483	LValue EmitCXXConstructLValue(const CXXConstructExpr *E);
4484	LValue EmitCXXBindTemporaryLValue(const CXXBindTemporaryExpr *E);
4485	LValue EmitCXXTypeidLValue(const CXXTypeidExpr *E);
4486	LValue EmitCXXUuidofLValue(const CXXUuidofExpr *E);
4487
4488	LValue EmitObjCMessageExprLValue(const ObjCMessageExpr *E);
4489	LValue EmitObjCIvarRefLValue(const ObjCIvarRefExpr *E);
4490	LValue EmitStmtExprLValue(const StmtExpr *E);
4491	LValue EmitPointerToDataMemberBinaryExpr(const BinaryOperator *E);
4492	LValue EmitObjCSelectorLValue(const ObjCSelectorExpr *E);
4493	void EmitDeclRefExprDbgValue(const DeclRefExpr *E, const APValue &Init);
4494
4495	//===--------------------------------------------------------------------===//
4496	// Scalar Expression Emission
4497	//===--------------------------------------------------------------------===//
4498
4499	/// EmitCall - Generate a call of the given function, expecting the given
4500	/// result type, and using the given argument list which specifies both the
4501	/// LLVM arguments and the types they were derived from.
4502	RValue EmitCall(const CGFunctionInfo &CallInfo, const CGCallee &Callee,
4503	ReturnValueSlot ReturnValue, const CallArgList &Args,
4504	llvm::CallBase **CallOrInvoke, bool IsMustTail,
4505	SourceLocation Loc,
4506	bool IsVirtualFunctionPointerThunk = false);
4507	RValue EmitCall(const CGFunctionInfo &CallInfo, const CGCallee &Callee,
4508	ReturnValueSlot ReturnValue, const CallArgList &Args,
4509	llvm::CallBase **CallOrInvoke = nullptr,
4510	bool IsMustTail = false) {
4511	return EmitCall(CallInfo, Callee, ReturnValue, Args, CallOrInvoke,
4512	IsMustTail, Loc: SourceLocation());
4513	}
4514	RValue EmitCall(QualType FnType, const CGCallee &Callee, const CallExpr *E,
4515	ReturnValueSlot ReturnValue, llvm::Value *Chain = nullptr,
4516	llvm::CallBase **CallOrInvoke = nullptr,
4517	CGFunctionInfo const **ResolvedFnInfo = nullptr);
4518
4519	// If a Call or Invoke instruction was emitted for this CallExpr, this method
4520	// writes the pointer to `CallOrInvoke` if it's not null.
4521	RValue EmitCallExpr(const CallExpr *E,
4522	ReturnValueSlot ReturnValue = ReturnValueSlot(),
4523	llvm::CallBase **CallOrInvoke = nullptr);
4524	RValue EmitSimpleCallExpr(const CallExpr *E, ReturnValueSlot ReturnValue,
4525	llvm::CallBase **CallOrInvoke = nullptr);
4526	CGCallee EmitCallee(const Expr *E);
4527
4528	void checkTargetFeatures(const CallExpr *E, const FunctionDecl *TargetDecl);
4529	void checkTargetFeatures(SourceLocation Loc, const FunctionDecl *TargetDecl);
4530
4531	llvm::CallInst *EmitRuntimeCall(llvm::FunctionCallee callee,
4532	const Twine &name = "");
4533	llvm::CallInst *EmitRuntimeCall(llvm::FunctionCallee callee,
4534	ArrayRef<llvm::Value *> args,
4535	const Twine &name = "");
4536	llvm::CallInst *EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
4537	const Twine &name = "");
4538	llvm::CallInst *EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
4539	ArrayRef<Address> args,
4540	const Twine &name = "");
4541	llvm::CallInst *EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
4542	ArrayRef<llvm::Value *> args,
4543	const Twine &name = "");
4544
4545	SmallVector<llvm::OperandBundleDef, 1>
4546	getBundlesForFunclet(llvm::Value *Callee);
4547
4548	llvm::CallBase *EmitCallOrInvoke(llvm::FunctionCallee Callee,
4549	ArrayRef<llvm::Value *> Args,
4550	const Twine &Name = "");
4551	llvm::CallBase *EmitRuntimeCallOrInvoke(llvm::FunctionCallee callee,
4552	ArrayRef<llvm::Value *> args,
4553	const Twine &name = "");
4554	llvm::CallBase *EmitRuntimeCallOrInvoke(llvm::FunctionCallee callee,
4555	const Twine &name = "");
4556	void EmitNoreturnRuntimeCallOrInvoke(llvm::FunctionCallee callee,
4557	ArrayRef<llvm::Value *> args);
4558
4559	CGCallee BuildAppleKextVirtualCall(const CXXMethodDecl *MD,
4560	NestedNameSpecifier *Qual, llvm::Type *Ty);
4561
4562	CGCallee BuildAppleKextVirtualDestructorCall(const CXXDestructorDecl *DD,
4563	CXXDtorType Type,
4564	const CXXRecordDecl *RD);
4565
4566	bool isPointerKnownNonNull(const Expr *E);
4567	/// Check whether the underlying base pointer is a constant null.
4568	bool isUnderlyingBasePointerConstantNull(const Expr *E);
4569
4570	/// Create the discriminator from the storage address and the entity hash.
4571	llvm::Value *EmitPointerAuthBlendDiscriminator(llvm::Value *StorageAddress,
4572	llvm::Value *Discriminator);
4573	CGPointerAuthInfo EmitPointerAuthInfo(const PointerAuthSchema &Schema,
4574	llvm::Value *StorageAddress,
4575	GlobalDecl SchemaDecl,
4576	QualType SchemaType);
4577
4578	llvm::Value *EmitPointerAuthSign(const CGPointerAuthInfo &Info,
4579	llvm::Value *Pointer);
4580
4581	llvm::Value *EmitPointerAuthAuth(const CGPointerAuthInfo &Info,
4582	llvm::Value *Pointer);
4583
4584	llvm::Value *emitPointerAuthResign(llvm::Value *Pointer, QualType PointerType,
4585	const CGPointerAuthInfo &CurAuthInfo,
4586	const CGPointerAuthInfo &NewAuthInfo,
4587	bool IsKnownNonNull);
4588	llvm::Value *emitPointerAuthResignCall(llvm::Value *Pointer,
4589	const CGPointerAuthInfo &CurInfo,
4590	const CGPointerAuthInfo &NewInfo);
4591
4592	void EmitPointerAuthOperandBundle(
4593	const CGPointerAuthInfo &Info,
4594	SmallVectorImpl<llvm::OperandBundleDef> &Bundles);
4595
4596	CGPointerAuthInfo EmitPointerAuthInfo(PointerAuthQualifier Qualifier,
4597	Address StorageAddress);
4598	llvm::Value *EmitPointerAuthQualify(PointerAuthQualifier Qualifier,
4599	llvm::Value *Pointer, QualType ValueType,
4600	Address StorageAddress,
4601	bool IsKnownNonNull);
4602	llvm::Value *EmitPointerAuthQualify(PointerAuthQualifier Qualifier,
4603	const Expr *PointerExpr,
4604	Address StorageAddress);
4605	llvm::Value *EmitPointerAuthUnqualify(PointerAuthQualifier Qualifier,
4606	llvm::Value *Pointer,
4607	QualType PointerType,
4608	Address StorageAddress,
4609	bool IsKnownNonNull);
4610	void EmitPointerAuthCopy(PointerAuthQualifier Qualifier, QualType Type,
4611	Address DestField, Address SrcField);
4612
4613	std::pair<llvm::Value *, CGPointerAuthInfo>
4614	EmitOrigPointerRValue(const Expr *E);
4615
4616	llvm::Value *authPointerToPointerCast(llvm::Value *ResultPtr,
4617	QualType SourceType, QualType DestType);
4618	Address authPointerToPointerCast(Address Ptr, QualType SourceType,
4619	QualType DestType);
4620
4621	Address getAsNaturalAddressOf(Address Addr, QualType PointeeTy);
4622
4623	llvm::Value *getAsNaturalPointerTo(Address Addr, QualType PointeeType) {
4624	return getAsNaturalAddressOf(Addr, PointeeTy: PointeeType).getBasePointer();
4625	}
4626
4627	// Return the copy constructor name with the prefix "__copy_constructor_"
4628	// removed.
4629	static std::string getNonTrivialCopyConstructorStr(QualType QT,
4630	CharUnits Alignment,
4631	bool IsVolatile,
4632	ASTContext &Ctx);
4633
4634	// Return the destructor name with the prefix "__destructor_" removed.
4635	static std::string getNonTrivialDestructorStr(QualType QT,
4636	CharUnits Alignment,
4637	bool IsVolatile,
4638	ASTContext &Ctx);
4639
4640	// These functions emit calls to the special functions of non-trivial C
4641	// structs.
4642	void defaultInitNonTrivialCStructVar(LValue Dst);
4643	void callCStructDefaultConstructor(LValue Dst);
4644	void callCStructDestructor(LValue Dst);
4645	void callCStructCopyConstructor(LValue Dst, LValue Src);
4646	void callCStructMoveConstructor(LValue Dst, LValue Src);
4647	void callCStructCopyAssignmentOperator(LValue Dst, LValue Src);
4648	void callCStructMoveAssignmentOperator(LValue Dst, LValue Src);
4649
4650	RValue EmitCXXMemberOrOperatorCall(
4651	const CXXMethodDecl *Method, const CGCallee &Callee,
4652	ReturnValueSlot ReturnValue, llvm::Value *This,
4653	llvm::Value *ImplicitParam, QualType ImplicitParamTy, const CallExpr *E,
4654	CallArgList *RtlArgs, llvm::CallBase **CallOrInvoke);
4655	RValue EmitCXXDestructorCall(GlobalDecl Dtor, const CGCallee &Callee,
4656	llvm::Value *This, QualType ThisTy,
4657	llvm::Value *ImplicitParam,
4658	QualType ImplicitParamTy, const CallExpr *E,
4659	llvm::CallBase **CallOrInvoke = nullptr);
4660	RValue EmitCXXMemberCallExpr(const CXXMemberCallExpr *E,
4661	ReturnValueSlot ReturnValue,
4662	llvm::CallBase **CallOrInvoke = nullptr);
4663	RValue EmitCXXMemberOrOperatorMemberCallExpr(
4664	const CallExpr *CE, const CXXMethodDecl *MD, ReturnValueSlot ReturnValue,
4665	bool HasQualifier, NestedNameSpecifier *Qualifier, bool IsArrow,
4666	const Expr *Base, llvm::CallBase **CallOrInvoke);
4667	// Compute the object pointer.
4668	Address EmitCXXMemberDataPointerAddress(
4669	const Expr *E, Address base, llvm::Value *memberPtr,
4670	const MemberPointerType *memberPtrType, bool IsInBounds,
4671	LValueBaseInfo *BaseInfo = nullptr, TBAAAccessInfo *TBAAInfo = nullptr);
4672	RValue EmitCXXMemberPointerCallExpr(const CXXMemberCallExpr *E,
4673	ReturnValueSlot ReturnValue,
4674	llvm::CallBase **CallOrInvoke);
4675
4676	RValue EmitCXXOperatorMemberCallExpr(const CXXOperatorCallExpr *E,
4677	const CXXMethodDecl *MD,
4678	ReturnValueSlot ReturnValue,
4679	llvm::CallBase **CallOrInvoke);
4680	RValue EmitCXXPseudoDestructorExpr(const CXXPseudoDestructorExpr *E);
4681
4682	RValue EmitCUDAKernelCallExpr(const CUDAKernelCallExpr *E,
4683	ReturnValueSlot ReturnValue,
4684	llvm::CallBase **CallOrInvoke);
4685
4686	RValue EmitNVPTXDevicePrintfCallExpr(const CallExpr *E);
4687	RValue EmitAMDGPUDevicePrintfCallExpr(const CallExpr *E);
4688
4689	RValue EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
4690	const CallExpr *E, ReturnValueSlot ReturnValue);
4691
4692	RValue emitRotate(const CallExpr *E, bool IsRotateRight);
4693
4694	/// Emit IR for __builtin_os_log_format.
4695	RValue emitBuiltinOSLogFormat(const CallExpr &E);
4696
4697	/// Emit IR for __builtin_is_aligned.
4698	RValue EmitBuiltinIsAligned(const CallExpr *E);
4699	/// Emit IR for __builtin_align_up/__builtin_align_down.
4700	RValue EmitBuiltinAlignTo(const CallExpr *E, bool AlignUp);
4701
4702	llvm::Function *generateBuiltinOSLogHelperFunction(
4703	const analyze_os_log::OSLogBufferLayout &Layout,
4704	CharUnits BufferAlignment);
4705
4706	RValue EmitBlockCallExpr(const CallExpr *E, ReturnValueSlot ReturnValue,
4707	llvm::CallBase **CallOrInvoke);
4708
4709	/// EmitTargetBuiltinExpr - Emit the given builtin call. Returns 0 if the call
4710	/// is unhandled by the current target.
4711	llvm::Value *EmitTargetBuiltinExpr(unsigned BuiltinID, const CallExpr *E,
4712	ReturnValueSlot ReturnValue);
4713
4714	llvm::Value *
4715	EmitAArch64CompareBuiltinExpr(llvm::Value *Op, llvm::Type *Ty,
4716	const llvm::CmpInst::Predicate Pred,
4717	const llvm::Twine &Name = "");
4718	llvm::Value *EmitARMBuiltinExpr(unsigned BuiltinID, const CallExpr *E,
4719	ReturnValueSlot ReturnValue,
4720	llvm::Triple::ArchType Arch);
4721	llvm::Value *EmitARMMVEBuiltinExpr(unsigned BuiltinID, const CallExpr *E,
4722	ReturnValueSlot ReturnValue,
4723	llvm::Triple::ArchType Arch);
4724	llvm::Value *EmitARMCDEBuiltinExpr(unsigned BuiltinID, const CallExpr *E,
4725	ReturnValueSlot ReturnValue,
4726	llvm::Triple::ArchType Arch);
4727	llvm::Value *EmitCMSEClearRecord(llvm::Value *V, llvm::IntegerType *ITy,
4728	QualType RTy);
4729	llvm::Value *EmitCMSEClearRecord(llvm::Value *V, llvm::ArrayType *ATy,
4730	QualType RTy);
4731
4732	llvm::Value *
4733	EmitCommonNeonBuiltinExpr(unsigned BuiltinID, unsigned LLVMIntrinsic,
4734	unsigned AltLLVMIntrinsic, const char *NameHint,
4735	unsigned Modifier, const CallExpr *E,
4736	SmallVectorImpl<llvm::Value *> &Ops, Address PtrOp0,
4737	Address PtrOp1, llvm::Triple::ArchType Arch);
4738
4739	llvm::Function *LookupNeonLLVMIntrinsic(unsigned IntrinsicID,
4740	unsigned Modifier, llvm::Type *ArgTy,
4741	const CallExpr *E);
4742	llvm::Value *EmitNeonCall(llvm::Function *F,
4743	SmallVectorImpl<llvm::Value *> &O, const char *name,
4744	unsigned shift = 0, bool rightshift = false);
4745	llvm::Value *EmitFP8NeonCall(unsigned IID, ArrayRef<llvm::Type *> Tys,
4746	SmallVectorImpl<llvm::Value *> &O,
4747	const CallExpr *E, const char *name);
4748	llvm::Value *EmitFP8NeonCvtCall(unsigned IID, llvm::Type *Ty0,
4749	llvm::Type *Ty1, bool Extract,
4750	SmallVectorImpl<llvm::Value *> &Ops,
4751	const CallExpr *E, const char *name);
4752	llvm::Value *EmitFP8NeonFDOTCall(unsigned IID, bool ExtendLaneArg,
4753	llvm::Type *RetTy,
4754	SmallVectorImpl<llvm::Value *> &Ops,
4755	const CallExpr *E, const char *name);
4756	llvm::Value *EmitFP8NeonFMLACall(unsigned IID, bool ExtendLaneArg,
4757	llvm::Type *RetTy,
4758	SmallVectorImpl<llvm::Value *> &Ops,
4759	const CallExpr *E, const char *name);
4760	llvm::Value *EmitNeonSplat(llvm::Value *V, llvm::Constant *Idx,
4761	const llvm::ElementCount &Count);
4762	llvm::Value *EmitNeonSplat(llvm::Value *V, llvm::Constant *Idx);
4763	llvm::Value *EmitNeonShiftVector(llvm::Value *V, llvm::Type *Ty,
4764	bool negateForRightShift);
4765	llvm::Value *EmitNeonRShiftImm(llvm::Value *Vec, llvm::Value *Amt,
4766	llvm::Type *Ty, bool usgn, const char *name);
4767	llvm::Value *vectorWrapScalar16(llvm::Value *Op);
4768	/// SVEBuiltinMemEltTy - Returns the memory element type for this memory
4769	/// access builtin. Only required if it can't be inferred from the base
4770	/// pointer operand.
4771	llvm::Type *SVEBuiltinMemEltTy(const SVETypeFlags &TypeFlags);
4772
4773	SmallVector<llvm::Type *, 2>
4774	getSVEOverloadTypes(const SVETypeFlags &TypeFlags, llvm::Type *ReturnType,
4775	ArrayRef<llvm::Value *> Ops);
4776	llvm::Type *getEltType(const SVETypeFlags &TypeFlags);
4777	llvm::ScalableVectorType *getSVEType(const SVETypeFlags &TypeFlags);
4778	llvm::ScalableVectorType *getSVEPredType(const SVETypeFlags &TypeFlags);
4779	llvm::Value *EmitSVETupleSetOrGet(const SVETypeFlags &TypeFlags,
4780	ArrayRef<llvm::Value *> Ops);
4781	llvm::Value *EmitSVETupleCreate(const SVETypeFlags &TypeFlags,
4782	llvm::Type *ReturnType,
4783	ArrayRef<llvm::Value *> Ops);
4784	llvm::Value *EmitSVEAllTruePred(const SVETypeFlags &TypeFlags);
4785	llvm::Value *EmitSVEDupX(llvm::Value *Scalar);
4786	llvm::Value *EmitSVEDupX(llvm::Value *Scalar, llvm::Type *Ty);
4787	llvm::Value *EmitSVEReinterpret(llvm::Value *Val, llvm::Type *Ty);
4788	llvm::Value *EmitSVEPMull(const SVETypeFlags &TypeFlags,
4789	llvm::SmallVectorImpl<llvm::Value *> &Ops,
4790	unsigned BuiltinID);
4791	llvm::Value *EmitSVEMovl(const SVETypeFlags &TypeFlags,
4792	llvm::ArrayRef<llvm::Value *> Ops,
4793	unsigned BuiltinID);
4794	llvm::Value *EmitSVEPredicateCast(llvm::Value *Pred,
4795	llvm::ScalableVectorType *VTy);
4796	llvm::Value *EmitSVEPredicateTupleCast(llvm::Value *PredTuple,
4797	llvm::StructType *Ty);
4798	llvm::Value *EmitSVEGatherLoad(const SVETypeFlags &TypeFlags,
4799	llvm::SmallVectorImpl<llvm::Value *> &Ops,
4800	unsigned IntID);
4801	llvm::Value *EmitSVEScatterStore(const SVETypeFlags &TypeFlags,
4802	llvm::SmallVectorImpl<llvm::Value *> &Ops,
4803	unsigned IntID);
4804	llvm::Value *EmitSVEMaskedLoad(const CallExpr *, llvm::Type *ReturnTy,
4805	SmallVectorImpl<llvm::Value *> &Ops,
4806	unsigned BuiltinID, bool IsZExtReturn);
4807	llvm::Value *EmitSVEMaskedStore(const CallExpr *,
4808	SmallVectorImpl<llvm::Value *> &Ops,
4809	unsigned BuiltinID);
4810	llvm::Value *EmitSVEPrefetchLoad(const SVETypeFlags &TypeFlags,
4811	SmallVectorImpl<llvm::Value *> &Ops,
4812	unsigned BuiltinID);
4813	llvm::Value *EmitSVEGatherPrefetch(const SVETypeFlags &TypeFlags,
4814	SmallVectorImpl<llvm::Value *> &Ops,
4815	unsigned IntID);
4816	llvm::Value *EmitSVEStructLoad(const SVETypeFlags &TypeFlags,
4817	SmallVectorImpl<llvm::Value *> &Ops,
4818	unsigned IntID);
4819	llvm::Value *EmitSVEStructStore(const SVETypeFlags &TypeFlags,
4820	SmallVectorImpl<llvm::Value *> &Ops,
4821	unsigned IntID);
4822	llvm::Value *EmitAArch64SVEBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
4823
4824	llvm::Value *EmitSMELd1St1(const SVETypeFlags &TypeFlags,
4825	llvm::SmallVectorImpl<llvm::Value *> &Ops,
4826	unsigned IntID);
4827	llvm::Value *EmitSMEReadWrite(const SVETypeFlags &TypeFlags,
4828	llvm::SmallVectorImpl<llvm::Value *> &Ops,
4829	unsigned IntID);
4830	llvm::Value *EmitSMEZero(const SVETypeFlags &TypeFlags,
4831	llvm::SmallVectorImpl<llvm::Value *> &Ops,
4832	unsigned IntID);
4833	llvm::Value *EmitSMELdrStr(const SVETypeFlags &TypeFlags,
4834	llvm::SmallVectorImpl<llvm::Value *> &Ops,
4835	unsigned IntID);
4836
4837	void GetAArch64SVEProcessedOperands(unsigned BuiltinID, const CallExpr *E,
4838	SmallVectorImpl<llvm::Value *> &Ops,
4839	SVETypeFlags TypeFlags);
4840
4841	llvm::Value *EmitAArch64SMEBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
4842
4843	llvm::Value *EmitAArch64BuiltinExpr(unsigned BuiltinID, const CallExpr *E,
4844	llvm::Triple::ArchType Arch);
4845	llvm::Value *EmitBPFBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
4846
4847	llvm::Value *BuildVector(ArrayRef<llvm::Value *> Ops);
4848	llvm::Value *EmitX86BuiltinExpr(unsigned BuiltinID, const CallExpr *E);
4849	llvm::Value *EmitPPCBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
4850	llvm::Value *EmitAMDGPUBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
4851	llvm::Value *EmitHLSLBuiltinExpr(unsigned BuiltinID, const CallExpr *E,
4852	ReturnValueSlot ReturnValue);
4853	llvm::Value *EmitDirectXBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
4854	llvm::Value *EmitSPIRVBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
4855	llvm::Value *EmitScalarOrConstFoldImmArg(unsigned ICEArguments, unsigned Idx,
4856	const CallExpr *E);
4857	llvm::Value *EmitSystemZBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
4858	llvm::Value *EmitNVPTXBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
4859	llvm::Value *EmitWebAssemblyBuiltinExpr(unsigned BuiltinID,
4860	const CallExpr *E);
4861	llvm::Value *EmitHexagonBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
4862	llvm::Value *EmitRISCVBuiltinExpr(unsigned BuiltinID, const CallExpr *E,
4863	ReturnValueSlot ReturnValue);
4864
4865	llvm::Value *EmitRISCVCpuSupports(const CallExpr *E);
4866	llvm::Value *EmitRISCVCpuSupports(ArrayRef<StringRef> FeaturesStrs);
4867	llvm::Value *EmitRISCVCpuInit();
4868	llvm::Value *EmitRISCVCpuIs(const CallExpr *E);
4869	llvm::Value *EmitRISCVCpuIs(StringRef CPUStr);
4870
4871	void AddAMDGPUFenceAddressSpaceMMRA(llvm::Instruction *Inst,
4872	const CallExpr *E);
4873	void ProcessOrderScopeAMDGCN(llvm::Value *Order, llvm::Value *Scope,
4874	llvm::AtomicOrdering &AO,
4875	llvm::SyncScope::ID &SSID);
4876
4877	enum class MSVCIntrin;
4878	llvm::Value *EmitMSVCBuiltinExpr(MSVCIntrin BuiltinID, const CallExpr *E);
4879
4880	llvm::Value *EmitBuiltinAvailable(const VersionTuple &Version);
4881
4882	llvm::Value *EmitObjCProtocolExpr(const ObjCProtocolExpr *E);
4883	llvm::Value *EmitObjCStringLiteral(const ObjCStringLiteral *E);
4884	llvm::Value *EmitObjCBoxedExpr(const ObjCBoxedExpr *E);
4885	llvm::Value *EmitObjCArrayLiteral(const ObjCArrayLiteral *E);
4886	llvm::Value *EmitObjCDictionaryLiteral(const ObjCDictionaryLiteral *E);
4887	llvm::Value *
4888	EmitObjCCollectionLiteral(const Expr *E,
4889	const ObjCMethodDecl *MethodWithObjects);
4890	llvm::Value *EmitObjCSelectorExpr(const ObjCSelectorExpr *E);
4891	RValue EmitObjCMessageExpr(const ObjCMessageExpr *E,
4892	ReturnValueSlot Return = ReturnValueSlot());
4893
4894	/// Retrieves the default cleanup kind for an ARC cleanup.
4895	/// Except under -fobjc-arc-eh, ARC cleanups are normal-only.
4896	CleanupKind getARCCleanupKind() {
4897	return CGM.getCodeGenOpts().ObjCAutoRefCountExceptions ? NormalAndEHCleanup
4898	: NormalCleanup;
4899	}
4900
4901	// ARC primitives.
4902	void EmitARCInitWeak(Address addr, llvm::Value *value);
4903	void EmitARCDestroyWeak(Address addr);
4904	llvm::Value *EmitARCLoadWeak(Address addr);
4905	llvm::Value *EmitARCLoadWeakRetained(Address addr);
4906	llvm::Value *EmitARCStoreWeak(Address addr, llvm::Value *value, bool ignored);
4907	void emitARCCopyAssignWeak(QualType Ty, Address DstAddr, Address SrcAddr);
4908	void emitARCMoveAssignWeak(QualType Ty, Address DstAddr, Address SrcAddr);
4909	void EmitARCCopyWeak(Address dst, Address src);
4910	void EmitARCMoveWeak(Address dst, Address src);
4911	llvm::Value *EmitARCRetainAutorelease(QualType type, llvm::Value *value);
4912	llvm::Value *EmitARCRetainAutoreleaseNonBlock(llvm::Value *value);
4913	llvm::Value *EmitARCStoreStrong(LValue lvalue, llvm::Value *value,
4914	bool resultIgnored);
4915	llvm::Value *EmitARCStoreStrongCall(Address addr, llvm::Value *value,
4916	bool resultIgnored);
4917	llvm::Value *EmitARCRetain(QualType type, llvm::Value *value);
4918	llvm::Value *EmitARCRetainNonBlock(llvm::Value *value);
4919	llvm::Value *EmitARCRetainBlock(llvm::Value *value, bool mandatory);
4920	void EmitARCDestroyStrong(Address addr, ARCPreciseLifetime_t precise);
4921	void EmitARCRelease(llvm::Value *value, ARCPreciseLifetime_t precise);
4922	llvm::Value *EmitARCAutorelease(llvm::Value *value);
4923	llvm::Value *EmitARCAutoreleaseReturnValue(llvm::Value *value);
4924	llvm::Value *EmitARCRetainAutoreleaseReturnValue(llvm::Value *value);
4925	llvm::Value *EmitARCRetainAutoreleasedReturnValue(llvm::Value *value);
4926	llvm::Value *EmitARCUnsafeClaimAutoreleasedReturnValue(llvm::Value *value);
4927
4928	llvm::Value *EmitObjCAutorelease(llvm::Value *value, llvm::Type *returnType);
4929	llvm::Value *EmitObjCRetainNonBlock(llvm::Value *value,
4930	llvm::Type *returnType);
4931	void EmitObjCRelease(llvm::Value *value, ARCPreciseLifetime_t precise);
4932
4933	std::pair<LValue, llvm::Value *>
4934	EmitARCStoreAutoreleasing(const BinaryOperator *e);
4935	std::pair<LValue, llvm::Value *> EmitARCStoreStrong(const BinaryOperator *e,
4936	bool ignored);
4937	std::pair<LValue, llvm::Value *>
4938	EmitARCStoreUnsafeUnretained(const BinaryOperator *e, bool ignored);
4939
4940	llvm::Value *EmitObjCAlloc(llvm::Value *value, llvm::Type *returnType);
4941	llvm::Value *EmitObjCAllocWithZone(llvm::Value *value,
4942	llvm::Type *returnType);
4943	llvm::Value *EmitObjCAllocInit(llvm::Value *value, llvm::Type *resultType);
4944
4945	llvm::Value *EmitObjCThrowOperand(const Expr *expr);
4946	llvm::Value *EmitObjCConsumeObject(QualType T, llvm::Value *Ptr);
4947	llvm::Value *EmitObjCExtendObjectLifetime(QualType T, llvm::Value *Ptr);
4948
4949	llvm::Value *EmitARCExtendBlockObject(const Expr *expr);
4950	llvm::Value *EmitARCReclaimReturnedObject(const Expr *e,
4951	bool allowUnsafeClaim);
4952	llvm::Value *EmitARCRetainScalarExpr(const Expr *expr);
4953	llvm::Value *EmitARCRetainAutoreleaseScalarExpr(const Expr *expr);
4954	llvm::Value *EmitARCUnsafeUnretainedScalarExpr(const Expr *expr);
4955
4956	void EmitARCIntrinsicUse(ArrayRef<llvm::Value *> values);
4957
4958	void EmitARCNoopIntrinsicUse(ArrayRef<llvm::Value *> values);
4959
4960	static Destroyer destroyARCStrongImprecise;
4961	static Destroyer destroyARCStrongPrecise;
4962	static Destroyer destroyARCWeak;
4963	static Destroyer emitARCIntrinsicUse;
4964	static Destroyer destroyNonTrivialCStruct;
4965
4966	void EmitObjCAutoreleasePoolPop(llvm::Value *Ptr);
4967	llvm::Value *EmitObjCAutoreleasePoolPush();
4968	llvm::Value *EmitObjCMRRAutoreleasePoolPush();
4969	void EmitObjCAutoreleasePoolCleanup(llvm::Value *Ptr);
4970	void EmitObjCMRRAutoreleasePoolPop(llvm::Value *Ptr);
4971
4972	/// Emits a reference binding to the passed in expression.
4973	RValue EmitReferenceBindingToExpr(const Expr *E);
4974
4975	//===--------------------------------------------------------------------===//
4976	// Expression Emission
4977	//===--------------------------------------------------------------------===//
4978
4979	// Expressions are broken into three classes: scalar, complex, aggregate.
4980
4981	/// EmitScalarExpr - Emit the computation of the specified expression of LLVM
4982	/// scalar type, returning the result.
4983	llvm::Value *EmitScalarExpr(const Expr *E, bool IgnoreResultAssign = false);
4984
4985	/// Emit a conversion from the specified type to the specified destination
4986	/// type, both of which are LLVM scalar types.
4987	llvm::Value *EmitScalarConversion(llvm::Value *Src, QualType SrcTy,
4988	QualType DstTy, SourceLocation Loc);
4989
4990	/// Emit a conversion from the specified complex type to the specified
4991	/// destination type, where the destination type is an LLVM scalar type.
4992	llvm::Value *EmitComplexToScalarConversion(ComplexPairTy Src, QualType SrcTy,
4993	QualType DstTy,
4994	SourceLocation Loc);
4995
4996	/// EmitAggExpr - Emit the computation of the specified expression
4997	/// of aggregate type. The result is computed into the given slot,
4998	/// which may be null to indicate that the value is not needed.
4999	void EmitAggExpr(const Expr *E, AggValueSlot AS);
5000
5001	/// EmitAggExprToLValue - Emit the computation of the specified expression of
5002	/// aggregate type into a temporary LValue.
5003	LValue EmitAggExprToLValue(const Expr *E);
5004
5005	enum ExprValueKind { EVK_RValue, EVK_NonRValue };
5006
5007	/// EmitAggFinalDestCopy - Emit copy of the specified aggregate into
5008	/// destination address.
5009	void EmitAggFinalDestCopy(QualType Type, AggValueSlot Dest, const LValue &Src,
5010	ExprValueKind SrcKind);
5011
5012	/// Create a store to \arg DstPtr from \arg Src, truncating the stored value
5013	/// to at most \arg DstSize bytes.
5014	void CreateCoercedStore(llvm::Value *Src, Address Dst, llvm::TypeSize DstSize,
5015	bool DstIsVolatile);
5016
5017	/// EmitExtendGCLifetime - Given a pointer to an Objective-C object,
5018	/// make sure it survives garbage collection until this point.
5019	void EmitExtendGCLifetime(llvm::Value *object);
5020
5021	/// EmitComplexExpr - Emit the computation of the specified expression of
5022	/// complex type, returning the result.
5023	ComplexPairTy EmitComplexExpr(const Expr *E, bool IgnoreReal = false,
5024	bool IgnoreImag = false);
5025
5026	/// EmitComplexExprIntoLValue - Emit the given expression of complex
5027	/// type and place its result into the specified l-value.
5028	void EmitComplexExprIntoLValue(const Expr *E, LValue dest, bool isInit);
5029
5030	/// EmitStoreOfComplex - Store a complex number into the specified l-value.
5031	void EmitStoreOfComplex(ComplexPairTy V, LValue dest, bool isInit);
5032
5033	/// EmitLoadOfComplex - Load a complex number from the specified l-value.
5034	ComplexPairTy EmitLoadOfComplex(LValue src, SourceLocation loc);
5035
5036	ComplexPairTy EmitPromotedComplexExpr(const Expr *E, QualType PromotionType);
5037	llvm::Value *EmitPromotedScalarExpr(const Expr *E, QualType PromotionType);
5038	ComplexPairTy EmitPromotedValue(ComplexPairTy result, QualType PromotionType);
5039	ComplexPairTy EmitUnPromotedValue(ComplexPairTy result,
5040	QualType PromotionType);
5041
5042	Address emitAddrOfRealComponent(Address complex, QualType complexType);
5043	Address emitAddrOfImagComponent(Address complex, QualType complexType);
5044
5045	/// AddInitializerToStaticVarDecl - Add the initializer for 'D' to the
5046	/// global variable that has already been created for it. If the initializer
5047	/// has a different type than GV does, this may free GV and return a different
5048	/// one. Otherwise it just returns GV.
5049	llvm::GlobalVariable *AddInitializerToStaticVarDecl(const VarDecl &D,
5050	llvm::GlobalVariable *GV);
5051
5052	// Emit an @llvm.invariant.start call for the given memory region.
5053	void EmitInvariantStart(llvm::Constant *Addr, CharUnits Size);
5054
5055	/// EmitCXXGlobalVarDeclInit - Create the initializer for a C++
5056	/// variable with global storage.
5057	void EmitCXXGlobalVarDeclInit(const VarDecl &D, llvm::GlobalVariable *GV,
5058	bool PerformInit);
5059
5060	llvm::Constant *createAtExitStub(const VarDecl &VD, llvm::FunctionCallee Dtor,
5061	llvm::Constant *Addr);
5062
5063	llvm::Function *createTLSAtExitStub(const VarDecl &VD,
5064	llvm::FunctionCallee Dtor,
5065	llvm::Constant *Addr,
5066	llvm::FunctionCallee &AtExit);
5067
5068	/// Call atexit() with a function that passes the given argument to
5069	/// the given function.
5070	void registerGlobalDtorWithAtExit(const VarDecl &D, llvm::FunctionCallee fn,
5071	llvm::Constant *addr);
5072
5073	/// Registers the dtor using 'llvm.global_dtors' for platforms that do not
5074	/// support an 'atexit()' function.
5075	void registerGlobalDtorWithLLVM(const VarDecl &D, llvm::FunctionCallee fn,
5076	llvm::Constant *addr);
5077
5078	/// Call atexit() with function dtorStub.
5079	void registerGlobalDtorWithAtExit(llvm::Constant *dtorStub);
5080
5081	/// Call unatexit() with function dtorStub.
5082	llvm::Value *unregisterGlobalDtorWithUnAtExit(llvm::Constant *dtorStub);
5083
5084	/// Emit code in this function to perform a guarded variable
5085	/// initialization. Guarded initializations are used when it's not
5086	/// possible to prove that an initialization will be done exactly
5087	/// once, e.g. with a static local variable or a static data member
5088	/// of a class template.
5089	void EmitCXXGuardedInit(const VarDecl &D, llvm::GlobalVariable *DeclPtr,
5090	bool PerformInit);
5091
5092	enum class GuardKind { VariableGuard, TlsGuard };
5093
5094	/// Emit a branch to select whether or not to perform guarded initialization.
5095	void EmitCXXGuardedInitBranch(llvm::Value *NeedsInit,
5096	llvm::BasicBlock *InitBlock,
5097	llvm::BasicBlock *NoInitBlock, GuardKind Kind,
5098	const VarDecl *D);
5099
5100	/// GenerateCXXGlobalInitFunc - Generates code for initializing global
5101	/// variables.
5102	void
5103	GenerateCXXGlobalInitFunc(llvm::Function *Fn,
5104	ArrayRef<llvm::Function *> CXXThreadLocals,
5105	ConstantAddress Guard = ConstantAddress::invalid());
5106
5107	/// GenerateCXXGlobalCleanUpFunc - Generates code for cleaning up global
5108	/// variables.
5109	void GenerateCXXGlobalCleanUpFunc(
5110	llvm::Function *Fn,
5111	ArrayRef<std::tuple<llvm::FunctionType *, llvm::WeakTrackingVH,
5112	llvm::Constant *>>
5113	DtorsOrStermFinalizers);
5114
5115	void GenerateCXXGlobalVarDeclInitFunc(llvm::Function *Fn, const VarDecl *D,
5116	llvm::GlobalVariable *Addr,
5117	bool PerformInit);
5118
5119	void EmitCXXConstructExpr(const CXXConstructExpr *E, AggValueSlot Dest);
5120
5121	void EmitSynthesizedCXXCopyCtor(Address Dest, Address Src, const Expr *Exp);
5122
5123	void EmitCXXThrowExpr(const CXXThrowExpr *E, bool KeepInsertionPoint = true);
5124
5125	RValue EmitAtomicExpr(AtomicExpr *E);
5126
5127	void EmitFakeUse(Address Addr);
5128
5129	//===--------------------------------------------------------------------===//
5130	// Annotations Emission
5131	//===--------------------------------------------------------------------===//
5132
5133	/// Emit an annotation call (intrinsic).
5134	llvm::Value *EmitAnnotationCall(llvm::Function *AnnotationFn,
5135	llvm::Value *AnnotatedVal,
5136	StringRef AnnotationStr,
5137	SourceLocation Location,
5138	const AnnotateAttr *Attr);
5139
5140	/// Emit local annotations for the local variable V, declared by D.
5141	void EmitVarAnnotations(const VarDecl *D, llvm::Value *V);
5142
5143	/// Emit field annotations for the given field & value. Returns the
5144	/// annotation result.
5145	Address EmitFieldAnnotations(const FieldDecl *D, Address V);
5146
5147	//===--------------------------------------------------------------------===//
5148	// Internal Helpers
5149	//===--------------------------------------------------------------------===//
5150
5151	/// ContainsLabel - Return true if the statement contains a label in it. If
5152	/// this statement is not executed normally, it not containing a label means
5153	/// that we can just remove the code.
5154	static bool ContainsLabel(const Stmt *S, bool IgnoreCaseStmts = false);
5155
5156	/// containsBreak - Return true if the statement contains a break out of it.
5157	/// If the statement (recursively) contains a switch or loop with a break
5158	/// inside of it, this is fine.
5159	static bool containsBreak(const Stmt *S);
5160
5161	/// Determine if the given statement might introduce a declaration into the
5162	/// current scope, by being a (possibly-labelled) DeclStmt.
5163	static bool mightAddDeclToScope(const Stmt *S);
5164
5165	/// ConstantFoldsToSimpleInteger - If the specified expression does not fold
5166	/// to a constant, or if it does but contains a label, return false. If it
5167	/// constant folds return true and set the boolean result in Result.
5168	bool ConstantFoldsToSimpleInteger(const Expr *Cond, bool &Result,
5169	bool AllowLabels = false);
5170
5171	/// ConstantFoldsToSimpleInteger - If the specified expression does not fold
5172	/// to a constant, or if it does but contains a label, return false. If it
5173	/// constant folds return true and set the folded value.
5174	bool ConstantFoldsToSimpleInteger(const Expr *Cond, llvm::APSInt &Result,
5175	bool AllowLabels = false);
5176
5177	/// Ignore parentheses and logical-NOT to track conditions consistently.
5178	static const Expr *stripCond(const Expr *C);
5179
5180	/// isInstrumentedCondition - Determine whether the given condition is an
5181	/// instrumentable condition (i.e. no "&&" or "||").
5182	static bool isInstrumentedCondition(const Expr *C);
5183
5184	/// EmitBranchToCounterBlock - Emit a conditional branch to a new block that
5185	/// increments a profile counter based on the semantics of the given logical
5186	/// operator opcode. This is used to instrument branch condition coverage
5187	/// for logical operators.
5188	void EmitBranchToCounterBlock(const Expr *Cond, BinaryOperator::Opcode LOp,
5189	llvm::BasicBlock *TrueBlock,
5190	llvm::BasicBlock *FalseBlock,
5191	uint64_t TrueCount = 0,
5192	Stmt::Likelihood LH = Stmt::LH_None,
5193	const Expr *CntrIdx = nullptr);
5194
5195	/// EmitBranchOnBoolExpr - Emit a branch on a boolean condition (e.g. for an
5196	/// if statement) to the specified blocks. Based on the condition, this might
5197	/// try to simplify the codegen of the conditional based on the branch.
5198	/// TrueCount should be the number of times we expect the condition to
5199	/// evaluate to true based on PGO data.
5200	void EmitBranchOnBoolExpr(const Expr *Cond, llvm::BasicBlock *TrueBlock,
5201	llvm::BasicBlock *FalseBlock, uint64_t TrueCount,
5202	Stmt::Likelihood LH = Stmt::LH_None,
5203	const Expr *ConditionalOp = nullptr,
5204	const VarDecl *ConditionalDecl = nullptr);
5205
5206	/// Given an assignment `*LHS = RHS`, emit a test that checks if \p RHS is
5207	/// nonnull, if \p LHS is marked _Nonnull.
5208	void EmitNullabilityCheck(LValue LHS, llvm::Value *RHS, SourceLocation Loc);
5209
5210	/// An enumeration which makes it easier to specify whether or not an
5211	/// operation is a subtraction.
5212	enum { NotSubtraction = false, IsSubtraction = true };
5213
5214	/// Same as IRBuilder::CreateInBoundsGEP, but additionally emits a check to
5215	/// detect undefined behavior when the pointer overflow sanitizer is enabled.
5216	/// \p SignedIndices indicates whether any of the GEP indices are signed.
5217	/// \p IsSubtraction indicates whether the expression used to form the GEP
5218	/// is a subtraction.
5219	llvm::Value *EmitCheckedInBoundsGEP(llvm::Type *ElemTy, llvm::Value *Ptr,
5220	ArrayRef<llvm::Value *> IdxList,
5221	bool SignedIndices, bool IsSubtraction,
5222	SourceLocation Loc,
5223	const Twine &Name = "");
5224
5225	Address EmitCheckedInBoundsGEP(Address Addr, ArrayRef<llvm::Value *> IdxList,
5226	llvm::Type *elementType, bool SignedIndices,
5227	bool IsSubtraction, SourceLocation Loc,
5228	CharUnits Align, const Twine &Name = "");
5229
5230	/// Specifies which type of sanitizer check to apply when handling a
5231	/// particular builtin.
5232	enum BuiltinCheckKind {
5233	BCK_CTZPassedZero,
5234	BCK_CLZPassedZero,
5235	BCK_AssumePassedFalse,
5236	};
5237
5238	/// Emits an argument for a call to a builtin. If the builtin sanitizer is
5239	/// enabled, a runtime check specified by \p Kind is also emitted.
5240	llvm::Value *EmitCheckedArgForBuiltin(const Expr *E, BuiltinCheckKind Kind);
5241
5242	/// Emits an argument for a call to a `__builtin_assume`. If the builtin
5243	/// sanitizer is enabled, a runtime check is also emitted.
5244	llvm::Value *EmitCheckedArgForAssume(const Expr *E);
5245
5246	/// Emit a description of a type in a format suitable for passing to
5247	/// a runtime sanitizer handler.
5248	llvm::Constant *EmitCheckTypeDescriptor(QualType T);
5249
5250	/// Convert a value into a format suitable for passing to a runtime
5251	/// sanitizer handler.
5252	llvm::Value *EmitCheckValue(llvm::Value *V);
5253
5254	/// Emit a description of a source location in a format suitable for
5255	/// passing to a runtime sanitizer handler.
5256	llvm::Constant *EmitCheckSourceLocation(SourceLocation Loc);
5257
5258	void EmitKCFIOperandBundle(const CGCallee &Callee,
5259	SmallVectorImpl<llvm::OperandBundleDef> &Bundles);
5260
5261	/// Create a basic block that will either trap or call a handler function in
5262	/// the UBSan runtime with the provided arguments, and create a conditional
5263	/// branch to it.
5264	void
5265	EmitCheck(ArrayRef<std::pair<llvm::Value *, SanitizerKind::SanitizerOrdinal>>
5266	Checked,
5267	SanitizerHandler Check, ArrayRef<llvm::Constant *> StaticArgs,
5268	ArrayRef<llvm::Value *> DynamicArgs);
5269
5270	/// Emit a slow path cross-DSO CFI check which calls __cfi_slowpath
5271	/// if Cond if false.
5272	void EmitCfiSlowPathCheck(SanitizerKind::SanitizerOrdinal Ordinal,
5273	llvm::Value *Cond, llvm::ConstantInt *TypeId,
5274	llvm::Value *Ptr,
5275	ArrayRef<llvm::Constant *> StaticArgs);
5276
5277	/// Emit a reached-unreachable diagnostic if \p Loc is valid and runtime
5278	/// checking is enabled. Otherwise, just emit an unreachable instruction.
5279	void EmitUnreachable(SourceLocation Loc);
5280
5281	/// Create a basic block that will call the trap intrinsic, and emit a
5282	/// conditional branch to it, for the -ftrapv checks.
5283	void EmitTrapCheck(llvm::Value *Checked, SanitizerHandler CheckHandlerID,
5284	bool NoMerge = false);
5285
5286	/// Emit a call to trap or debugtrap and attach function attribute
5287	/// "trap-func-name" if specified.
5288	llvm::CallInst *EmitTrapCall(llvm::Intrinsic::ID IntrID);
5289
5290	/// Emit a stub for the cross-DSO CFI check function.
5291	void EmitCfiCheckStub();
5292
5293	/// Emit a cross-DSO CFI failure handling function.
5294	void EmitCfiCheckFail();
5295
5296	/// Create a check for a function parameter that may potentially be
5297	/// declared as non-null.
5298	void EmitNonNullArgCheck(RValue RV, QualType ArgType, SourceLocation ArgLoc,
5299	AbstractCallee AC, unsigned ParmNum);
5300
5301	void EmitNonNullArgCheck(Address Addr, QualType ArgType,
5302	SourceLocation ArgLoc, AbstractCallee AC,
5303	unsigned ParmNum);
5304
5305	/// EmitWriteback - Emit callbacks for function.
5306	void EmitWritebacks(const CallArgList &Args);
5307
5308	/// EmitCallArg - Emit a single call argument.
5309	void EmitCallArg(CallArgList &args, const Expr *E, QualType ArgType);
5310
5311	/// EmitDelegateCallArg - We are performing a delegate call; that
5312	/// is, the current function is delegating to another one. Produce
5313	/// a r-value suitable for passing the given parameter.
5314	void EmitDelegateCallArg(CallArgList &args, const VarDecl *param,
5315	SourceLocation loc);
5316
5317	/// SetFPAccuracy - Set the minimum required accuracy of the given floating
5318	/// point operation, expressed as the maximum relative error in ulp.
5319	void SetFPAccuracy(llvm::Value *Val, float Accuracy);
5320
5321	/// Set the minimum required accuracy of the given sqrt operation
5322	/// based on CodeGenOpts.
5323	void SetSqrtFPAccuracy(llvm::Value *Val);
5324
5325	/// Set the minimum required accuracy of the given sqrt operation based on
5326	/// CodeGenOpts.
5327	void SetDivFPAccuracy(llvm::Value *Val);
5328
5329	/// Set the codegen fast-math flags.
5330	void SetFastMathFlags(FPOptions FPFeatures);
5331
5332	// Truncate or extend a boolean vector to the requested number of elements.
5333	llvm::Value *emitBoolVecConversion(llvm::Value *SrcVec,
5334	unsigned NumElementsDst,
5335	const llvm::Twine &Name = "");
5336
5337	void maybeAttachRangeForLoad(llvm::LoadInst *Load, QualType Ty,
5338	SourceLocation Loc);
5339
5340	private:
5341	// Emits a convergence_loop instruction for the given |BB|, with |ParentToken|
5342	// as it's parent convergence instr.
5343	llvm::ConvergenceControlInst *emitConvergenceLoopToken(llvm::BasicBlock *BB);
5344
5345	// Adds a convergence_ctrl token with |ParentToken| as parent convergence
5346	// instr to the call |Input|.
5347	llvm::CallBase *addConvergenceControlToken(llvm::CallBase *Input);
5348
5349	// Find the convergence_entry instruction |F|, or emits ones if none exists.
5350	// Returns the convergence instruction.
5351	llvm::ConvergenceControlInst *
5352	getOrEmitConvergenceEntryToken(llvm::Function *F);
5353
5354	private:
5355	llvm::MDNode *getRangeForLoadFromType(QualType Ty);
5356	void EmitReturnOfRValue(RValue RV, QualType Ty);
5357
5358	void deferPlaceholderReplacement(llvm::Instruction *Old, llvm::Value *New);
5359
5360	llvm::SmallVector<std::pair<llvm::WeakTrackingVH, llvm::Value *>, 4>
5361	DeferredReplacements;
5362
5363	/// Set the address of a local variable.
5364	void setAddrOfLocalVar(const VarDecl *VD, Address Addr) {
5365	assert(!LocalDeclMap.count(VD) && "Decl already exists in LocalDeclMap!");
5366	LocalDeclMap.insert({VD, Addr});
5367	}
5368
5369	/// ExpandTypeFromArgs - Reconstruct a structure of type \arg Ty
5370	/// from function arguments into \arg Dst. See ABIArgInfo::Expand.
5371	///
5372	/// \param AI - The first function argument of the expansion.
5373	void ExpandTypeFromArgs(QualType Ty, LValue Dst,
5374	llvm::Function::arg_iterator &AI);
5375
5376	/// ExpandTypeToArgs - Expand an CallArg \arg Arg, with the LLVM type for \arg
5377	/// Ty, into individual arguments on the provided vector \arg IRCallArgs,
5378	/// starting at index \arg IRCallArgPos. See ABIArgInfo::Expand.
5379	void ExpandTypeToArgs(QualType Ty, CallArg Arg, llvm::FunctionType *IRFuncTy,
5380	SmallVectorImpl<llvm::Value *> &IRCallArgs,
5381	unsigned &IRCallArgPos);
5382
5383	std::pair<llvm::Value *, llvm::Type *>
5384	EmitAsmInput(const TargetInfo::ConstraintInfo &Info, const Expr *InputExpr,
5385	std::string &ConstraintStr);
5386
5387	std::pair<llvm::Value *, llvm::Type *>
5388	EmitAsmInputLValue(const TargetInfo::ConstraintInfo &Info, LValue InputValue,
5389	QualType InputType, std::string &ConstraintStr,
5390	SourceLocation Loc);
5391
5392	/// Attempts to statically evaluate the object size of E. If that
5393	/// fails, emits code to figure the size of E out for us. This is
5394	/// pass_object_size aware.
5395	///
5396	/// If EmittedExpr is non-null, this will use that instead of re-emitting E.
5397	llvm::Value *evaluateOrEmitBuiltinObjectSize(const Expr *E, unsigned Type,
5398	llvm::IntegerType *ResType,
5399	llvm::Value *EmittedE,
5400	bool IsDynamic);
5401
5402	/// Emits the size of E, as required by __builtin_object_size. This
5403	/// function is aware of pass_object_size parameters, and will act accordingly
5404	/// if E is a parameter with the pass_object_size attribute.
5405	llvm::Value *emitBuiltinObjectSize(const Expr *E, unsigned Type,
5406	llvm::IntegerType *ResType,
5407	llvm::Value *EmittedE, bool IsDynamic);
5408
5409	llvm::Value *emitCountedBySize(const Expr *E, llvm::Value *EmittedE,
5410	unsigned Type, llvm::IntegerType *ResType);
5411
5412	llvm::Value *emitCountedByMemberSize(const MemberExpr *E, const Expr *Idx,
5413	llvm::Value *EmittedE,
5414	QualType CastedArrayElementTy,
5415	unsigned Type,
5416	llvm::IntegerType *ResType);
5417
5418	llvm::Value *emitCountedByPointerSize(const ImplicitCastExpr *E,
5419	const Expr *Idx, llvm::Value *EmittedE,
5420	QualType CastedArrayElementTy,
5421	unsigned Type,
5422	llvm::IntegerType *ResType);
5423
5424	void emitZeroOrPatternForAutoVarInit(QualType type, const VarDecl &D,
5425	Address Loc);
5426
5427	public:
5428	enum class EvaluationOrder {
5429	///! No language constraints on evaluation order.
5430	Default,
5431	///! Language semantics require left-to-right evaluation.
5432	ForceLeftToRight,
5433	///! Language semantics require right-to-left evaluation.
5434	ForceRightToLeft
5435	};
5436
5437	// Wrapper for function prototype sources. Wraps either a FunctionProtoType or
5438	// an ObjCMethodDecl.
5439	struct PrototypeWrapper {
5440	llvm::PointerUnion<const FunctionProtoType *, const ObjCMethodDecl *> P;
5441
5442	PrototypeWrapper(const FunctionProtoType *FT) : P(FT) {}
5443	PrototypeWrapper(const ObjCMethodDecl *MD) : P(MD) {}
5444	};
5445
5446	void EmitCallArgs(CallArgList &Args, PrototypeWrapper Prototype,
5447	llvm::iterator_range<CallExpr::const_arg_iterator> ArgRange,
5448	AbstractCallee AC = AbstractCallee(),
5449	unsigned ParamsToSkip = 0,
5450	EvaluationOrder Order = EvaluationOrder::Default);
5451
5452	/// EmitPointerWithAlignment - Given an expression with a pointer type,
5453	/// emit the value and compute our best estimate of the alignment of the
5454	/// pointee.
5455	///
5456	/// \param BaseInfo - If non-null, this will be initialized with
5457	/// information about the source of the alignment and the may-alias
5458	/// attribute. Note that this function will conservatively fall back on
5459	/// the type when it doesn't recognize the expression and may-alias will
5460	/// be set to false.
5461	///
5462	/// One reasonable way to use this information is when there's a language
5463	/// guarantee that the pointer must be aligned to some stricter value, and
5464	/// we're simply trying to ensure that sufficiently obvious uses of under-
5465	/// aligned objects don't get miscompiled; for example, a placement new
5466	/// into the address of a local variable. In such a case, it's quite
5467	/// reasonable to just ignore the returned alignment when it isn't from an
5468	/// explicit source.
5469	Address
5470	EmitPointerWithAlignment(const Expr *Addr, LValueBaseInfo *BaseInfo = nullptr,
5471	TBAAAccessInfo *TBAAInfo = nullptr,
5472	KnownNonNull_t IsKnownNonNull = NotKnownNonNull);
5473
5474	/// If \p E references a parameter with pass_object_size info or a constant
5475	/// array size modifier, emit the object size divided by the size of \p EltTy.
5476	/// Otherwise return null.
5477	llvm::Value *LoadPassedObjectSize(const Expr *E, QualType EltTy);
5478
5479	void EmitSanitizerStatReport(llvm::SanitizerStatKind SSK);
5480
5481	struct FMVResolverOption {
5482	llvm::Function *Function;
5483	llvm::SmallVector<StringRef, 8> Features;
5484	std::optional<StringRef> Architecture;
5485
5486	FMVResolverOption(llvm::Function *F, ArrayRef<StringRef> Feats,
5487	std::optional<StringRef> Arch = std::nullopt)
5488	: Function(F), Features(Feats), Architecture(Arch) {}
5489	};
5490
5491	// Emits the body of a multiversion function's resolver. Assumes that the
5492	// options are already sorted in the proper order, with the 'default' option
5493	// last (if it exists).
5494	void EmitMultiVersionResolver(llvm::Function *Resolver,
5495	ArrayRef<FMVResolverOption> Options);
5496	void EmitX86MultiVersionResolver(llvm::Function *Resolver,
5497	ArrayRef<FMVResolverOption> Options);
5498	void EmitAArch64MultiVersionResolver(llvm::Function *Resolver,
5499	ArrayRef<FMVResolverOption> Options);
5500	void EmitRISCVMultiVersionResolver(llvm::Function *Resolver,
5501	ArrayRef<FMVResolverOption> Options);
5502
5503	private:
5504	QualType getVarArgType(const Expr *Arg);
5505
5506	void EmitDeclMetadata();
5507
5508	BlockByrefHelpers *buildByrefHelpers(llvm::StructType &byrefType,
5509	const AutoVarEmission &emission);
5510
5511	void AddObjCARCExceptionMetadata(llvm::Instruction *Inst);
5512
5513	llvm::Value *GetValueForARMHint(unsigned BuiltinID);
5514	llvm::Value *EmitX86CpuIs(const CallExpr *E);
5515	llvm::Value *EmitX86CpuIs(StringRef CPUStr);
5516	llvm::Value *EmitX86CpuSupports(const CallExpr *E);
5517	llvm::Value *EmitX86CpuSupports(ArrayRef<StringRef> FeatureStrs);
5518	llvm::Value *EmitX86CpuSupports(std::array<uint32_t, 4> FeatureMask);
5519	llvm::Value *EmitX86CpuInit();
5520	llvm::Value *FormX86ResolverCondition(const FMVResolverOption &RO);
5521	llvm::Value *EmitAArch64CpuInit();
5522	llvm::Value *FormAArch64ResolverCondition(const FMVResolverOption &RO);
5523	llvm::Value *EmitAArch64CpuSupports(const CallExpr *E);
5524	llvm::Value *EmitAArch64CpuSupports(ArrayRef<StringRef> FeatureStrs);
5525	};
5526
5527	inline DominatingLLVMValue::saved_type
5528	DominatingLLVMValue::save(CodeGenFunction &CGF, llvm::Value *value) {
5529	if (!needsSaving(value))
5530	return saved_type(value, false);
5531
5532	// Otherwise, we need an alloca.
5533	auto align = CharUnits::fromQuantity(
5534	Quantity: CGF.CGM.getDataLayout().getPrefTypeAlign(Ty: value->getType()));
5535	Address alloca =
5536	CGF.CreateTempAlloca(Ty: value->getType(), align, Name: "cond-cleanup.save");
5537	CGF.Builder.CreateStore(Val: value, Addr: alloca);
5538
5539	return saved_type(alloca.emitRawPointer(CGF), true);
5540	}
5541
5542	inline llvm::Value *DominatingLLVMValue::restore(CodeGenFunction &CGF,
5543	saved_type value) {
5544	// If the value says it wasn't saved, trust that it's still dominating.
5545	if (!value.getInt())
5546	return value.getPointer();
5547
5548	// Otherwise, it should be an alloca instruction, as set up in save().
5549	auto alloca = cast<llvm::AllocaInst>(Val: value.getPointer());
5550	return CGF.Builder.CreateAlignedLoad(Ty: alloca->getAllocatedType(), Ptr: alloca,
5551	Align: alloca->getAlign());
5552	}
5553
5554	} // end namespace CodeGen
5555
5556	// Map the LangOption for floating point exception behavior into
5557	// the corresponding enum in the IR.
5558	llvm::fp::ExceptionBehavior
5559	ToConstrainedExceptMD(LangOptions::FPExceptionModeKind Kind);
5560	} // end namespace clang
5561
5562	#endif
5563