1	//===-- CodeGenFunction.h - Per-Function state for LLVM CodeGen -*- C++ -*-===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This is the internal per-function state used for llvm translation.
10	//
11	//===----------------------------------------------------------------------===//
12
13	#ifndef LLVM_CLANG_LIB_CODEGEN_CODEGENFUNCTION_H
14	#define LLVM_CLANG_LIB_CODEGEN_CODEGENFUNCTION_H
15
16	#include "CGBuilder.h"
17	#include "CGLoopInfo.h"
18	#include "CGValue.h"
19	#include "CodeGenModule.h"
20	#include "EHScopeStack.h"
21	#include "SanitizerHandler.h"
22	#include "VarBypassDetector.h"
23	#include "clang/AST/CharUnits.h"
24	#include "clang/AST/CurrentSourceLocExprScope.h"
25	#include "clang/AST/ExprCXX.h"
26	#include "clang/AST/ExprObjC.h"
27	#include "clang/AST/ExprOpenMP.h"
28	#include "clang/AST/StmtOpenACC.h"
29	#include "clang/AST/StmtOpenMP.h"
30	#include "clang/AST/StmtSYCL.h"
31	#include "clang/AST/Type.h"
32	#include "clang/Basic/ABI.h"
33	#include "clang/Basic/CapturedStmt.h"
34	#include "clang/Basic/CodeGenOptions.h"
35	#include "clang/Basic/OpenMPKinds.h"
36	#include "clang/Basic/TargetInfo.h"
37	#include "llvm/ADT/ArrayRef.h"
38	#include "llvm/ADT/DenseMap.h"
39	#include "llvm/ADT/MapVector.h"
40	#include "llvm/ADT/SmallVector.h"
41	#include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
42	#include "llvm/IR/Instructions.h"
43	#include "llvm/IR/ValueHandle.h"
44	#include "llvm/Support/Debug.h"
45	#include "llvm/Transforms/Utils/SanitizerStats.h"
46	#include <optional>
47
48	namespace llvm {
49	class BasicBlock;
50	class ConvergenceControlInst;
51	class LLVMContext;
52	class MDNode;
53	class SwitchInst;
54	class Twine;
55	class Value;
56	class CanonicalLoopInfo;
57	} // namespace llvm
58
59	namespace clang {
60	class ASTContext;
61	class CXXDestructorDecl;
62	class CXXForRangeStmt;
63	class CXXTryStmt;
64	class Decl;
65	class LabelDecl;
66	class FunctionDecl;
67	class FunctionProtoType;
68	class LabelStmt;
69	class ObjCContainerDecl;
70	class ObjCInterfaceDecl;
71	class ObjCIvarDecl;
72	class ObjCMethodDecl;
73	class ObjCImplementationDecl;
74	class ObjCPropertyImplDecl;
75	class TargetInfo;
76	class VarDecl;
77	class ObjCForCollectionStmt;
78	class ObjCAtTryStmt;
79	class ObjCAtThrowStmt;
80	class ObjCAtSynchronizedStmt;
81	class ObjCAutoreleasePoolStmt;
82	class OMPUseDevicePtrClause;
83	class OMPUseDeviceAddrClause;
84	class SVETypeFlags;
85	class OMPExecutableDirective;
86
87	namespace analyze_os_log {
88	class OSLogBufferLayout;
89	}
90
91	namespace CodeGen {
92	class CodeGenTypes;
93	class CodeGenPGO;
94	class CGCallee;
95	class CGFunctionInfo;
96	class CGBlockInfo;
97	class CGCXXABI;
98	class BlockByrefHelpers;
99	class BlockByrefInfo;
100	class BlockFieldFlags;
101	class RegionCodeGenTy;
102	class TargetCodeGenInfo;
103	struct OMPTaskDataTy;
104	struct CGCoroData;
105
106	// clang-format off
107	/// The kind of evaluation to perform on values of a particular
108	/// type. Basically, is the code in CGExprScalar, CGExprComplex, or
109	/// CGExprAgg?
110	///
111	/// TODO: should vectors maybe be split out into their own thing?
112	enum TypeEvaluationKind {
113	TEK_Scalar,
114	TEK_Complex,
115	TEK_Aggregate
116	};
117	// clang-format on
118
119	/// Helper class with most of the code for saving a value for a
120	/// conditional expression cleanup.
121	struct DominatingLLVMValue {
122	typedef llvm::PointerIntPair<llvm::Value *, 1, bool> saved_type;
123
124	/// Answer whether the given value needs extra work to be saved.
125	static bool needsSaving(llvm::Value *value) {
126	if (!value)
127	return false;
128
129	// If it's not an instruction, we don't need to save.
130	if (!isa<llvm::Instruction>(Val: value))
131	return false;
132
133	// If it's an instruction in the entry block, we don't need to save.
134	llvm::BasicBlock *block = cast<llvm::Instruction>(Val: value)->getParent();
135	return (block != &block->getParent()->getEntryBlock());
136	}
137
138	static saved_type save(CodeGenFunction &CGF, llvm::Value *value);
139	static llvm::Value *restore(CodeGenFunction &CGF, saved_type value);
140	};
141
142	/// A partial specialization of DominatingValue for llvm::Values that
143	/// might be llvm::Instructions.
144	template <class T> struct DominatingPointer<T, true> : DominatingLLVMValue {
145	typedef T *type;
146	static type restore(CodeGenFunction &CGF, saved_type value) {
147	return static_cast<T *>(DominatingLLVMValue::restore(CGF, value));
148	}
149	};
150
151	/// A specialization of DominatingValue for Address.
152	template <> struct DominatingValue<Address> {
153	typedef Address type;
154
155	struct saved_type {
156	DominatingLLVMValue::saved_type BasePtr;
157	llvm::Type *ElementType;
158	CharUnits Alignment;
159	DominatingLLVMValue::saved_type Offset;
160	llvm::PointerType *EffectiveType;
161	};
162
163	static bool needsSaving(type value) {
164	if (DominatingLLVMValue::needsSaving(value: value.getBasePointer()) ||
165	DominatingLLVMValue::needsSaving(value: value.getOffset()))
166	return true;
167	return false;
168	}
169	static saved_type save(CodeGenFunction &CGF, type value) {
170	return {.BasePtr: DominatingLLVMValue::save(CGF, value: value.getBasePointer()),
171	.ElementType: value.getElementType(), .Alignment: value.getAlignment(),
172	.Offset: DominatingLLVMValue::save(CGF, value: value.getOffset()), .EffectiveType: value.getType()};
173	}
174	static type restore(CodeGenFunction &CGF, saved_type value) {
175	return Address(DominatingLLVMValue::restore(CGF, value: value.BasePtr),
176	value.ElementType, value.Alignment, CGPointerAuthInfo(),
177	DominatingLLVMValue::restore(CGF, value: value.Offset));
178	}
179	};
180
181	/// A specialization of DominatingValue for RValue.
182	template <> struct DominatingValue<RValue> {
183	typedef RValue type;
184	class saved_type {
185	enum Kind {
186	ScalarLiteral,
187	ScalarAddress,
188	AggregateLiteral,
189	AggregateAddress,
190	ComplexAddress
191	};
192	union {
193	struct {
194	DominatingLLVMValue::saved_type first, second;
195	} Vals;
196	DominatingValue<Address>::saved_type AggregateAddr;
197	};
198	LLVM_PREFERRED_TYPE(Kind)
199	unsigned K : 3;
200
201	saved_type(DominatingLLVMValue::saved_type Val1, unsigned K)
202	: Vals{.first: Val1, .second: DominatingLLVMValue::saved_type()}, K(K) {}
203
204	saved_type(DominatingLLVMValue::saved_type Val1,
205	DominatingLLVMValue::saved_type Val2)
206	: Vals{.first: Val1, .second: Val2}, K(ComplexAddress) {}
207
208	saved_type(DominatingValue<Address>::saved_type AggregateAddr, unsigned K)
209	: AggregateAddr(AggregateAddr), K(K) {}
210
211	public:
212	static bool needsSaving(RValue value);
213	static saved_type save(CodeGenFunction &CGF, RValue value);
214	RValue restore(CodeGenFunction &CGF);
215
216	// implementations in CGCleanup.cpp
217	};
218
219	static bool needsSaving(type value) { return saved_type::needsSaving(value); }
220	static saved_type save(CodeGenFunction &CGF, type value) {
221	return saved_type::save(CGF, value);
222	}
223	static type restore(CodeGenFunction &CGF, saved_type value) {
224	return value.restore(CGF);
225	}
226	};
227
228	/// A scoped helper to set the current source atom group for
229	/// CGDebugInfo::addInstToCurrentSourceAtom. A source atom is a source construct
230	/// that is "interesting" for debug stepping purposes. We use an atom group
231	/// number to track the instruction(s) that implement the functionality for the
232	/// atom, plus backup instructions/source locations.
233	class ApplyAtomGroup {
234	uint64_t OriginalAtom = 0;
235	CGDebugInfo *DI = nullptr;
236
237	ApplyAtomGroup(const ApplyAtomGroup &) = delete;
238	void operator=(const ApplyAtomGroup &) = delete;
239
240	public:
241	ApplyAtomGroup(CGDebugInfo *DI);
242	~ApplyAtomGroup();
243	};
244
245	/// CodeGenFunction - This class organizes the per-function state that is used
246	/// while generating LLVM code.
247	class CodeGenFunction : public CodeGenTypeCache {
248	CodeGenFunction(const CodeGenFunction &) = delete;
249	void operator=(const CodeGenFunction &) = delete;
250
251	friend class CGCXXABI;
252
253	public:
254	/// A jump destination is an abstract label, branching to which may
255	/// require a jump out through normal cleanups.
256	struct JumpDest {
257	JumpDest() : Block(nullptr), Index(0) {}
258	JumpDest(llvm::BasicBlock *Block, EHScopeStack::stable_iterator Depth,
259	unsigned Index)
260	: Block(Block), ScopeDepth(Depth), Index(Index) {}
261
262	bool isValid() const { return Block != nullptr; }
263	llvm::BasicBlock *getBlock() const { return Block; }
264	EHScopeStack::stable_iterator getScopeDepth() const { return ScopeDepth; }
265	unsigned getDestIndex() const { return Index; }
266
267	// This should be used cautiously.
268	void setScopeDepth(EHScopeStack::stable_iterator depth) {
269	ScopeDepth = depth;
270	}
271
272	private:
273	llvm::BasicBlock *Block;
274	EHScopeStack::stable_iterator ScopeDepth;
275	unsigned Index;
276	};
277
278	CodeGenModule &CGM; // Per-module state.
279	const TargetInfo &Target;
280
281	// For EH/SEH outlined funclets, this field points to parent's CGF
282	CodeGenFunction *ParentCGF = nullptr;
283
284	typedef std::pair<llvm::Value *, llvm::Value *> ComplexPairTy;
285	LoopInfoStack LoopStack;
286	CGBuilderTy Builder;
287
288	// Stores variables for which we can't generate correct lifetime markers
289	// because of jumps.
290	VarBypassDetector Bypasses;
291
292	/// List of recently emitted OMPCanonicalLoops.
293	///
294	/// Since OMPCanonicalLoops are nested inside other statements (in particular
295	/// CapturedStmt generated by OMPExecutableDirective and non-perfectly nested
296	/// loops), we cannot directly call OMPEmitOMPCanonicalLoop and receive its
297	/// llvm::CanonicalLoopInfo. Instead, we call EmitStmt and any
298	/// OMPEmitOMPCanonicalLoop called by it will add its CanonicalLoopInfo to
299	/// this stack when done. Entering a new loop requires clearing this list; it
300	/// either means we start parsing a new loop nest (in which case the previous
301	/// loop nest goes out of scope) or a second loop in the same level in which
302	/// case it would be ambiguous into which of the two (or more) loops the loop
303	/// nest would extend.
304	SmallVector<llvm::CanonicalLoopInfo *, 4> OMPLoopNestStack;
305
306	/// Stack to track the Logical Operator recursion nest for MC/DC.
307	SmallVector<const BinaryOperator *, 16> MCDCLogOpStack;
308
309	/// Stack to track the controlled convergence tokens.
310	SmallVector<llvm::ConvergenceControlInst *, 4> ConvergenceTokenStack;
311
312	/// Number of nested loop to be consumed by the last surrounding
313	/// loop-associated directive.
314	int ExpectedOMPLoopDepth = 0;
315
316	// CodeGen lambda for loops and support for ordered clause
317	typedef llvm::function_ref<void(CodeGenFunction &, const OMPLoopDirective &,
318	JumpDest)>
319	CodeGenLoopTy;
320	typedef llvm::function_ref<void(CodeGenFunction &, SourceLocation,
321	const unsigned, const bool)>
322	CodeGenOrderedTy;
323
324	// Codegen lambda for loop bounds in worksharing loop constructs
325	typedef llvm::function_ref<std::pair<LValue, LValue>(
326	CodeGenFunction &, const OMPExecutableDirective &S)>
327	CodeGenLoopBoundsTy;
328
329	// Codegen lambda for loop bounds in dispatch-based loop implementation
330	typedef llvm::function_ref<std::pair<llvm::Value *, llvm::Value *>(
331	CodeGenFunction &, const OMPExecutableDirective &S, Address LB,
332	Address UB)>
333	CodeGenDispatchBoundsTy;
334
335	/// CGBuilder insert helper. This function is called after an
336	/// instruction is created using Builder.
337	void InsertHelper(llvm::Instruction *I, const llvm::Twine &Name,
338	llvm::BasicBlock::iterator InsertPt) const;
339
340	/// CurFuncDecl - Holds the Decl for the current outermost
341	/// non-closure context.
342	const Decl *CurFuncDecl = nullptr;
343	/// CurCodeDecl - This is the inner-most code context, which includes blocks.
344	const Decl *CurCodeDecl = nullptr;
345	const CGFunctionInfo *CurFnInfo = nullptr;
346	QualType FnRetTy;
347	llvm::Function *CurFn = nullptr;
348
349	/// Save Parameter Decl for coroutine.
350	llvm::SmallVector<const ParmVarDecl *, 4> FnArgs;
351
352	// Holds coroutine data if the current function is a coroutine. We use a
353	// wrapper to manage its lifetime, so that we don't have to define CGCoroData
354	// in this header.
355	struct CGCoroInfo {
356	std::unique_ptr<CGCoroData> Data;
357	bool InSuspendBlock = false;
358	CGCoroInfo();
359	~CGCoroInfo();
360	};
361	CGCoroInfo CurCoro;
362
363	bool isCoroutine() const { return CurCoro.Data != nullptr; }
364
365	bool inSuspendBlock() const {
366	return isCoroutine() && CurCoro.InSuspendBlock;
367	}
368
369	// Holds FramePtr for await_suspend wrapper generation,
370	// so that __builtin_coro_frame call can be lowered
371	// directly to value of its second argument
372	struct AwaitSuspendWrapperInfo {
373	llvm::Value *FramePtr = nullptr;
374	};
375	AwaitSuspendWrapperInfo CurAwaitSuspendWrapper;
376
377	// Generates wrapper function for `llvm.coro.await.suspend.*` intrinisics.
378	// It encapsulates SuspendExpr in a function, to separate it's body
379	// from the main coroutine to avoid miscompilations. Intrinisic
380	// is lowered to this function call in CoroSplit pass
381	// Function signature is:
382	// <type> __await_suspend_wrapper_<name>(ptr %awaiter, ptr %hdl)
383	// where type is one of (void, i1, ptr)
384	llvm::Function *generateAwaitSuspendWrapper(Twine const &CoroName,
385	Twine const &SuspendPointName,
386	CoroutineSuspendExpr const &S);
387
388	/// CurGD - The GlobalDecl for the current function being compiled.
389	GlobalDecl CurGD;
390
391	/// PrologueCleanupDepth - The cleanup depth enclosing all the
392	/// cleanups associated with the parameters.
393	EHScopeStack::stable_iterator PrologueCleanupDepth;
394
395	/// ReturnBlock - Unified return block.
396	JumpDest ReturnBlock;
397
398	/// ReturnValue - The temporary alloca to hold the return
399	/// value. This is invalid iff the function has no return value.
400	Address ReturnValue = Address::invalid();
401
402	/// ReturnValuePointer - The temporary alloca to hold a pointer to sret.
403	/// This is invalid if sret is not in use.
404	Address ReturnValuePointer = Address::invalid();
405
406	/// If a return statement is being visited, this holds the return statment's
407	/// result expression.
408	const Expr *RetExpr = nullptr;
409
410	/// Return true if a label was seen in the current scope.
411	bool hasLabelBeenSeenInCurrentScope() const {
412	if (CurLexicalScope)
413	return CurLexicalScope->hasLabels();
414	return !LabelMap.empty();
415	}
416
417	/// AllocaInsertPoint - This is an instruction in the entry block before which
418	/// we prefer to insert allocas.
419	llvm::AssertingVH<llvm::Instruction> AllocaInsertPt;
420
421	private:
422	/// PostAllocaInsertPt - This is a place in the prologue where code can be
423	/// inserted that will be dominated by all the static allocas. This helps
424	/// achieve two things:
425	/// 1. Contiguity of all static allocas (within the prologue) is maintained.
426	/// 2. All other prologue code (which are dominated by static allocas) do
427	/// appear in the source order immediately after all static allocas.
428	///
429	/// PostAllocaInsertPt will be lazily created when it is *really* required.
430	llvm::AssertingVH<llvm::Instruction> PostAllocaInsertPt = nullptr;
431
432	public:
433	/// Return PostAllocaInsertPt. If it is not yet created, then insert it
434	/// immediately after AllocaInsertPt.
435	llvm::Instruction *getPostAllocaInsertPoint() {
436	if (!PostAllocaInsertPt) {
437	assert(AllocaInsertPt &&
438	"Expected static alloca insertion point at function prologue");
439	assert(AllocaInsertPt->getParent()->isEntryBlock() &&
440	"EBB should be entry block of the current code gen function");
441	PostAllocaInsertPt = AllocaInsertPt->clone();
442	PostAllocaInsertPt->setName("postallocapt");
443	PostAllocaInsertPt->insertAfter(InsertPos: AllocaInsertPt->getIterator());
444	}
445
446	return PostAllocaInsertPt;
447	}
448
449	/// API for captured statement code generation.
450	class CGCapturedStmtInfo {
451	public:
452	explicit CGCapturedStmtInfo(CapturedRegionKind K = CR_Default)
453	: Kind(K), ThisValue(nullptr), CXXThisFieldDecl(nullptr) {}
454	explicit CGCapturedStmtInfo(const CapturedStmt &S,
455	CapturedRegionKind K = CR_Default)
456	: Kind(K), ThisValue(nullptr), CXXThisFieldDecl(nullptr) {
457
458	RecordDecl::field_iterator Field =
459	S.getCapturedRecordDecl()->field_begin();
460	for (CapturedStmt::const_capture_iterator I = S.capture_begin(),
461	E = S.capture_end();
462	I != E; ++I, ++Field) {
463	if (I->capturesThis())
464	CXXThisFieldDecl = *Field;
465	else if (I->capturesVariable())
466	CaptureFields[I->getCapturedVar()->getCanonicalDecl()] = *Field;
467	else if (I->capturesVariableByCopy())
468	CaptureFields[I->getCapturedVar()->getCanonicalDecl()] = *Field;
469	}
470	}
471
472	virtual ~CGCapturedStmtInfo();
473
474	CapturedRegionKind getKind() const { return Kind; }
475
476	virtual void setContextValue(llvm::Value *V) { ThisValue = V; }
477	// Retrieve the value of the context parameter.
478	virtual llvm::Value *getContextValue() const { return ThisValue; }
479
480	/// Lookup the captured field decl for a variable.
481	virtual const FieldDecl *lookup(const VarDecl *VD) const {
482	return CaptureFields.lookup(Val: VD->getCanonicalDecl());
483	}
484
485	bool isCXXThisExprCaptured() const { return getThisFieldDecl() != nullptr; }
486	virtual FieldDecl *getThisFieldDecl() const { return CXXThisFieldDecl; }
487
488	static bool classof(const CGCapturedStmtInfo *) { return true; }
489
490	/// Emit the captured statement body.
491	virtual void EmitBody(CodeGenFunction &CGF, const Stmt *S) {
492	CGF.incrementProfileCounter(S);
493	CGF.EmitStmt(S);
494	}
495
496	/// Get the name of the capture helper.
497	virtual StringRef getHelperName() const { return "__captured_stmt"; }
498
499	/// Get the CaptureFields
500	llvm::SmallDenseMap<const VarDecl *, FieldDecl *> getCaptureFields() {
501	return CaptureFields;
502	}
503
504	private:
505	/// The kind of captured statement being generated.
506	CapturedRegionKind Kind;
507
508	/// Keep the map between VarDecl and FieldDecl.
509	llvm::SmallDenseMap<const VarDecl *, FieldDecl *> CaptureFields;
510
511	/// The base address of the captured record, passed in as the first
512	/// argument of the parallel region function.
513	llvm::Value *ThisValue;
514
515	/// Captured 'this' type.
516	FieldDecl *CXXThisFieldDecl;
517	};
518	CGCapturedStmtInfo *CapturedStmtInfo = nullptr;
519
520	/// RAII for correct setting/restoring of CapturedStmtInfo.
521	class CGCapturedStmtRAII {
522	private:
523	CodeGenFunction &CGF;
524	CGCapturedStmtInfo *PrevCapturedStmtInfo;
525
526	public:
527	CGCapturedStmtRAII(CodeGenFunction &CGF,
528	CGCapturedStmtInfo *NewCapturedStmtInfo)
529	: CGF(CGF), PrevCapturedStmtInfo(CGF.CapturedStmtInfo) {
530	CGF.CapturedStmtInfo = NewCapturedStmtInfo;
531	}
532	~CGCapturedStmtRAII() { CGF.CapturedStmtInfo = PrevCapturedStmtInfo; }
533	};
534
535	/// An abstract representation of regular/ObjC call/message targets.
536	class AbstractCallee {
537	/// The function declaration of the callee.
538	const Decl *CalleeDecl;
539
540	public:
541	AbstractCallee() : CalleeDecl(nullptr) {}
542	AbstractCallee(const FunctionDecl *FD) : CalleeDecl(FD) {}
543	AbstractCallee(const ObjCMethodDecl *OMD) : CalleeDecl(OMD) {}
544	bool hasFunctionDecl() const {
545	return isa_and_nonnull<FunctionDecl>(Val: CalleeDecl);
546	}
547	const Decl *getDecl() const { return CalleeDecl; }
548	unsigned getNumParams() const {
549	if (const auto *FD = dyn_cast<FunctionDecl>(Val: CalleeDecl))
550	return FD->getNumParams();
551	return cast<ObjCMethodDecl>(Val: CalleeDecl)->param_size();
552	}
553	const ParmVarDecl *getParamDecl(unsigned I) const {
554	if (const auto *FD = dyn_cast<FunctionDecl>(Val: CalleeDecl))
555	return FD->getParamDecl(i: I);
556	return *(cast<ObjCMethodDecl>(Val: CalleeDecl)->param_begin() + I);
557	}
558	};
559
560	/// Sanitizers enabled for this function.
561	SanitizerSet SanOpts;
562
563	/// True if CodeGen currently emits code implementing sanitizer checks.
564	bool IsSanitizerScope = false;
565
566	/// RAII object to set/unset CodeGenFunction::IsSanitizerScope.
567	class SanitizerScope {
568	CodeGenFunction *CGF;
569
570	public:
571	SanitizerScope(CodeGenFunction *CGF);
572	~SanitizerScope();
573	};
574
575	/// In C++, whether we are code generating a thunk. This controls whether we
576	/// should emit cleanups.
577	bool CurFuncIsThunk = false;
578
579	/// In ARC, whether we should autorelease the return value.
580	bool AutoreleaseResult = false;
581
582	/// Whether we processed a Microsoft-style asm block during CodeGen. These can
583	/// potentially set the return value.
584	bool SawAsmBlock = false;
585
586	GlobalDecl CurSEHParent;
587
588	/// True if the current function is an outlined SEH helper. This can be a
589	/// finally block or filter expression.
590	bool IsOutlinedSEHHelper = false;
591
592	/// True if CodeGen currently emits code inside presereved access index
593	/// region.
594	bool IsInPreservedAIRegion = false;
595
596	/// True if the current statement has nomerge attribute.
597	bool InNoMergeAttributedStmt = false;
598
599	/// True if the current statement has noinline attribute.
600	bool InNoInlineAttributedStmt = false;
601
602	/// True if the current statement has always_inline attribute.
603	bool InAlwaysInlineAttributedStmt = false;
604
605	/// True if the current statement has noconvergent attribute.
606	bool InNoConvergentAttributedStmt = false;
607
608	/// HLSL Branch attribute.
609	HLSLControlFlowHintAttr::Spelling HLSLControlFlowAttr =
610	HLSLControlFlowHintAttr::SpellingNotCalculated;
611
612	// The CallExpr within the current statement that the musttail attribute
613	// applies to. nullptr if there is no 'musttail' on the current statement.
614	const CallExpr *MustTailCall = nullptr;
615
616	/// Returns true if a function must make progress, which means the
617	/// mustprogress attribute can be added.
618	bool checkIfFunctionMustProgress() {
619	if (CGM.getCodeGenOpts().getFiniteLoops() ==
620	CodeGenOptions::FiniteLoopsKind::Never)
621	return false;
622
623	// C++11 and later guarantees that a thread eventually will do one of the
624	// following (C++11 [intro.multithread]p24 and C++17 [intro.progress]p1):
625	// - terminate,
626	// - make a call to a library I/O function,
627	// - perform an access through a volatile glvalue, or
628	// - perform a synchronization operation or an atomic operation.
629	//
630	// Hence each function is 'mustprogress' in C++11 or later.
631	return getLangOpts().CPlusPlus11;
632	}
633
634	/// Returns true if a loop must make progress, which means the mustprogress
635	/// attribute can be added. \p HasConstantCond indicates whether the branch
636	/// condition is a known constant.
637	bool checkIfLoopMustProgress(const Expr *, bool HasEmptyBody);
638
639	const CodeGen::CGBlockInfo *BlockInfo = nullptr;
640	llvm::Value *BlockPointer = nullptr;
641
642	llvm::DenseMap<const ValueDecl *, FieldDecl *> LambdaCaptureFields;
643	FieldDecl *LambdaThisCaptureField = nullptr;
644
645	/// A mapping from NRVO variables to the flags used to indicate
646	/// when the NRVO has been applied to this variable.
647	llvm::DenseMap<const VarDecl *, llvm::Value *> NRVOFlags;
648
649	EHScopeStack EHStack;
650	llvm::SmallVector<char, 256> LifetimeExtendedCleanupStack;
651
652	// A stack of cleanups which were added to EHStack but have to be deactivated
653	// later before being popped or emitted. These are usually deactivated on
654	// exiting a `CleanupDeactivationScope` scope. For instance, after a
655	// full-expr.
656	//
657	// These are specially useful for correctly emitting cleanups while
658	// encountering branches out of expression (through stmt-expr or coroutine
659	// suspensions).
660	struct DeferredDeactivateCleanup {
661	EHScopeStack::stable_iterator Cleanup;
662	llvm::Instruction *DominatingIP;
663	};
664	llvm::SmallVector<DeferredDeactivateCleanup> DeferredDeactivationCleanupStack;
665
666	// Enters a new scope for capturing cleanups which are deferred to be
667	// deactivated, all of which will be deactivated once the scope is exited.
668	struct CleanupDeactivationScope {
669	CodeGenFunction &CGF;
670	size_t OldDeactivateCleanupStackSize;
671	bool Deactivated;
672	CleanupDeactivationScope(CodeGenFunction &CGF)
673	: CGF(CGF), OldDeactivateCleanupStackSize(
674	CGF.DeferredDeactivationCleanupStack.size()),
675	Deactivated(false) {}
676
677	void ForceDeactivate() {
678	assert(!Deactivated && "Deactivating already deactivated scope");
679	auto &Stack = CGF.DeferredDeactivationCleanupStack;
680	for (size_t I = Stack.size(); I > OldDeactivateCleanupStackSize; I--) {
681	CGF.DeactivateCleanupBlock(Cleanup: Stack[I - 1].Cleanup,
682	DominatingIP: Stack[I - 1].DominatingIP);
683	Stack[I - 1].DominatingIP->eraseFromParent();
684	}
685	Stack.resize(N: OldDeactivateCleanupStackSize);
686	Deactivated = true;
687	}
688
689	~CleanupDeactivationScope() {
690	if (Deactivated)
691	return;
692	ForceDeactivate();
693	}
694	};
695
696	llvm::SmallVector<const JumpDest *, 2> SEHTryEpilogueStack;
697
698	llvm::Instruction *CurrentFuncletPad = nullptr;
699
700	class CallLifetimeEnd final : public EHScopeStack::Cleanup {
701	bool isRedundantBeforeReturn() override { return true; }
702
703	llvm::Value *Addr;
704	llvm::Value *Size;
705
706	public:
707	CallLifetimeEnd(RawAddress addr, llvm::Value *size)
708	: Addr(addr.getPointer()), Size(size) {}
709
710	void Emit(CodeGenFunction &CGF, Flags flags) override {
711	CGF.EmitLifetimeEnd(Size, Addr);
712	}
713	};
714
715	// We are using objects of this 'cleanup' class to emit fake.use calls
716	// for -fextend-variable-liveness. They are placed at the end of a variable's
717	// scope analogous to lifetime markers.
718	class FakeUse final : public EHScopeStack::Cleanup {
719	Address Addr;
720
721	public:
722	FakeUse(Address addr) : Addr(addr) {}
723
724	void Emit(CodeGenFunction &CGF, Flags flags) override {
725	CGF.EmitFakeUse(Addr);
726	}
727	};
728
729	/// Header for data within LifetimeExtendedCleanupStack.
730	struct alignas(uint64_t) LifetimeExtendedCleanupHeader {
731	/// The size of the following cleanup object.
732	unsigned Size;
733	/// The kind of cleanup to push.
734	LLVM_PREFERRED_TYPE(CleanupKind)
735	unsigned Kind : 31;
736	/// Whether this is a conditional cleanup.
737	LLVM_PREFERRED_TYPE(bool)
738	unsigned IsConditional : 1;
739
740	size_t getSize() const { return Size; }
741	CleanupKind getKind() const { return (CleanupKind)Kind; }
742	bool isConditional() const { return IsConditional; }
743	};
744
745	/// i32s containing the indexes of the cleanup destinations.
746	RawAddress NormalCleanupDest = RawAddress::invalid();
747
748	unsigned NextCleanupDestIndex = 1;
749
750	/// EHResumeBlock - Unified block containing a call to llvm.eh.resume.
751	llvm::BasicBlock *EHResumeBlock = nullptr;
752
753	/// The exception slot. All landing pads write the current exception pointer
754	/// into this alloca.
755	llvm::Value *ExceptionSlot = nullptr;
756
757	/// The selector slot. Under the MandatoryCleanup model, all landing pads
758	/// write the current selector value into this alloca.
759	llvm::AllocaInst *EHSelectorSlot = nullptr;
760
761	/// A stack of exception code slots. Entering an __except block pushes a slot
762	/// on the stack and leaving pops one. The __exception_code() intrinsic loads
763	/// a value from the top of the stack.
764	SmallVector<Address, 1> SEHCodeSlotStack;
765
766	/// Value returned by __exception_info intrinsic.
767	llvm::Value *SEHInfo = nullptr;
768
769	/// Emits a landing pad for the current EH stack.
770	llvm::BasicBlock *EmitLandingPad();
771
772	llvm::BasicBlock *getInvokeDestImpl();
773
774	/// Parent loop-based directive for scan directive.
775	const OMPExecutableDirective *OMPParentLoopDirectiveForScan = nullptr;
776	llvm::BasicBlock *OMPBeforeScanBlock = nullptr;
777	llvm::BasicBlock *OMPAfterScanBlock = nullptr;
778	llvm::BasicBlock *OMPScanExitBlock = nullptr;
779	llvm::BasicBlock *OMPScanDispatch = nullptr;
780	bool OMPFirstScanLoop = false;
781
782	/// Manages parent directive for scan directives.
783	class ParentLoopDirectiveForScanRegion {
784	CodeGenFunction &CGF;
785	const OMPExecutableDirective *ParentLoopDirectiveForScan;
786
787	public:
788	ParentLoopDirectiveForScanRegion(
789	CodeGenFunction &CGF,
790	const OMPExecutableDirective &ParentLoopDirectiveForScan)
791	: CGF(CGF),
792	ParentLoopDirectiveForScan(CGF.OMPParentLoopDirectiveForScan) {
793	CGF.OMPParentLoopDirectiveForScan = &ParentLoopDirectiveForScan;
794	}
795	~ParentLoopDirectiveForScanRegion() {
796	CGF.OMPParentLoopDirectiveForScan = ParentLoopDirectiveForScan;
797	}
798	};
799
800	template <class T>
801	typename DominatingValue<T>::saved_type saveValueInCond(T value) {
802	return DominatingValue<T>::save(*this, value);
803	}
804
805	class CGFPOptionsRAII {
806	public:
807	CGFPOptionsRAII(CodeGenFunction &CGF, FPOptions FPFeatures);
808	CGFPOptionsRAII(CodeGenFunction &CGF, const Expr *E);
809	~CGFPOptionsRAII();
810
811	private:
812	void ConstructorHelper(FPOptions FPFeatures);
813	CodeGenFunction &CGF;
814	FPOptions OldFPFeatures;
815	llvm::fp::ExceptionBehavior OldExcept;
816	llvm::RoundingMode OldRounding;
817	std::optional<CGBuilderTy::FastMathFlagGuard> FMFGuard;
818	};
819	FPOptions CurFPFeatures;
820
821	class CGAtomicOptionsRAII {
822	public:
823	CGAtomicOptionsRAII(CodeGenModule &CGM_, AtomicOptions AO)
824	: CGM(CGM_), SavedAtomicOpts(CGM.getAtomicOpts()) {
825	CGM.setAtomicOpts(AO);
826	}
827	CGAtomicOptionsRAII(CodeGenModule &CGM_, const AtomicAttr *AA)
828	: CGM(CGM_), SavedAtomicOpts(CGM.getAtomicOpts()) {
829	if (!AA)
830	return;
831	AtomicOptions AO = SavedAtomicOpts;
832	for (auto Option : AA->atomicOptions()) {
833	switch (Option) {
834	case AtomicAttr::remote_memory:
835	AO.remote_memory = true;
836	break;
837	case AtomicAttr::no_remote_memory:
838	AO.remote_memory = false;
839	break;
840	case AtomicAttr::fine_grained_memory:
841	AO.fine_grained_memory = true;
842	break;
843	case AtomicAttr::no_fine_grained_memory:
844	AO.fine_grained_memory = false;
845	break;
846	case AtomicAttr::ignore_denormal_mode:
847	AO.ignore_denormal_mode = true;
848	break;
849	case AtomicAttr::no_ignore_denormal_mode:
850	AO.ignore_denormal_mode = false;
851	break;
852	}
853	}
854	CGM.setAtomicOpts(AO);
855	}
856
857	CGAtomicOptionsRAII(const CGAtomicOptionsRAII &) = delete;
858	CGAtomicOptionsRAII &operator=(const CGAtomicOptionsRAII &) = delete;
859	~CGAtomicOptionsRAII() { CGM.setAtomicOpts(SavedAtomicOpts); }
860
861	private:
862	CodeGenModule &CGM;
863	AtomicOptions SavedAtomicOpts;
864	};
865
866	public:
867	/// ObjCEHValueStack - Stack of Objective-C exception values, used for
868	/// rethrows.
869	SmallVector<llvm::Value *, 8> ObjCEHValueStack;
870
871	/// A class controlling the emission of a finally block.
872	class FinallyInfo {
873	/// Where the catchall's edge through the cleanup should go.
874	JumpDest RethrowDest;
875
876	/// A function to call to enter the catch.
877	llvm::FunctionCallee BeginCatchFn;
878
879	/// An i1 variable indicating whether or not the @finally is
880	/// running for an exception.
881	llvm::AllocaInst *ForEHVar = nullptr;
882
883	/// An i8* variable into which the exception pointer to rethrow
884	/// has been saved.
885	llvm::AllocaInst *SavedExnVar = nullptr;
886
887	public:
888	void enter(CodeGenFunction &CGF, const Stmt *Finally,
889	llvm::FunctionCallee beginCatchFn,
890	llvm::FunctionCallee endCatchFn, llvm::FunctionCallee rethrowFn);
891	void exit(CodeGenFunction &CGF);
892	};
893
894	/// Returns true inside SEH __try blocks.
895	bool isSEHTryScope() const { return !SEHTryEpilogueStack.empty(); }
896
897	/// Returns true while emitting a cleanuppad.
898	bool isCleanupPadScope() const {
899	return CurrentFuncletPad && isa<llvm::CleanupPadInst>(Val: CurrentFuncletPad);
900	}
901
902	/// pushFullExprCleanup - Push a cleanup to be run at the end of the
903	/// current full-expression. Safe against the possibility that
904	/// we're currently inside a conditionally-evaluated expression.
905	template <class T, class... As>
906	void pushFullExprCleanup(CleanupKind kind, As... A) {
907	// If we're not in a conditional branch, or if none of the
908	// arguments requires saving, then use the unconditional cleanup.
909	if (!isInConditionalBranch())
910	return EHStack.pushCleanup<T>(kind, A...);
911
912	// Stash values in a tuple so we can guarantee the order of saves.
913	typedef std::tuple<typename DominatingValue<As>::saved_type...> SavedTuple;
914	SavedTuple Saved{saveValueInCond(A)...};
915
916	typedef EHScopeStack::ConditionalCleanup<T, As...> CleanupType;
917	EHStack.pushCleanupTuple<CleanupType>(kind, Saved);
918	initFullExprCleanup();
919	}
920
921	/// Queue a cleanup to be pushed after finishing the current full-expression,
922	/// potentially with an active flag.
923	template <class T, class... As>
924	void pushCleanupAfterFullExpr(CleanupKind Kind, As... A) {
925	if (!isInConditionalBranch())
926	return pushCleanupAfterFullExprWithActiveFlag<T>(
927	Kind, RawAddress::invalid(), A...);
928
929	RawAddress ActiveFlag = createCleanupActiveFlag();
930	assert(!DominatingValue<Address>::needsSaving(ActiveFlag) &&
931	"cleanup active flag should never need saving");
932
933	typedef std::tuple<typename DominatingValue<As>::saved_type...> SavedTuple;
934	SavedTuple Saved{saveValueInCond(A)...};
935
936	typedef EHScopeStack::ConditionalCleanup<T, As...> CleanupType;
937	pushCleanupAfterFullExprWithActiveFlag<CleanupType>(Kind, ActiveFlag,
938	Saved);
939	}
940
941	template <class T, class... As>
942	void pushCleanupAfterFullExprWithActiveFlag(CleanupKind Kind,
943	RawAddress ActiveFlag, As... A) {
944	LifetimeExtendedCleanupHeader Header = {.Size: sizeof(T), .Kind: Kind,
945	.IsConditional: ActiveFlag.isValid()};
946
947	size_t OldSize = LifetimeExtendedCleanupStack.size();
948	LifetimeExtendedCleanupStack.resize(
949	N: LifetimeExtendedCleanupStack.size() + sizeof(Header) + Header.Size +
950	(Header.IsConditional ? sizeof(ActiveFlag) : 0));
951
952	static_assert((alignof(LifetimeExtendedCleanupHeader) == alignof(T)) &&
953	(alignof(T) == alignof(RawAddress)),
954	"Cleanup will be allocated on misaligned address");
955	char *Buffer = &LifetimeExtendedCleanupStack[OldSize];
956	new (Buffer) LifetimeExtendedCleanupHeader(Header);
957	new (Buffer + sizeof(Header)) T(A...);
958	if (Header.IsConditional)
959	new (Buffer + sizeof(Header) + sizeof(T)) RawAddress(ActiveFlag);
960	}
961
962	// Push a cleanup onto EHStack and deactivate it later. It is usually
963	// deactivated when exiting a `CleanupDeactivationScope` (for example: after a
964	// full expression).
965	template <class T, class... As>
966	void pushCleanupAndDeferDeactivation(CleanupKind Kind, As... A) {
967	// Placeholder dominating IP for this cleanup.
968	llvm::Instruction *DominatingIP =
969	Builder.CreateFlagLoad(Addr: llvm::Constant::getNullValue(Ty: Int8PtrTy));
970	EHStack.pushCleanup<T>(Kind, A...);
971	DeferredDeactivationCleanupStack.push_back(
972	Elt: {.Cleanup: EHStack.stable_begin(), .DominatingIP: DominatingIP});
973	}
974
975	/// Set up the last cleanup that was pushed as a conditional
976	/// full-expression cleanup.
977	void initFullExprCleanup() {
978	initFullExprCleanupWithFlag(ActiveFlag: createCleanupActiveFlag());
979	}
980
981	void initFullExprCleanupWithFlag(RawAddress ActiveFlag);
982	RawAddress createCleanupActiveFlag();
983
984	/// PushDestructorCleanup - Push a cleanup to call the
985	/// complete-object destructor of an object of the given type at the
986	/// given address. Does nothing if T is not a C++ class type with a
987	/// non-trivial destructor.
988	void PushDestructorCleanup(QualType T, Address Addr);
989
990	/// PushDestructorCleanup - Push a cleanup to call the
991	/// complete-object variant of the given destructor on the object at
992	/// the given address.
993	void PushDestructorCleanup(const CXXDestructorDecl *Dtor, QualType T,
994	Address Addr);
995
996	/// PopCleanupBlock - Will pop the cleanup entry on the stack and
997	/// process all branch fixups.
998	void PopCleanupBlock(bool FallThroughIsBranchThrough = false,
999	bool ForDeactivation = false);
1000
1001	/// DeactivateCleanupBlock - Deactivates the given cleanup block.
1002	/// The block cannot be reactivated. Pops it if it's the top of the
1003	/// stack.
1004	///
1005	/// \param DominatingIP - An instruction which is known to
1006	/// dominate the current IP (if set) and which lies along
1007	/// all paths of execution between the current IP and the
1008	/// the point at which the cleanup comes into scope.
1009	void DeactivateCleanupBlock(EHScopeStack::stable_iterator Cleanup,
1010	llvm::Instruction *DominatingIP);
1011
1012	/// ActivateCleanupBlock - Activates an initially-inactive cleanup.
1013	/// Cannot be used to resurrect a deactivated cleanup.
1014	///
1015	/// \param DominatingIP - An instruction which is known to
1016	/// dominate the current IP (if set) and which lies along
1017	/// all paths of execution between the current IP and the
1018	/// the point at which the cleanup comes into scope.
1019	void ActivateCleanupBlock(EHScopeStack::stable_iterator Cleanup,
1020	llvm::Instruction *DominatingIP);
1021
1022	/// Enters a new scope for capturing cleanups, all of which
1023	/// will be executed once the scope is exited.
1024	class RunCleanupsScope {
1025	EHScopeStack::stable_iterator CleanupStackDepth, OldCleanupScopeDepth;
1026	size_t LifetimeExtendedCleanupStackSize;
1027	CleanupDeactivationScope DeactivateCleanups;
1028	bool OldDidCallStackSave;
1029
1030	protected:
1031	bool PerformCleanup;
1032
1033	private:
1034	RunCleanupsScope(const RunCleanupsScope &) = delete;
1035	void operator=(const RunCleanupsScope &) = delete;
1036
1037	protected:
1038	CodeGenFunction &CGF;
1039
1040	public:
1041	/// Enter a new cleanup scope.
1042	explicit RunCleanupsScope(CodeGenFunction &CGF)
1043	: DeactivateCleanups(CGF), PerformCleanup(true), CGF(CGF) {
1044	CleanupStackDepth = CGF.EHStack.stable_begin();
1045	LifetimeExtendedCleanupStackSize =
1046	CGF.LifetimeExtendedCleanupStack.size();
1047	OldDidCallStackSave = CGF.DidCallStackSave;
1048	CGF.DidCallStackSave = false;
1049	OldCleanupScopeDepth = CGF.CurrentCleanupScopeDepth;
1050	CGF.CurrentCleanupScopeDepth = CleanupStackDepth;
1051	}
1052
1053	/// Exit this cleanup scope, emitting any accumulated cleanups.
1054	~RunCleanupsScope() {
1055	if (PerformCleanup)
1056	ForceCleanup();
1057	}
1058
1059	/// Determine whether this scope requires any cleanups.
1060	bool requiresCleanups() const {
1061	return CGF.EHStack.stable_begin() != CleanupStackDepth;
1062	}
1063
1064	/// Force the emission of cleanups now, instead of waiting
1065	/// until this object is destroyed.
1066	/// \param ValuesToReload - A list of values that need to be available at
1067	/// the insertion point after cleanup emission. If cleanup emission created
1068	/// a shared cleanup block, these value pointers will be rewritten.
1069	/// Otherwise, they not will be modified.
1070	void
1071	ForceCleanup(std::initializer_list<llvm::Value **> ValuesToReload = {}) {
1072	assert(PerformCleanup && "Already forced cleanup");
1073	CGF.DidCallStackSave = OldDidCallStackSave;
1074	DeactivateCleanups.ForceDeactivate();
1075	CGF.PopCleanupBlocks(OldCleanupStackSize: CleanupStackDepth, OldLifetimeExtendedStackSize: LifetimeExtendedCleanupStackSize,
1076	ValuesToReload);
1077	PerformCleanup = false;
1078	CGF.CurrentCleanupScopeDepth = OldCleanupScopeDepth;
1079	}
1080	};
1081
1082	// Cleanup stack depth of the RunCleanupsScope that was pushed most recently.
1083	EHScopeStack::stable_iterator CurrentCleanupScopeDepth =
1084	EHScopeStack::stable_end();
1085
1086	class LexicalScope : public RunCleanupsScope {
1087	SourceRange Range;
1088	SmallVector<const LabelDecl *, 4> Labels;
1089	LexicalScope *ParentScope;
1090
1091	LexicalScope(const LexicalScope &) = delete;
1092	void operator=(const LexicalScope &) = delete;
1093
1094	public:
1095	/// Enter a new cleanup scope.
1096	explicit LexicalScope(CodeGenFunction &CGF, SourceRange Range);
1097
1098	void addLabel(const LabelDecl *label) {
1099	assert(PerformCleanup && "adding label to dead scope?");
1100	Labels.push_back(Elt: label);
1101	}
1102
1103	/// Exit this cleanup scope, emitting any accumulated
1104	/// cleanups.
1105	~LexicalScope();
1106
1107	/// Force the emission of cleanups now, instead of waiting
1108	/// until this object is destroyed.
1109	void ForceCleanup() {
1110	CGF.CurLexicalScope = ParentScope;
1111	RunCleanupsScope::ForceCleanup();
1112
1113	if (!Labels.empty())
1114	rescopeLabels();
1115	}
1116
1117	bool hasLabels() const { return !Labels.empty(); }
1118
1119	void rescopeLabels();
1120	};
1121
1122	typedef llvm::DenseMap<const Decl *, Address> DeclMapTy;
1123
1124	/// The class used to assign some variables some temporarily addresses.
1125	class OMPMapVars {
1126	DeclMapTy SavedLocals;
1127	DeclMapTy SavedTempAddresses;
1128	OMPMapVars(const OMPMapVars &) = delete;
1129	void operator=(const OMPMapVars &) = delete;
1130
1131	public:
1132	explicit OMPMapVars() = default;
1133	~OMPMapVars() {
1134	assert(SavedLocals.empty() && "Did not restored original addresses.");
1135	};
1136
1137	/// Sets the address of the variable \p LocalVD to be \p TempAddr in
1138	/// function \p CGF.
1139	/// \return true if at least one variable was set already, false otherwise.
1140	bool setVarAddr(CodeGenFunction &CGF, const VarDecl *LocalVD,
1141	Address TempAddr) {
1142	LocalVD = LocalVD->getCanonicalDecl();
1143	// Only save it once.
1144	if (SavedLocals.count(Val: LocalVD))
1145	return false;
1146
1147	// Copy the existing local entry to SavedLocals.
1148	auto it = CGF.LocalDeclMap.find(Val: LocalVD);
1149	if (it != CGF.LocalDeclMap.end())
1150	SavedLocals.try_emplace(Key: LocalVD, Args&: it->second);
1151	else
1152	SavedLocals.try_emplace(Key: LocalVD, Args: Address::invalid());
1153
1154	// Generate the private entry.
1155	QualType VarTy = LocalVD->getType();
1156	if (VarTy->isReferenceType()) {
1157	Address Temp = CGF.CreateMemTemp(T: VarTy);
1158	CGF.Builder.CreateStore(Val: TempAddr.emitRawPointer(CGF), Addr: Temp);
1159	TempAddr = Temp;
1160	}
1161	SavedTempAddresses.try_emplace(Key: LocalVD, Args&: TempAddr);
1162
1163	return true;
1164	}
1165
1166	/// Applies new addresses to the list of the variables.
1167	/// \return true if at least one variable is using new address, false
1168	/// otherwise.
1169	bool apply(CodeGenFunction &CGF) {
1170	copyInto(Src: SavedTempAddresses, Dest&: CGF.LocalDeclMap);
1171	SavedTempAddresses.clear();
1172	return !SavedLocals.empty();
1173	}
1174
1175	/// Restores original addresses of the variables.
1176	void restore(CodeGenFunction &CGF) {
1177	if (!SavedLocals.empty()) {
1178	copyInto(Src: SavedLocals, Dest&: CGF.LocalDeclMap);
1179	SavedLocals.clear();
1180	}
1181	}
1182
1183	private:
1184	/// Copy all the entries in the source map over the corresponding
1185	/// entries in the destination, which must exist.
1186	static void copyInto(const DeclMapTy &Src, DeclMapTy &Dest) {
1187	for (auto &[Decl, Addr] : Src) {
1188	if (!Addr.isValid())
1189	Dest.erase(Val: Decl);
1190	else
1191	Dest.insert_or_assign(Key: Decl, Val: Addr);
1192	}
1193	}
1194	};
1195
1196	/// The scope used to remap some variables as private in the OpenMP loop body
1197	/// (or other captured region emitted without outlining), and to restore old
1198	/// vars back on exit.
1199	class OMPPrivateScope : public RunCleanupsScope {
1200	OMPMapVars MappedVars;
1201	OMPPrivateScope(const OMPPrivateScope &) = delete;
1202	void operator=(const OMPPrivateScope &) = delete;
1203
1204	public:
1205	/// Enter a new OpenMP private scope.
1206	explicit OMPPrivateScope(CodeGenFunction &CGF) : RunCleanupsScope(CGF) {}
1207
1208	/// Registers \p LocalVD variable as a private with \p Addr as the address
1209	/// of the corresponding private variable. \p
1210	/// PrivateGen is the address of the generated private variable.
1211	/// \return true if the variable is registered as private, false if it has
1212	/// been privatized already.
1213	bool addPrivate(const VarDecl *LocalVD, Address Addr) {
1214	assert(PerformCleanup && "adding private to dead scope");
1215	return MappedVars.setVarAddr(CGF, LocalVD, TempAddr: Addr);
1216	}
1217
1218	/// Privatizes local variables previously registered as private.
1219	/// Registration is separate from the actual privatization to allow
1220	/// initializers use values of the original variables, not the private one.
1221	/// This is important, for example, if the private variable is a class
1222	/// variable initialized by a constructor that references other private
1223	/// variables. But at initialization original variables must be used, not
1224	/// private copies.
1225	/// \return true if at least one variable was privatized, false otherwise.
1226	bool Privatize() { return MappedVars.apply(CGF); }
1227
1228	void ForceCleanup() {
1229	RunCleanupsScope::ForceCleanup();
1230	restoreMap();
1231	}
1232
1233	/// Exit scope - all the mapped variables are restored.
1234	~OMPPrivateScope() {
1235	if (PerformCleanup)
1236	ForceCleanup();
1237	}
1238
1239	/// Checks if the global variable is captured in current function.
1240	bool isGlobalVarCaptured(const VarDecl *VD) const {
1241	VD = VD->getCanonicalDecl();
1242	return !VD->isLocalVarDeclOrParm() && CGF.LocalDeclMap.count(Val: VD) > 0;
1243	}
1244
1245	/// Restore all mapped variables w/o clean up. This is usefully when we want
1246	/// to reference the original variables but don't want the clean up because
1247	/// that could emit lifetime end too early, causing backend issue #56913.
1248	void restoreMap() { MappedVars.restore(CGF); }
1249	};
1250
1251	/// Save/restore original map of previously emitted local vars in case when we
1252	/// need to duplicate emission of the same code several times in the same
1253	/// function for OpenMP code.
1254	class OMPLocalDeclMapRAII {
1255	CodeGenFunction &CGF;
1256	DeclMapTy SavedMap;
1257
1258	public:
1259	OMPLocalDeclMapRAII(CodeGenFunction &CGF)
1260	: CGF(CGF), SavedMap(CGF.LocalDeclMap) {}
1261	~OMPLocalDeclMapRAII() { SavedMap.swap(RHS&: CGF.LocalDeclMap); }
1262	};
1263
1264	/// Takes the old cleanup stack size and emits the cleanup blocks
1265	/// that have been added.
1266	void
1267	PopCleanupBlocks(EHScopeStack::stable_iterator OldCleanupStackSize,
1268	std::initializer_list<llvm::Value **> ValuesToReload = {});
1269
1270	/// Takes the old cleanup stack size and emits the cleanup blocks
1271	/// that have been added, then adds all lifetime-extended cleanups from
1272	/// the given position to the stack.
1273	void
1274	PopCleanupBlocks(EHScopeStack::stable_iterator OldCleanupStackSize,
1275	size_t OldLifetimeExtendedStackSize,
1276	std::initializer_list<llvm::Value **> ValuesToReload = {});
1277
1278	void ResolveBranchFixups(llvm::BasicBlock *Target);
1279
1280	/// The given basic block lies in the current EH scope, but may be a
1281	/// target of a potentially scope-crossing jump; get a stable handle
1282	/// to which we can perform this jump later.
1283	JumpDest getJumpDestInCurrentScope(llvm::BasicBlock *Target) {
1284	return JumpDest(Target, EHStack.getInnermostNormalCleanup(),
1285	NextCleanupDestIndex++);
1286	}
1287
1288	/// The given basic block lies in the current EH scope, but may be a
1289	/// target of a potentially scope-crossing jump; get a stable handle
1290	/// to which we can perform this jump later.
1291	JumpDest getJumpDestInCurrentScope(StringRef Name = StringRef()) {
1292	return getJumpDestInCurrentScope(Target: createBasicBlock(name: Name));
1293	}
1294
1295	/// EmitBranchThroughCleanup - Emit a branch from the current insert
1296	/// block through the normal cleanup handling code (if any) and then
1297	/// on to \arg Dest.
1298	void EmitBranchThroughCleanup(JumpDest Dest);
1299
1300	/// isObviouslyBranchWithoutCleanups - Return true if a branch to the
1301	/// specified destination obviously has no cleanups to run. 'false' is always
1302	/// a conservatively correct answer for this method.
1303	bool isObviouslyBranchWithoutCleanups(JumpDest Dest) const;
1304
1305	/// popCatchScope - Pops the catch scope at the top of the EHScope
1306	/// stack, emitting any required code (other than the catch handlers
1307	/// themselves).
1308	void popCatchScope();
1309
1310	llvm::BasicBlock *getEHResumeBlock(bool isCleanup);
1311	llvm::BasicBlock *getEHDispatchBlock(EHScopeStack::stable_iterator scope);
1312	llvm::BasicBlock *
1313	getFuncletEHDispatchBlock(EHScopeStack::stable_iterator scope);
1314
1315	/// An object to manage conditionally-evaluated expressions.
1316	class ConditionalEvaluation {
1317	llvm::BasicBlock *StartBB;
1318
1319	public:
1320	ConditionalEvaluation(CodeGenFunction &CGF)
1321	: StartBB(CGF.Builder.GetInsertBlock()) {}
1322
1323	void begin(CodeGenFunction &CGF) {
1324	assert(CGF.OutermostConditional != this);
1325	if (!CGF.OutermostConditional)
1326	CGF.OutermostConditional = this;
1327	}
1328
1329	void end(CodeGenFunction &CGF) {
1330	assert(CGF.OutermostConditional != nullptr);
1331	if (CGF.OutermostConditional == this)
1332	CGF.OutermostConditional = nullptr;
1333	}
1334
1335	/// Returns a block which will be executed prior to each
1336	/// evaluation of the conditional code.
1337	llvm::BasicBlock *getStartingBlock() const { return StartBB; }
1338	};
1339
1340	/// isInConditionalBranch - Return true if we're currently emitting
1341	/// one branch or the other of a conditional expression.
1342	bool isInConditionalBranch() const { return OutermostConditional != nullptr; }
1343
1344	void setBeforeOutermostConditional(llvm::Value *value, Address addr,
1345	CodeGenFunction &CGF) {
1346	assert(isInConditionalBranch());
1347	llvm::BasicBlock *block = OutermostConditional->getStartingBlock();
1348	auto store = new llvm::StoreInst(value, addr.emitRawPointer(CGF),
1349	block->back().getIterator());
1350	store->setAlignment(addr.getAlignment().getAsAlign());
1351	}
1352
1353	/// An RAII object to record that we're evaluating a statement
1354	/// expression.
1355	class StmtExprEvaluation {
1356	CodeGenFunction &CGF;
1357
1358	/// We have to save the outermost conditional: cleanups in a
1359	/// statement expression aren't conditional just because the
1360	/// StmtExpr is.
1361	ConditionalEvaluation *SavedOutermostConditional;
1362
1363	public:
1364	StmtExprEvaluation(CodeGenFunction &CGF)
1365	: CGF(CGF), SavedOutermostConditional(CGF.OutermostConditional) {
1366	CGF.OutermostConditional = nullptr;
1367	}
1368
1369	~StmtExprEvaluation() {
1370	CGF.OutermostConditional = SavedOutermostConditional;
1371	CGF.EnsureInsertPoint();
1372	}
1373	};
1374
1375	/// An object which temporarily prevents a value from being
1376	/// destroyed by aggressive peephole optimizations that assume that
1377	/// all uses of a value have been realized in the IR.
1378	class PeepholeProtection {
1379	llvm::Instruction *Inst = nullptr;
1380	friend class CodeGenFunction;
1381
1382	public:
1383	PeepholeProtection() = default;
1384	};
1385
1386	/// A non-RAII class containing all the information about a bound
1387	/// opaque value. OpaqueValueMapping, below, is a RAII wrapper for
1388	/// this which makes individual mappings very simple; using this
1389	/// class directly is useful when you have a variable number of
1390	/// opaque values or don't want the RAII functionality for some
1391	/// reason.
1392	class OpaqueValueMappingData {
1393	const OpaqueValueExpr *OpaqueValue;
1394	bool BoundLValue;
1395	CodeGenFunction::PeepholeProtection Protection;
1396
1397	OpaqueValueMappingData(const OpaqueValueExpr *ov, bool boundLValue)
1398	: OpaqueValue(ov), BoundLValue(boundLValue) {}
1399
1400	public:
1401	OpaqueValueMappingData() : OpaqueValue(nullptr) {}
1402
1403	static bool shouldBindAsLValue(const Expr *expr) {
1404	// gl-values should be bound as l-values for obvious reasons.
1405	// Records should be bound as l-values because IR generation
1406	// always keeps them in memory. Expressions of function type
1407	// act exactly like l-values but are formally required to be
1408	// r-values in C.
1409	return expr->isGLValue() || expr->getType()->isFunctionType() ||
1410	hasAggregateEvaluationKind(T: expr->getType());
1411	}
1412
1413	static OpaqueValueMappingData
1414	bind(CodeGenFunction &CGF, const OpaqueValueExpr *ov, const Expr *e) {
1415	if (shouldBindAsLValue(expr: ov))
1416	return bind(CGF, ov, lv: CGF.EmitLValue(E: e));
1417	return bind(CGF, ov, rv: CGF.EmitAnyExpr(E: e));
1418	}
1419
1420	static OpaqueValueMappingData
1421	bind(CodeGenFunction &CGF, const OpaqueValueExpr *ov, const LValue &lv) {
1422	assert(shouldBindAsLValue(ov));
1423	CGF.OpaqueLValues.insert(KV: std::make_pair(x&: ov, y: lv));
1424	return OpaqueValueMappingData(ov, true);
1425	}
1426
1427	static OpaqueValueMappingData
1428	bind(CodeGenFunction &CGF, const OpaqueValueExpr *ov, const RValue &rv) {
1429	assert(!shouldBindAsLValue(ov));
1430	CGF.OpaqueRValues.insert(KV: std::make_pair(x&: ov, y: rv));
1431
1432	OpaqueValueMappingData data(ov, false);
1433
1434	// Work around an extremely aggressive peephole optimization in
1435	// EmitScalarConversion which assumes that all other uses of a
1436	// value are extant.
1437	data.Protection = CGF.protectFromPeepholes(rvalue: rv);
1438
1439	return data;
1440	}
1441
1442	bool isValid() const { return OpaqueValue != nullptr; }
1443	void clear() { OpaqueValue = nullptr; }
1444
1445	void unbind(CodeGenFunction &CGF) {
1446	assert(OpaqueValue && "no data to unbind!");
1447
1448	if (BoundLValue) {
1449	CGF.OpaqueLValues.erase(Val: OpaqueValue);
1450	} else {
1451	CGF.OpaqueRValues.erase(Val: OpaqueValue);
1452	CGF.unprotectFromPeepholes(protection: Protection);
1453	}
1454	}
1455	};
1456
1457	/// An RAII object to set (and then clear) a mapping for an OpaqueValueExpr.
1458	class OpaqueValueMapping {
1459	CodeGenFunction &CGF;
1460	OpaqueValueMappingData Data;
1461
1462	public:
1463	static bool shouldBindAsLValue(const Expr *expr) {
1464	return OpaqueValueMappingData::shouldBindAsLValue(expr);
1465	}
1466
1467	/// Build the opaque value mapping for the given conditional
1468	/// operator if it's the GNU ?: extension. This is a common
1469	/// enough pattern that the convenience operator is really
1470	/// helpful.
1471	///
1472	OpaqueValueMapping(CodeGenFunction &CGF,
1473	const AbstractConditionalOperator *op)
1474	: CGF(CGF) {
1475	if (isa<ConditionalOperator>(Val: op))
1476	// Leave Data empty.
1477	return;
1478
1479	const BinaryConditionalOperator *e = cast<BinaryConditionalOperator>(Val: op);
1480	Data = OpaqueValueMappingData::bind(CGF, ov: e->getOpaqueValue(),
1481	e: e->getCommon());
1482	}
1483
1484	/// Build the opaque value mapping for an OpaqueValueExpr whose source
1485	/// expression is set to the expression the OVE represents.
1486	OpaqueValueMapping(CodeGenFunction &CGF, const OpaqueValueExpr *OV)
1487	: CGF(CGF) {
1488	if (OV) {
1489	assert(OV->getSourceExpr() && "wrong form of OpaqueValueMapping used "
1490	"for OVE with no source expression");
1491	Data = OpaqueValueMappingData::bind(CGF, ov: OV, e: OV->getSourceExpr());
1492	}
1493	}
1494
1495	OpaqueValueMapping(CodeGenFunction &CGF, const OpaqueValueExpr *opaqueValue,
1496	LValue lvalue)
1497	: CGF(CGF),
1498	Data(OpaqueValueMappingData::bind(CGF, ov: opaqueValue, lv: lvalue)) {}
1499
1500	OpaqueValueMapping(CodeGenFunction &CGF, const OpaqueValueExpr *opaqueValue,
1501	RValue rvalue)
1502	: CGF(CGF),
1503	Data(OpaqueValueMappingData::bind(CGF, ov: opaqueValue, rv: rvalue)) {}
1504
1505	void pop() {
1506	Data.unbind(CGF);
1507	Data.clear();
1508	}
1509
1510	~OpaqueValueMapping() {
1511	if (Data.isValid())
1512	Data.unbind(CGF);
1513	}
1514	};
1515
1516	private:
1517	CGDebugInfo *DebugInfo;
1518	/// Used to create unique names for artificial VLA size debug info variables.
1519	unsigned VLAExprCounter = 0;
1520	bool DisableDebugInfo = false;
1521
1522	/// DidCallStackSave - Whether llvm.stacksave has been called. Used to avoid
1523	/// calling llvm.stacksave for multiple VLAs in the same scope.
1524	bool DidCallStackSave = false;
1525
1526	/// IndirectBranch - The first time an indirect goto is seen we create a block
1527	/// with an indirect branch. Every time we see the address of a label taken,
1528	/// we add the label to the indirect goto. Every subsequent indirect goto is
1529	/// codegen'd as a jump to the IndirectBranch's basic block.
1530	llvm::IndirectBrInst *IndirectBranch = nullptr;
1531
1532	/// LocalDeclMap - This keeps track of the LLVM allocas or globals for local C
1533	/// decls.
1534	DeclMapTy LocalDeclMap;
1535
1536	// Keep track of the cleanups for callee-destructed parameters pushed to the
1537	// cleanup stack so that they can be deactivated later.
1538	llvm::DenseMap<const ParmVarDecl *, EHScopeStack::stable_iterator>
1539	CalleeDestructedParamCleanups;
1540
1541	/// SizeArguments - If a ParmVarDecl had the pass_object_size attribute, this
1542	/// will contain a mapping from said ParmVarDecl to its implicit "object_size"
1543	/// parameter.
1544	llvm::SmallDenseMap<const ParmVarDecl *, const ImplicitParamDecl *, 2>
1545	SizeArguments;
1546
1547	/// Track escaped local variables with auto storage. Used during SEH
1548	/// outlining to produce a call to llvm.localescape.
1549	llvm::DenseMap<llvm::AllocaInst *, int> EscapedLocals;
1550
1551	/// LabelMap - This keeps track of the LLVM basic block for each C label.
1552	llvm::DenseMap<const LabelDecl *, JumpDest> LabelMap;
1553
1554	// BreakContinueStack - This keeps track of where break and continue
1555	// statements should jump to.
1556	struct BreakContinue {
1557	BreakContinue(JumpDest Break, JumpDest Continue)
1558	: BreakBlock(Break), ContinueBlock(Continue) {}
1559
1560	JumpDest BreakBlock;
1561	JumpDest ContinueBlock;
1562	};
1563	SmallVector<BreakContinue, 8> BreakContinueStack;
1564
1565	/// Handles cancellation exit points in OpenMP-related constructs.
1566	class OpenMPCancelExitStack {
1567	/// Tracks cancellation exit point and join point for cancel-related exit
1568	/// and normal exit.
1569	struct CancelExit {
1570	CancelExit() = default;
1571	CancelExit(OpenMPDirectiveKind Kind, JumpDest ExitBlock,
1572	JumpDest ContBlock)
1573	: Kind(Kind), ExitBlock(ExitBlock), ContBlock(ContBlock) {}
1574	OpenMPDirectiveKind Kind = llvm::omp::OMPD_unknown;
1575	/// true if the exit block has been emitted already by the special
1576	/// emitExit() call, false if the default codegen is used.
1577	bool HasBeenEmitted = false;
1578	JumpDest ExitBlock;
1579	JumpDest ContBlock;
1580	};
1581
1582	SmallVector<CancelExit, 8> Stack;
1583
1584	public:
1585	OpenMPCancelExitStack() : Stack(1) {}
1586	~OpenMPCancelExitStack() = default;
1587	/// Fetches the exit block for the current OpenMP construct.
1588	JumpDest getExitBlock() const { return Stack.back().ExitBlock; }
1589	/// Emits exit block with special codegen procedure specific for the related
1590	/// OpenMP construct + emits code for normal construct cleanup.
1591	void emitExit(CodeGenFunction &CGF, OpenMPDirectiveKind Kind,
1592	const llvm::function_ref<void(CodeGenFunction &)> CodeGen) {
1593	if (Stack.back().Kind == Kind && getExitBlock().isValid()) {
1594	assert(CGF.getOMPCancelDestination(Kind).isValid());
1595	assert(CGF.HaveInsertPoint());
1596	assert(!Stack.back().HasBeenEmitted);
1597	auto IP = CGF.Builder.saveAndClearIP();
1598	CGF.EmitBlock(BB: Stack.back().ExitBlock.getBlock());
1599	CodeGen(CGF);
1600	CGF.EmitBranch(Block: Stack.back().ContBlock.getBlock());
1601	CGF.Builder.restoreIP(IP);
1602	Stack.back().HasBeenEmitted = true;
1603	}
1604	CodeGen(CGF);
1605	}
1606	/// Enter the cancel supporting \a Kind construct.
1607	/// \param Kind OpenMP directive that supports cancel constructs.
1608	/// \param HasCancel true, if the construct has inner cancel directive,
1609	/// false otherwise.
1610	void enter(CodeGenFunction &CGF, OpenMPDirectiveKind Kind, bool HasCancel) {
1611	Stack.push_back(Elt: {Kind,
1612	HasCancel ? CGF.getJumpDestInCurrentScope(Name: "cancel.exit")
1613	: JumpDest(),
1614	HasCancel ? CGF.getJumpDestInCurrentScope(Name: "cancel.cont")
1615	: JumpDest()});
1616	}
1617	/// Emits default exit point for the cancel construct (if the special one
1618	/// has not be used) + join point for cancel/normal exits.
1619	void exit(CodeGenFunction &CGF) {
1620	if (getExitBlock().isValid()) {
1621	assert(CGF.getOMPCancelDestination(Stack.back().Kind).isValid());
1622	bool HaveIP = CGF.HaveInsertPoint();
1623	if (!Stack.back().HasBeenEmitted) {
1624	if (HaveIP)
1625	CGF.EmitBranchThroughCleanup(Dest: Stack.back().ContBlock);
1626	CGF.EmitBlock(BB: Stack.back().ExitBlock.getBlock());
1627	CGF.EmitBranchThroughCleanup(Dest: Stack.back().ContBlock);
1628	}
1629	CGF.EmitBlock(BB: Stack.back().ContBlock.getBlock());
1630	if (!HaveIP) {
1631	CGF.Builder.CreateUnreachable();
1632	CGF.Builder.ClearInsertionPoint();
1633	}
1634	}
1635	Stack.pop_back();
1636	}
1637	};
1638	OpenMPCancelExitStack OMPCancelStack;
1639
1640	/// Lower the Likelihood knowledge about the \p Cond via llvm.expect intrin.
1641	llvm::Value *emitCondLikelihoodViaExpectIntrinsic(llvm::Value *Cond,
1642	Stmt::Likelihood LH);
1643
1644	std::unique_ptr<CodeGenPGO> PGO;
1645
1646	/// Bitmap used by MC/DC to track condition outcomes of a boolean expression.
1647	Address MCDCCondBitmapAddr = Address::invalid();
1648
1649	/// Calculate branch weights appropriate for PGO data
1650	llvm::MDNode *createProfileWeights(uint64_t TrueCount,
1651	uint64_t FalseCount) const;
1652	llvm::MDNode *createProfileWeights(ArrayRef<uint64_t> Weights) const;
1653	llvm::MDNode *createProfileWeightsForLoop(const Stmt *Cond,
1654	uint64_t LoopCount) const;
1655
1656	public:
1657	std::pair<bool, bool> getIsCounterPair(const Stmt *S) const;
1658	void markStmtAsUsed(bool Skipped, const Stmt *S);
1659	void markStmtMaybeUsed(const Stmt *S);
1660
1661	/// Increment the profiler's counter for the given statement by \p StepV.
1662	/// If \p StepV is null, the default increment is 1.
1663	void incrementProfileCounter(const Stmt *S, llvm::Value *StepV = nullptr);
1664
1665	bool isMCDCCoverageEnabled() const {
1666	return (CGM.getCodeGenOpts().hasProfileClangInstr() &&
1667	CGM.getCodeGenOpts().MCDCCoverage &&
1668	!CurFn->hasFnAttribute(Kind: llvm::Attribute::NoProfile));
1669	}
1670
1671	/// Allocate a temp value on the stack that MCDC can use to track condition
1672	/// results.
1673	void maybeCreateMCDCCondBitmap();
1674
1675	bool isBinaryLogicalOp(const Expr *E) const {
1676	const BinaryOperator *BOp = dyn_cast<BinaryOperator>(Val: E->IgnoreParens());
1677	return (BOp && BOp->isLogicalOp());
1678	}
1679
1680	/// Zero-init the MCDC temp value.
1681	void maybeResetMCDCCondBitmap(const Expr *E);
1682
1683	/// Increment the profiler's counter for the given expression by \p StepV.
1684	/// If \p StepV is null, the default increment is 1.
1685	void maybeUpdateMCDCTestVectorBitmap(const Expr *E);
1686
1687	/// Update the MCDC temp value with the condition's evaluated result.
1688	void maybeUpdateMCDCCondBitmap(const Expr *E, llvm::Value *Val);
1689
1690	/// Get the profiler's count for the given statement.
1691	uint64_t getProfileCount(const Stmt *S);
1692
1693	/// Set the profiler's current count.
1694	void setCurrentProfileCount(uint64_t Count);
1695
1696	/// Get the profiler's current count. This is generally the count for the most
1697	/// recently incremented counter.
1698	uint64_t getCurrentProfileCount();
1699
1700	/// See CGDebugInfo::addInstToCurrentSourceAtom.
1701	void addInstToCurrentSourceAtom(llvm::Instruction *KeyInstruction,
1702	llvm::Value *Backup);
1703
1704	/// See CGDebugInfo::addInstToSpecificSourceAtom.
1705	void addInstToSpecificSourceAtom(llvm::Instruction *KeyInstruction,
1706	llvm::Value *Backup, uint64_t Atom);
1707
1708	/// Add \p KeyInstruction and an optional \p Backup instruction to a new atom
1709	/// group (See ApplyAtomGroup for more info).
1710	void addInstToNewSourceAtom(llvm::Instruction *KeyInstruction,
1711	llvm::Value *Backup);
1712
1713	private:
1714	/// SwitchInsn - This is nearest current switch instruction. It is null if
1715	/// current context is not in a switch.
1716	llvm::SwitchInst *SwitchInsn = nullptr;
1717	/// The branch weights of SwitchInsn when doing instrumentation based PGO.
1718	SmallVector<uint64_t, 16> *SwitchWeights = nullptr;
1719
1720	/// The likelihood attributes of the SwitchCase.
1721	SmallVector<Stmt::Likelihood, 16> *SwitchLikelihood = nullptr;
1722
1723	/// CaseRangeBlock - This block holds if condition check for last case
1724	/// statement range in current switch instruction.
1725	llvm::BasicBlock *CaseRangeBlock = nullptr;
1726
1727	/// OpaqueLValues - Keeps track of the current set of opaque value
1728	/// expressions.
1729	llvm::DenseMap<const OpaqueValueExpr *, LValue> OpaqueLValues;
1730	llvm::DenseMap<const OpaqueValueExpr *, RValue> OpaqueRValues;
1731
1732	// VLASizeMap - This keeps track of the associated size for each VLA type.
1733	// We track this by the size expression rather than the type itself because
1734	// in certain situations, like a const qualifier applied to an VLA typedef,
1735	// multiple VLA types can share the same size expression.
1736	// FIXME: Maybe this could be a stack of maps that is pushed/popped as we
1737	// enter/leave scopes.
1738	llvm::DenseMap<const Expr *, llvm::Value *> VLASizeMap;
1739
1740	/// A block containing a single 'unreachable' instruction. Created
1741	/// lazily by getUnreachableBlock().
1742	llvm::BasicBlock *UnreachableBlock = nullptr;
1743
1744	/// Counts of the number return expressions in the function.
1745	unsigned NumReturnExprs = 0;
1746
1747	/// Count the number of simple (constant) return expressions in the function.
1748	unsigned NumSimpleReturnExprs = 0;
1749
1750	/// The last regular (non-return) debug location (breakpoint) in the function.
1751	SourceLocation LastStopPoint;
1752
1753	public:
1754	/// Source location information about the default argument or member
1755	/// initializer expression we're evaluating, if any.
1756	CurrentSourceLocExprScope CurSourceLocExprScope;
1757	using SourceLocExprScopeGuard =
1758	CurrentSourceLocExprScope::SourceLocExprScopeGuard;
1759
1760	/// A scope within which we are constructing the fields of an object which
1761	/// might use a CXXDefaultInitExpr. This stashes away a 'this' value to use
1762	/// if we need to evaluate a CXXDefaultInitExpr within the evaluation.
1763	class FieldConstructionScope {
1764	public:
1765	FieldConstructionScope(CodeGenFunction &CGF, Address This)
1766	: CGF(CGF), OldCXXDefaultInitExprThis(CGF.CXXDefaultInitExprThis) {
1767	CGF.CXXDefaultInitExprThis = This;
1768	}
1769	~FieldConstructionScope() {
1770	CGF.CXXDefaultInitExprThis = OldCXXDefaultInitExprThis;
1771	}
1772
1773	private:
1774	CodeGenFunction &CGF;
1775	Address OldCXXDefaultInitExprThis;
1776	};
1777
1778	/// The scope of a CXXDefaultInitExpr. Within this scope, the value of 'this'
1779	/// is overridden to be the object under construction.
1780	class CXXDefaultInitExprScope {
1781	public:
1782	CXXDefaultInitExprScope(CodeGenFunction &CGF, const CXXDefaultInitExpr *E)
1783	: CGF(CGF), OldCXXThisValue(CGF.CXXThisValue),
1784	OldCXXThisAlignment(CGF.CXXThisAlignment),
1785	SourceLocScope(E, CGF.CurSourceLocExprScope) {
1786	CGF.CXXThisValue = CGF.CXXDefaultInitExprThis.getBasePointer();
1787	CGF.CXXThisAlignment = CGF.CXXDefaultInitExprThis.getAlignment();
1788	}
1789	~CXXDefaultInitExprScope() {
1790	CGF.CXXThisValue = OldCXXThisValue;
1791	CGF.CXXThisAlignment = OldCXXThisAlignment;
1792	}
1793
1794	public:
1795	CodeGenFunction &CGF;
1796	llvm::Value *OldCXXThisValue;
1797	CharUnits OldCXXThisAlignment;
1798	SourceLocExprScopeGuard SourceLocScope;
1799	};
1800
1801	struct CXXDefaultArgExprScope : SourceLocExprScopeGuard {
1802	CXXDefaultArgExprScope(CodeGenFunction &CGF, const CXXDefaultArgExpr *E)
1803	: SourceLocExprScopeGuard(E, CGF.CurSourceLocExprScope) {}
1804	};
1805
1806	/// The scope of an ArrayInitLoopExpr. Within this scope, the value of the
1807	/// current loop index is overridden.
1808	class ArrayInitLoopExprScope {
1809	public:
1810	ArrayInitLoopExprScope(CodeGenFunction &CGF, llvm::Value *Index)
1811	: CGF(CGF), OldArrayInitIndex(CGF.ArrayInitIndex) {
1812	CGF.ArrayInitIndex = Index;
1813	}
1814	~ArrayInitLoopExprScope() { CGF.ArrayInitIndex = OldArrayInitIndex; }
1815
1816	private:
1817	CodeGenFunction &CGF;
1818	llvm::Value *OldArrayInitIndex;
1819	};
1820
1821	class InlinedInheritingConstructorScope {
1822	public:
1823	InlinedInheritingConstructorScope(CodeGenFunction &CGF, GlobalDecl GD)
1824	: CGF(CGF), OldCurGD(CGF.CurGD), OldCurFuncDecl(CGF.CurFuncDecl),
1825	OldCurCodeDecl(CGF.CurCodeDecl),
1826	OldCXXABIThisDecl(CGF.CXXABIThisDecl),
1827	OldCXXABIThisValue(CGF.CXXABIThisValue),
1828	OldCXXThisValue(CGF.CXXThisValue),
1829	OldCXXABIThisAlignment(CGF.CXXABIThisAlignment),
1830	OldCXXThisAlignment(CGF.CXXThisAlignment),
1831	OldReturnValue(CGF.ReturnValue), OldFnRetTy(CGF.FnRetTy),
1832	OldCXXInheritedCtorInitExprArgs(
1833	std::move(CGF.CXXInheritedCtorInitExprArgs)) {
1834	CGF.CurGD = GD;
1835	CGF.CurFuncDecl = CGF.CurCodeDecl =
1836	cast<CXXConstructorDecl>(Val: GD.getDecl());
1837	CGF.CXXABIThisDecl = nullptr;
1838	CGF.CXXABIThisValue = nullptr;
1839	CGF.CXXThisValue = nullptr;
1840	CGF.CXXABIThisAlignment = CharUnits();
1841	CGF.CXXThisAlignment = CharUnits();
1842	CGF.ReturnValue = Address::invalid();
1843	CGF.FnRetTy = QualType();
1844	CGF.CXXInheritedCtorInitExprArgs.clear();
1845	}
1846	~InlinedInheritingConstructorScope() {
1847	CGF.CurGD = OldCurGD;
1848	CGF.CurFuncDecl = OldCurFuncDecl;
1849	CGF.CurCodeDecl = OldCurCodeDecl;
1850	CGF.CXXABIThisDecl = OldCXXABIThisDecl;
1851	CGF.CXXABIThisValue = OldCXXABIThisValue;
1852	CGF.CXXThisValue = OldCXXThisValue;
1853	CGF.CXXABIThisAlignment = OldCXXABIThisAlignment;
1854	CGF.CXXThisAlignment = OldCXXThisAlignment;
1855	CGF.ReturnValue = OldReturnValue;
1856	CGF.FnRetTy = OldFnRetTy;
1857	CGF.CXXInheritedCtorInitExprArgs =
1858	std::move(OldCXXInheritedCtorInitExprArgs);
1859	}
1860
1861	private:
1862	CodeGenFunction &CGF;
1863	GlobalDecl OldCurGD;
1864	const Decl *OldCurFuncDecl;
1865	const Decl *OldCurCodeDecl;
1866	ImplicitParamDecl *OldCXXABIThisDecl;
1867	llvm::Value *OldCXXABIThisValue;
1868	llvm::Value *OldCXXThisValue;
1869	CharUnits OldCXXABIThisAlignment;
1870	CharUnits OldCXXThisAlignment;
1871	Address OldReturnValue;
1872	QualType OldFnRetTy;
1873	CallArgList OldCXXInheritedCtorInitExprArgs;
1874	};
1875
1876	// Helper class for the OpenMP IR Builder. Allows reusability of code used for
1877	// region body, and finalization codegen callbacks. This will class will also
1878	// contain privatization functions used by the privatization call backs
1879	//
1880	// TODO: this is temporary class for things that are being moved out of
1881	// CGOpenMPRuntime, new versions of current CodeGenFunction methods, or
1882	// utility function for use with the OMPBuilder. Once that move to use the
1883	// OMPBuilder is done, everything here will either become part of CodeGenFunc.
1884	// directly, or a new helper class that will contain functions used by both
1885	// this and the OMPBuilder
1886
1887	struct OMPBuilderCBHelpers {
1888
1889	OMPBuilderCBHelpers() = delete;
1890	OMPBuilderCBHelpers(const OMPBuilderCBHelpers &) = delete;
1891	OMPBuilderCBHelpers &operator=(const OMPBuilderCBHelpers &) = delete;
1892
1893	using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
1894
1895	/// Cleanup action for allocate support.
1896	class OMPAllocateCleanupTy final : public EHScopeStack::Cleanup {
1897
1898	private:
1899	llvm::CallInst *RTLFnCI;
1900
1901	public:
1902	OMPAllocateCleanupTy(llvm::CallInst *RLFnCI) : RTLFnCI(RLFnCI) {
1903	RLFnCI->removeFromParent();
1904	}
1905
1906	void Emit(CodeGenFunction &CGF, Flags /*flags*/) override {
1907	if (!CGF.HaveInsertPoint())
1908	return;
1909	CGF.Builder.Insert(I: RTLFnCI);
1910	}
1911	};
1912
1913	/// Returns address of the threadprivate variable for the current
1914	/// thread. This Also create any necessary OMP runtime calls.
1915	///
1916	/// \param VD VarDecl for Threadprivate variable.
1917	/// \param VDAddr Address of the Vardecl
1918	/// \param Loc The location where the barrier directive was encountered
1919	static Address getAddrOfThreadPrivate(CodeGenFunction &CGF,
1920	const VarDecl *VD, Address VDAddr,
1921	SourceLocation Loc);
1922
1923	/// Gets the OpenMP-specific address of the local variable /p VD.
1924	static Address getAddressOfLocalVariable(CodeGenFunction &CGF,
1925	const VarDecl *VD);
1926	/// Get the platform-specific name separator.
1927	/// \param Parts different parts of the final name that needs separation
1928	/// \param FirstSeparator First separator used between the initial two
1929	/// parts of the name.
1930	/// \param Separator separator used between all of the rest consecutinve
1931	/// parts of the name
1932	static std::string getNameWithSeparators(ArrayRef<StringRef> Parts,
1933	StringRef FirstSeparator = ".",
1934	StringRef Separator = ".");
1935	/// Emit the Finalization for an OMP region
1936	/// \param CGF The Codegen function this belongs to
1937	/// \param IP Insertion point for generating the finalization code.
1938	static void FinalizeOMPRegion(CodeGenFunction &CGF, InsertPointTy IP) {
1939	CGBuilderTy::InsertPointGuard IPG(CGF.Builder);
1940	assert(IP.getBlock()->end() != IP.getPoint() &&
1941	"OpenMP IR Builder should cause terminated block!");
1942
1943	llvm::BasicBlock *IPBB = IP.getBlock();
1944	llvm::BasicBlock *DestBB = IPBB->getUniqueSuccessor();
1945	assert(DestBB && "Finalization block should have one successor!");
1946
1947	// erase and replace with cleanup branch.
1948	IPBB->getTerminator()->eraseFromParent();
1949	CGF.Builder.SetInsertPoint(IPBB);
1950	CodeGenFunction::JumpDest Dest = CGF.getJumpDestInCurrentScope(Target: DestBB);
1951	CGF.EmitBranchThroughCleanup(Dest);
1952	}
1953
1954	/// Emit the body of an OMP region
1955	/// \param CGF The Codegen function this belongs to
1956	/// \param RegionBodyStmt The body statement for the OpenMP region being
1957	/// generated
1958	/// \param AllocaIP Where to insert alloca instructions
1959	/// \param CodeGenIP Where to insert the region code
1960	/// \param RegionName Name to be used for new blocks
1961	static void EmitOMPInlinedRegionBody(CodeGenFunction &CGF,
1962	const Stmt *RegionBodyStmt,
1963	InsertPointTy AllocaIP,
1964	InsertPointTy CodeGenIP,
1965	Twine RegionName);
1966
1967	static void EmitCaptureStmt(CodeGenFunction &CGF, InsertPointTy CodeGenIP,
1968	llvm::BasicBlock &FiniBB, llvm::Function *Fn,
1969	ArrayRef<llvm::Value *> Args) {
1970	llvm::BasicBlock *CodeGenIPBB = CodeGenIP.getBlock();
1971	if (llvm::Instruction *CodeGenIPBBTI = CodeGenIPBB->getTerminator())
1972	CodeGenIPBBTI->eraseFromParent();
1973
1974	CGF.Builder.SetInsertPoint(CodeGenIPBB);
1975
1976	if (Fn->doesNotThrow())
1977	CGF.EmitNounwindRuntimeCall(callee: Fn, args: Args);
1978	else
1979	CGF.EmitRuntimeCall(callee: Fn, args: Args);
1980
1981	if (CGF.Builder.saveIP().isSet())
1982	CGF.Builder.CreateBr(Dest: &FiniBB);
1983	}
1984
1985	/// Emit the body of an OMP region that will be outlined in
1986	/// OpenMPIRBuilder::finalize().
1987	/// \param CGF The Codegen function this belongs to
1988	/// \param RegionBodyStmt The body statement for the OpenMP region being
1989	/// generated
1990	/// \param AllocaIP Where to insert alloca instructions
1991	/// \param CodeGenIP Where to insert the region code
1992	/// \param RegionName Name to be used for new blocks
1993	static void EmitOMPOutlinedRegionBody(CodeGenFunction &CGF,
1994	const Stmt *RegionBodyStmt,
1995	InsertPointTy AllocaIP,
1996	InsertPointTy CodeGenIP,
1997	Twine RegionName);
1998
1999	/// RAII for preserving necessary info during Outlined region body codegen.
2000	class OutlinedRegionBodyRAII {
2001
2002	llvm::AssertingVH<llvm::Instruction> OldAllocaIP;
2003	CodeGenFunction::JumpDest OldReturnBlock;
2004	CodeGenFunction &CGF;
2005
2006	public:
2007	OutlinedRegionBodyRAII(CodeGenFunction &cgf, InsertPointTy &AllocaIP,
2008	llvm::BasicBlock &RetBB)
2009	: CGF(cgf) {
2010	assert(AllocaIP.isSet() &&
2011	"Must specify Insertion point for allocas of outlined function");
2012	OldAllocaIP = CGF.AllocaInsertPt;
2013	CGF.AllocaInsertPt = &*AllocaIP.getPoint();
2014
2015	OldReturnBlock = CGF.ReturnBlock;
2016	CGF.ReturnBlock = CGF.getJumpDestInCurrentScope(Target: &RetBB);
2017	}
2018
2019	~OutlinedRegionBodyRAII() {
2020	CGF.AllocaInsertPt = OldAllocaIP;
2021	CGF.ReturnBlock = OldReturnBlock;
2022	}
2023	};
2024
2025	/// RAII for preserving necessary info during inlined region body codegen.
2026	class InlinedRegionBodyRAII {
2027
2028	llvm::AssertingVH<llvm::Instruction> OldAllocaIP;
2029	CodeGenFunction &CGF;
2030
2031	public:
2032	InlinedRegionBodyRAII(CodeGenFunction &cgf, InsertPointTy &AllocaIP,
2033	llvm::BasicBlock &FiniBB)
2034	: CGF(cgf) {
2035	// Alloca insertion block should be in the entry block of the containing
2036	// function so it expects an empty AllocaIP in which case will reuse the
2037	// old alloca insertion point, or a new AllocaIP in the same block as
2038	// the old one
2039	assert((!AllocaIP.isSet() ||
2040	CGF.AllocaInsertPt->getParent() == AllocaIP.getBlock()) &&
2041	"Insertion point should be in the entry block of containing "
2042	"function!");
2043	OldAllocaIP = CGF.AllocaInsertPt;
2044	if (AllocaIP.isSet())
2045	CGF.AllocaInsertPt = &*AllocaIP.getPoint();
2046
2047	// TODO: Remove the call, after making sure the counter is not used by
2048	// the EHStack.
2049	// Since this is an inlined region, it should not modify the
2050	// ReturnBlock, and should reuse the one for the enclosing outlined
2051	// region. So, the JumpDest being return by the function is discarded
2052	(void)CGF.getJumpDestInCurrentScope(Target: &FiniBB);
2053	}
2054
2055	~InlinedRegionBodyRAII() { CGF.AllocaInsertPt = OldAllocaIP; }
2056	};
2057	};
2058
2059	private:
2060	/// CXXThisDecl - When generating code for a C++ member function,
2061	/// this will hold the implicit 'this' declaration.
2062	ImplicitParamDecl *CXXABIThisDecl = nullptr;
2063	llvm::Value *CXXABIThisValue = nullptr;
2064	llvm::Value *CXXThisValue = nullptr;
2065	CharUnits CXXABIThisAlignment;
2066	CharUnits CXXThisAlignment;
2067
2068	/// The value of 'this' to use when evaluating CXXDefaultInitExprs within
2069	/// this expression.
2070	Address CXXDefaultInitExprThis = Address::invalid();
2071
2072	/// The current array initialization index when evaluating an
2073	/// ArrayInitIndexExpr within an ArrayInitLoopExpr.
2074	llvm::Value *ArrayInitIndex = nullptr;
2075
2076	/// The values of function arguments to use when evaluating
2077	/// CXXInheritedCtorInitExprs within this context.
2078	CallArgList CXXInheritedCtorInitExprArgs;
2079
2080	/// CXXStructorImplicitParamDecl - When generating code for a constructor or
2081	/// destructor, this will hold the implicit argument (e.g. VTT).
2082	ImplicitParamDecl *CXXStructorImplicitParamDecl = nullptr;
2083	llvm::Value *CXXStructorImplicitParamValue = nullptr;
2084
2085	/// OutermostConditional - Points to the outermost active
2086	/// conditional control. This is used so that we know if a
2087	/// temporary should be destroyed conditionally.
2088	ConditionalEvaluation *OutermostConditional = nullptr;
2089
2090	/// The current lexical scope.
2091	LexicalScope *CurLexicalScope = nullptr;
2092
2093	/// The current source location that should be used for exception
2094	/// handling code.
2095	SourceLocation CurEHLocation;
2096
2097	/// BlockByrefInfos - For each __block variable, contains
2098	/// information about the layout of the variable.
2099	llvm::DenseMap<const ValueDecl *, BlockByrefInfo> BlockByrefInfos;
2100
2101	/// Used by -fsanitize=nullability-return to determine whether the return
2102	/// value can be checked.
2103	llvm::Value *RetValNullabilityPrecondition = nullptr;
2104
2105	/// Check if -fsanitize=nullability-return instrumentation is required for
2106	/// this function.
2107	bool requiresReturnValueNullabilityCheck() const {
2108	return RetValNullabilityPrecondition;
2109	}
2110
2111	/// Used to store precise source locations for return statements by the
2112	/// runtime return value checks.
2113	Address ReturnLocation = Address::invalid();
2114
2115	/// Check if the return value of this function requires sanitization.
2116	bool requiresReturnValueCheck() const;
2117
2118	bool isInAllocaArgument(CGCXXABI &ABI, QualType Ty);
2119	bool hasInAllocaArg(const CXXMethodDecl *MD);
2120
2121	llvm::BasicBlock *TerminateLandingPad = nullptr;
2122	llvm::BasicBlock *TerminateHandler = nullptr;
2123	llvm::SmallVector<llvm::BasicBlock *, 2> TrapBBs;
2124
2125	/// Terminate funclets keyed by parent funclet pad.
2126	llvm::MapVector<llvm::Value *, llvm::BasicBlock *> TerminateFunclets;
2127
2128	/// Largest vector width used in ths function. Will be used to create a
2129	/// function attribute.
2130	unsigned LargestVectorWidth = 0;
2131
2132	/// True if we need emit the life-time markers. This is initially set in
2133	/// the constructor, but could be overwritten to true if this is a coroutine.
2134	bool ShouldEmitLifetimeMarkers;
2135
2136	/// Add OpenCL kernel arg metadata and the kernel attribute metadata to
2137	/// the function metadata.
2138	void EmitKernelMetadata(const FunctionDecl *FD, llvm::Function *Fn);
2139
2140	public:
2141	CodeGenFunction(CodeGenModule &cgm, bool suppressNewContext = false);
2142	~CodeGenFunction();
2143
2144	CodeGenTypes &getTypes() const { return CGM.getTypes(); }
2145	ASTContext &getContext() const { return CGM.getContext(); }
2146	CGDebugInfo *getDebugInfo() {
2147	if (DisableDebugInfo)
2148	return nullptr;
2149	return DebugInfo;
2150	}
2151	void disableDebugInfo() { DisableDebugInfo = true; }
2152	void enableDebugInfo() { DisableDebugInfo = false; }
2153
2154	bool shouldUseFusedARCCalls() {
2155	return CGM.getCodeGenOpts().OptimizationLevel == 0;
2156	}
2157
2158	const LangOptions &getLangOpts() const { return CGM.getLangOpts(); }
2159
2160	/// Returns a pointer to the function's exception object and selector slot,
2161	/// which is assigned in every landing pad.
2162	Address getExceptionSlot();
2163	Address getEHSelectorSlot();
2164
2165	/// Returns the contents of the function's exception object and selector
2166	/// slots.
2167	llvm::Value *getExceptionFromSlot();
2168	llvm::Value *getSelectorFromSlot();
2169
2170	RawAddress getNormalCleanupDestSlot();
2171
2172	llvm::BasicBlock *getUnreachableBlock() {
2173	if (!UnreachableBlock) {
2174	UnreachableBlock = createBasicBlock(name: "unreachable");
2175	new llvm::UnreachableInst(getLLVMContext(), UnreachableBlock);
2176	}
2177	return UnreachableBlock;
2178	}
2179
2180	llvm::BasicBlock *getInvokeDest() {
2181	if (!EHStack.requiresLandingPad())
2182	return nullptr;
2183	return getInvokeDestImpl();
2184	}
2185
2186	bool currentFunctionUsesSEHTry() const { return !!CurSEHParent; }
2187
2188	const TargetInfo &getTarget() const { return Target; }
2189	llvm::LLVMContext &getLLVMContext() { return CGM.getLLVMContext(); }
2190	const TargetCodeGenInfo &getTargetHooks() const {
2191	return CGM.getTargetCodeGenInfo();
2192	}
2193
2194	//===--------------------------------------------------------------------===//
2195	// Cleanups
2196	//===--------------------------------------------------------------------===//
2197
2198	typedef void Destroyer(CodeGenFunction &CGF, Address addr, QualType ty);
2199
2200	void pushIrregularPartialArrayCleanup(llvm::Value *arrayBegin,
2201	Address arrayEndPointer,
2202	QualType elementType,
2203	CharUnits elementAlignment,
2204	Destroyer *destroyer);
2205	void pushRegularPartialArrayCleanup(llvm::Value *arrayBegin,
2206	llvm::Value *arrayEnd,
2207	QualType elementType,
2208	CharUnits elementAlignment,
2209	Destroyer *destroyer);
2210
2211	void pushDestroy(QualType::DestructionKind dtorKind, Address addr,
2212	QualType type);
2213	void pushEHDestroy(QualType::DestructionKind dtorKind, Address addr,
2214	QualType type);
2215	void pushDestroy(CleanupKind kind, Address addr, QualType type,
2216	Destroyer *destroyer, bool useEHCleanupForArray);
2217	void pushDestroyAndDeferDeactivation(QualType::DestructionKind dtorKind,
2218	Address addr, QualType type);
2219	void pushDestroyAndDeferDeactivation(CleanupKind cleanupKind, Address addr,
2220	QualType type, Destroyer *destroyer,
2221	bool useEHCleanupForArray);
2222	void pushLifetimeExtendedDestroy(CleanupKind kind, Address addr,
2223	QualType type, Destroyer *destroyer,
2224	bool useEHCleanupForArray);
2225	void pushLifetimeExtendedDestroy(QualType::DestructionKind dtorKind,
2226	Address addr, QualType type);
2227	void pushCallObjectDeleteCleanup(const FunctionDecl *OperatorDelete,
2228	llvm::Value *CompletePtr,
2229	QualType ElementType);
2230	void pushStackRestore(CleanupKind kind, Address SPMem);
2231	void pushKmpcAllocFree(CleanupKind Kind,
2232	std::pair<llvm::Value *, llvm::Value *> AddrSizePair);
2233	void emitDestroy(Address addr, QualType type, Destroyer *destroyer,
2234	bool useEHCleanupForArray);
2235	llvm::Function *generateDestroyHelper(Address addr, QualType type,
2236	Destroyer *destroyer,
2237	bool useEHCleanupForArray,
2238	const VarDecl *VD);
2239	void emitArrayDestroy(llvm::Value *begin, llvm::Value *end,
2240	QualType elementType, CharUnits elementAlign,
2241	Destroyer *destroyer, bool checkZeroLength,
2242	bool useEHCleanup);
2243
2244	Destroyer *getDestroyer(QualType::DestructionKind destructionKind);
2245
2246	/// Determines whether an EH cleanup is required to destroy a type
2247	/// with the given destruction kind.
2248	bool needsEHCleanup(QualType::DestructionKind kind) {
2249	switch (kind) {
2250	case QualType::DK_none:
2251	return false;
2252	case QualType::DK_cxx_destructor:
2253	case QualType::DK_objc_weak_lifetime:
2254	case QualType::DK_nontrivial_c_struct:
2255	return getLangOpts().Exceptions;
2256	case QualType::DK_objc_strong_lifetime:
2257	return getLangOpts().Exceptions &&
2258	CGM.getCodeGenOpts().ObjCAutoRefCountExceptions;
2259	}
2260	llvm_unreachable("bad destruction kind");
2261	}
2262
2263	CleanupKind getCleanupKind(QualType::DestructionKind kind) {
2264	return (needsEHCleanup(kind) ? NormalAndEHCleanup : NormalCleanup);
2265	}
2266
2267	//===--------------------------------------------------------------------===//
2268	// Objective-C
2269	//===--------------------------------------------------------------------===//
2270
2271	void GenerateObjCMethod(const ObjCMethodDecl *OMD);
2272
2273	void StartObjCMethod(const ObjCMethodDecl *MD, const ObjCContainerDecl *CD);
2274
2275	/// GenerateObjCGetter - Synthesize an Objective-C property getter function.
2276	void GenerateObjCGetter(ObjCImplementationDecl *IMP,
2277	const ObjCPropertyImplDecl *PID);
2278	void generateObjCGetterBody(const ObjCImplementationDecl *classImpl,
2279	const ObjCPropertyImplDecl *propImpl,
2280	const ObjCMethodDecl *GetterMothodDecl,
2281	llvm::Constant *AtomicHelperFn);
2282
2283	void GenerateObjCCtorDtorMethod(ObjCImplementationDecl *IMP,
2284	ObjCMethodDecl *MD, bool ctor);
2285
2286	/// GenerateObjCSetter - Synthesize an Objective-C property setter function
2287	/// for the given property.
2288	void GenerateObjCSetter(ObjCImplementationDecl *IMP,
2289	const ObjCPropertyImplDecl *PID);
2290	void generateObjCSetterBody(const ObjCImplementationDecl *classImpl,
2291	const ObjCPropertyImplDecl *propImpl,
2292	llvm::Constant *AtomicHelperFn);
2293
2294	//===--------------------------------------------------------------------===//
2295	// Block Bits
2296	//===--------------------------------------------------------------------===//
2297
2298	/// Emit block literal.
2299	/// \return an LLVM value which is a pointer to a struct which contains
2300	/// information about the block, including the block invoke function, the
2301	/// captured variables, etc.
2302	llvm::Value *EmitBlockLiteral(const BlockExpr *);
2303
2304	llvm::Function *GenerateBlockFunction(GlobalDecl GD, const CGBlockInfo &Info,
2305	const DeclMapTy &ldm,
2306	bool IsLambdaConversionToBlock,
2307	bool BuildGlobalBlock);
2308
2309	/// Check if \p T is a C++ class that has a destructor that can throw.
2310	static bool cxxDestructorCanThrow(QualType T);
2311
2312	llvm::Constant *GenerateCopyHelperFunction(const CGBlockInfo &blockInfo);
2313	llvm::Constant *GenerateDestroyHelperFunction(const CGBlockInfo &blockInfo);
2314	llvm::Constant *
2315	GenerateObjCAtomicSetterCopyHelperFunction(const ObjCPropertyImplDecl *PID);
2316	llvm::Constant *
2317	GenerateObjCAtomicGetterCopyHelperFunction(const ObjCPropertyImplDecl *PID);
2318	llvm::Value *EmitBlockCopyAndAutorelease(llvm::Value *Block, QualType Ty);
2319
2320	void BuildBlockRelease(llvm::Value *DeclPtr, BlockFieldFlags flags,
2321	bool CanThrow);
2322
2323	class AutoVarEmission;
2324
2325	void emitByrefStructureInit(const AutoVarEmission &emission);
2326
2327	/// Enter a cleanup to destroy a __block variable. Note that this
2328	/// cleanup should be a no-op if the variable hasn't left the stack
2329	/// yet; if a cleanup is required for the variable itself, that needs
2330	/// to be done externally.
2331	///
2332	/// \param Kind Cleanup kind.
2333	///
2334	/// \param Addr When \p LoadBlockVarAddr is false, the address of the __block
2335	/// structure that will be passed to _Block_object_dispose. When
2336	/// \p LoadBlockVarAddr is true, the address of the field of the block
2337	/// structure that holds the address of the __block structure.
2338	///
2339	/// \param Flags The flag that will be passed to _Block_object_dispose.
2340	///
2341	/// \param LoadBlockVarAddr Indicates whether we need to emit a load from
2342	/// \p Addr to get the address of the __block structure.
2343	void enterByrefCleanup(CleanupKind Kind, Address Addr, BlockFieldFlags Flags,
2344	bool LoadBlockVarAddr, bool CanThrow);
2345
2346	void setBlockContextParameter(const ImplicitParamDecl *D, unsigned argNum,
2347	llvm::Value *ptr);
2348
2349	Address LoadBlockStruct();
2350	Address GetAddrOfBlockDecl(const VarDecl *var);
2351
2352	/// BuildBlockByrefAddress - Computes the location of the
2353	/// data in a variable which is declared as __block.
2354	Address emitBlockByrefAddress(Address baseAddr, const VarDecl *V,
2355	bool followForward = true);
2356	Address emitBlockByrefAddress(Address baseAddr, const BlockByrefInfo &info,
2357	bool followForward, const llvm::Twine &name);
2358
2359	const BlockByrefInfo &getBlockByrefInfo(const VarDecl *var);
2360
2361	QualType BuildFunctionArgList(GlobalDecl GD, FunctionArgList &Args);
2362
2363	void GenerateCode(GlobalDecl GD, llvm::Function *Fn,
2364	const CGFunctionInfo &FnInfo);
2365
2366	/// Annotate the function with an attribute that disables TSan checking at
2367	/// runtime.
2368	void markAsIgnoreThreadCheckingAtRuntime(llvm::Function *Fn);
2369
2370	/// Emit code for the start of a function.
2371	/// \param Loc The location to be associated with the function.
2372	/// \param StartLoc The location of the function body.
2373	void StartFunction(GlobalDecl GD, QualType RetTy, llvm::Function *Fn,
2374	const CGFunctionInfo &FnInfo, const FunctionArgList &Args,
2375	SourceLocation Loc = SourceLocation(),
2376	SourceLocation StartLoc = SourceLocation());
2377
2378	static bool IsConstructorDelegationValid(const CXXConstructorDecl *Ctor);
2379
2380	void EmitConstructorBody(FunctionArgList &Args);
2381	void EmitDestructorBody(FunctionArgList &Args);
2382	void emitImplicitAssignmentOperatorBody(FunctionArgList &Args);
2383	void EmitFunctionBody(const Stmt *Body);
2384	void EmitBlockWithFallThrough(llvm::BasicBlock *BB, const Stmt *S);
2385
2386	void EmitForwardingCallToLambda(const CXXMethodDecl *LambdaCallOperator,
2387	CallArgList &CallArgs,
2388	const CGFunctionInfo *CallOpFnInfo = nullptr,
2389	llvm::Constant *CallOpFn = nullptr);
2390	void EmitLambdaBlockInvokeBody();
2391	void EmitLambdaStaticInvokeBody(const CXXMethodDecl *MD);
2392	void EmitLambdaDelegatingInvokeBody(const CXXMethodDecl *MD,
2393	CallArgList &CallArgs);
2394	void EmitLambdaInAllocaImplFn(const CXXMethodDecl *CallOp,
2395	const CGFunctionInfo **ImplFnInfo,
2396	llvm::Function **ImplFn);
2397	void EmitLambdaInAllocaCallOpBody(const CXXMethodDecl *MD);
2398	void EmitLambdaVLACapture(const VariableArrayType *VAT, LValue LV) {
2399	EmitStoreThroughLValue(Src: RValue::get(V: VLASizeMap[VAT->getSizeExpr()]), Dst: LV);
2400	}
2401	void EmitAsanPrologueOrEpilogue(bool Prologue);
2402
2403	/// Emit the unified return block, trying to avoid its emission when
2404	/// possible.
2405	/// \return The debug location of the user written return statement if the
2406	/// return block is avoided.
2407	llvm::DebugLoc EmitReturnBlock();
2408
2409	/// FinishFunction - Complete IR generation of the current function. It is
2410	/// legal to call this function even if there is no current insertion point.
2411	void FinishFunction(SourceLocation EndLoc = SourceLocation());
2412
2413	void StartThunk(llvm::Function *Fn, GlobalDecl GD,
2414	const CGFunctionInfo &FnInfo, bool IsUnprototyped);
2415
2416	void EmitCallAndReturnForThunk(llvm::FunctionCallee Callee,
2417	const ThunkInfo *Thunk, bool IsUnprototyped);
2418
2419	void FinishThunk();
2420
2421	/// Emit a musttail call for a thunk with a potentially adjusted this pointer.
2422	void EmitMustTailThunk(GlobalDecl GD, llvm::Value *AdjustedThisPtr,
2423	llvm::FunctionCallee Callee);
2424
2425	/// Generate a thunk for the given method.
2426	void generateThunk(llvm::Function *Fn, const CGFunctionInfo &FnInfo,
2427	GlobalDecl GD, const ThunkInfo &Thunk,
2428	bool IsUnprototyped);
2429
2430	llvm::Function *GenerateVarArgsThunk(llvm::Function *Fn,
2431	const CGFunctionInfo &FnInfo,
2432	GlobalDecl GD, const ThunkInfo &Thunk);
2433
2434	void EmitCtorPrologue(const CXXConstructorDecl *CD, CXXCtorType Type,
2435	FunctionArgList &Args);
2436
2437	void EmitInitializerForField(FieldDecl *Field, LValue LHS, Expr *Init);
2438
2439	/// Struct with all information about dynamic [sub]class needed to set vptr.
2440	struct VPtr {
2441	BaseSubobject Base;
2442	const CXXRecordDecl *NearestVBase;
2443	CharUnits OffsetFromNearestVBase;
2444	const CXXRecordDecl *VTableClass;
2445	};
2446
2447	/// Initialize the vtable pointer of the given subobject.
2448	void InitializeVTablePointer(const VPtr &vptr);
2449
2450	typedef llvm::SmallVector<VPtr, 4> VPtrsVector;
2451
2452	typedef llvm::SmallPtrSet<const CXXRecordDecl *, 4> VisitedVirtualBasesSetTy;
2453	VPtrsVector getVTablePointers(const CXXRecordDecl *VTableClass);
2454
2455	void getVTablePointers(BaseSubobject Base, const CXXRecordDecl *NearestVBase,
2456	CharUnits OffsetFromNearestVBase,
2457	bool BaseIsNonVirtualPrimaryBase,
2458	const CXXRecordDecl *VTableClass,
2459	VisitedVirtualBasesSetTy &VBases, VPtrsVector &vptrs);
2460
2461	void InitializeVTablePointers(const CXXRecordDecl *ClassDecl);
2462
2463	// VTableTrapMode - whether we guarantee that loading the
2464	// vtable is guaranteed to trap on authentication failure,
2465	// even if the resulting vtable pointer is unused.
2466	enum class VTableAuthMode {
2467	Authenticate,
2468	MustTrap,
2469	UnsafeUbsanStrip // Should only be used for Vptr UBSan check
2470	};
2471	/// GetVTablePtr - Return the Value of the vtable pointer member pointed
2472	/// to by This.
2473	llvm::Value *
2474	GetVTablePtr(Address This, llvm::Type *VTableTy,
2475	const CXXRecordDecl *VTableClass,
2476	VTableAuthMode AuthMode = VTableAuthMode::Authenticate);
2477
2478	enum CFITypeCheckKind {
2479	CFITCK_VCall,
2480	CFITCK_NVCall,
2481	CFITCK_DerivedCast,
2482	CFITCK_UnrelatedCast,
2483	CFITCK_ICall,
2484	CFITCK_NVMFCall,
2485	CFITCK_VMFCall,
2486	};
2487
2488	/// Derived is the presumed address of an object of type T after a
2489	/// cast. If T is a polymorphic class type, emit a check that the virtual
2490	/// table for Derived belongs to a class derived from T.
2491	void EmitVTablePtrCheckForCast(QualType T, Address Derived, bool MayBeNull,
2492	CFITypeCheckKind TCK, SourceLocation Loc);
2493
2494	/// EmitVTablePtrCheckForCall - Virtual method MD is being called via VTable.
2495	/// If vptr CFI is enabled, emit a check that VTable is valid.
2496	void EmitVTablePtrCheckForCall(const CXXRecordDecl *RD, llvm::Value *VTable,
2497	CFITypeCheckKind TCK, SourceLocation Loc);
2498
2499	/// EmitVTablePtrCheck - Emit a check that VTable is a valid virtual table for
2500	/// RD using llvm.type.test.
2501	void EmitVTablePtrCheck(const CXXRecordDecl *RD, llvm::Value *VTable,
2502	CFITypeCheckKind TCK, SourceLocation Loc);
2503
2504	/// If whole-program virtual table optimization is enabled, emit an assumption
2505	/// that VTable is a member of RD's type identifier. Or, if vptr CFI is
2506	/// enabled, emit a check that VTable is a member of RD's type identifier.
2507	void EmitTypeMetadataCodeForVCall(const CXXRecordDecl *RD,
2508	llvm::Value *VTable, SourceLocation Loc);
2509
2510	/// Returns whether we should perform a type checked load when loading a
2511	/// virtual function for virtual calls to members of RD. This is generally
2512	/// true when both vcall CFI and whole-program-vtables are enabled.
2513	bool ShouldEmitVTableTypeCheckedLoad(const CXXRecordDecl *RD);
2514
2515	/// Emit a type checked load from the given vtable.
2516	llvm::Value *EmitVTableTypeCheckedLoad(const CXXRecordDecl *RD,
2517	llvm::Value *VTable,
2518	llvm::Type *VTableTy,
2519	uint64_t VTableByteOffset);
2520
2521	/// EnterDtorCleanups - Enter the cleanups necessary to complete the
2522	/// given phase of destruction for a destructor. The end result
2523	/// should call destructors on members and base classes in reverse
2524	/// order of their construction.
2525	void EnterDtorCleanups(const CXXDestructorDecl *Dtor, CXXDtorType Type);
2526
2527	/// ShouldInstrumentFunction - Return true if the current function should be
2528	/// instrumented with __cyg_profile_func_* calls
2529	bool ShouldInstrumentFunction();
2530
2531	/// ShouldSkipSanitizerInstrumentation - Return true if the current function
2532	/// should not be instrumented with sanitizers.
2533	bool ShouldSkipSanitizerInstrumentation();
2534
2535	/// ShouldXRayInstrument - Return true if the current function should be
2536	/// instrumented with XRay nop sleds.
2537	bool ShouldXRayInstrumentFunction() const;
2538
2539	/// AlwaysEmitXRayCustomEvents - Return true if we must unconditionally emit
2540	/// XRay custom event handling calls.
2541	bool AlwaysEmitXRayCustomEvents() const;
2542
2543	/// AlwaysEmitXRayTypedEvents - Return true if clang must unconditionally emit
2544	/// XRay typed event handling calls.
2545	bool AlwaysEmitXRayTypedEvents() const;
2546
2547	/// Return a type hash constant for a function instrumented by
2548	/// -fsanitize=function.
2549	llvm::ConstantInt *getUBSanFunctionTypeHash(QualType T) const;
2550
2551	/// EmitFunctionProlog - Emit the target specific LLVM code to load the
2552	/// arguments for the given function. This is also responsible for naming the
2553	/// LLVM function arguments.
2554	void EmitFunctionProlog(const CGFunctionInfo &FI, llvm::Function *Fn,
2555	const FunctionArgList &Args);
2556
2557	/// EmitFunctionEpilog - Emit the target specific LLVM code to return the
2558	/// given temporary. Specify the source location atom group (Key Instructions
2559	/// debug info feature) for the `ret` using \p RetKeyInstructionsSourceAtom.
2560	/// If it's 0, the `ret` will get added to a new source atom group.
2561	void EmitFunctionEpilog(const CGFunctionInfo &FI, bool EmitRetDbgLoc,
2562	SourceLocation EndLoc,
2563	uint64_t RetKeyInstructionsSourceAtom);
2564
2565	/// Emit a test that checks if the return value \p RV is nonnull.
2566	void EmitReturnValueCheck(llvm::Value *RV);
2567
2568	/// EmitStartEHSpec - Emit the start of the exception spec.
2569	void EmitStartEHSpec(const Decl *D);
2570
2571	/// EmitEndEHSpec - Emit the end of the exception spec.
2572	void EmitEndEHSpec(const Decl *D);
2573
2574	/// getTerminateLandingPad - Return a landing pad that just calls terminate.
2575	llvm::BasicBlock *getTerminateLandingPad();
2576
2577	/// getTerminateLandingPad - Return a cleanup funclet that just calls
2578	/// terminate.
2579	llvm::BasicBlock *getTerminateFunclet();
2580
2581	/// getTerminateHandler - Return a handler (not a landing pad, just
2582	/// a catch handler) that just calls terminate. This is used when
2583	/// a terminate scope encloses a try.
2584	llvm::BasicBlock *getTerminateHandler();
2585
2586	llvm::Type *ConvertTypeForMem(QualType T);
2587	llvm::Type *ConvertType(QualType T);
2588	llvm::Type *convertTypeForLoadStore(QualType ASTTy,
2589	llvm::Type *LLVMTy = nullptr);
2590	llvm::Type *ConvertType(const TypeDecl *T) {
2591	return ConvertType(T: getContext().getTypeDeclType(Decl: T));
2592	}
2593
2594	/// LoadObjCSelf - Load the value of self. This function is only valid while
2595	/// generating code for an Objective-C method.
2596	llvm::Value *LoadObjCSelf();
2597
2598	/// TypeOfSelfObject - Return type of object that this self represents.
2599	QualType TypeOfSelfObject();
2600
2601	/// getEvaluationKind - Return the TypeEvaluationKind of QualType \c T.
2602	static TypeEvaluationKind getEvaluationKind(QualType T);
2603
2604	static bool hasScalarEvaluationKind(QualType T) {
2605	return getEvaluationKind(T) == TEK_Scalar;
2606	}
2607
2608	static bool hasAggregateEvaluationKind(QualType T) {
2609	return getEvaluationKind(T) == TEK_Aggregate;
2610	}
2611
2612	/// createBasicBlock - Create an LLVM basic block.
2613	llvm::BasicBlock *createBasicBlock(const Twine &name = "",
2614	llvm::Function *parent = nullptr,
2615	llvm::BasicBlock *before = nullptr) {
2616	return llvm::BasicBlock::Create(Context&: getLLVMContext(), Name: name, Parent: parent, InsertBefore: before);
2617	}
2618
2619	/// getBasicBlockForLabel - Return the LLVM basicblock that the specified
2620	/// label maps to.
2621	JumpDest getJumpDestForLabel(const LabelDecl *S);
2622
2623	/// SimplifyForwardingBlocks - If the given basic block is only a branch to
2624	/// another basic block, simplify it. This assumes that no other code could
2625	/// potentially reference the basic block.
2626	void SimplifyForwardingBlocks(llvm::BasicBlock *BB);
2627
2628	/// EmitBlock - Emit the given block \arg BB and set it as the insert point,
2629	/// adding a fall-through branch from the current insert block if
2630	/// necessary. It is legal to call this function even if there is no current
2631	/// insertion point.
2632	///
2633	/// IsFinished - If true, indicates that the caller has finished emitting
2634	/// branches to the given block and does not expect to emit code into it. This
2635	/// means the block can be ignored if it is unreachable.
2636	void EmitBlock(llvm::BasicBlock *BB, bool IsFinished = false);
2637
2638	/// EmitBlockAfterUses - Emit the given block somewhere hopefully
2639	/// near its uses, and leave the insertion point in it.
2640	void EmitBlockAfterUses(llvm::BasicBlock *BB);
2641
2642	/// EmitBranch - Emit a branch to the specified basic block from the current
2643	/// insert block, taking care to avoid creation of branches from dummy
2644	/// blocks. It is legal to call this function even if there is no current
2645	/// insertion point.
2646	///
2647	/// This function clears the current insertion point. The caller should follow
2648	/// calls to this function with calls to Emit*Block prior to generation new
2649	/// code.
2650	void EmitBranch(llvm::BasicBlock *Block);
2651
2652	/// HaveInsertPoint - True if an insertion point is defined. If not, this
2653	/// indicates that the current code being emitted is unreachable.
2654	bool HaveInsertPoint() const { return Builder.GetInsertBlock() != nullptr; }
2655
2656	/// EnsureInsertPoint - Ensure that an insertion point is defined so that
2657	/// emitted IR has a place to go. Note that by definition, if this function
2658	/// creates a block then that block is unreachable; callers may do better to
2659	/// detect when no insertion point is defined and simply skip IR generation.
2660	void EnsureInsertPoint() {
2661	if (!HaveInsertPoint())
2662	EmitBlock(BB: createBasicBlock());
2663	}
2664
2665	/// ErrorUnsupported - Print out an error that codegen doesn't support the
2666	/// specified stmt yet.
2667	void ErrorUnsupported(const Stmt *S, const char *Type);
2668
2669	//===--------------------------------------------------------------------===//
2670	// Helpers
2671	//===--------------------------------------------------------------------===//
2672
2673	Address mergeAddressesInConditionalExpr(Address LHS, Address RHS,
2674	llvm::BasicBlock *LHSBlock,
2675	llvm::BasicBlock *RHSBlock,
2676	llvm::BasicBlock *MergeBlock,
2677	QualType MergedType) {
2678	Builder.SetInsertPoint(MergeBlock);
2679	llvm::PHINode *PtrPhi = Builder.CreatePHI(Ty: LHS.getType(), NumReservedValues: 2, Name: "cond");
2680	PtrPhi->addIncoming(V: LHS.getBasePointer(), BB: LHSBlock);
2681	PtrPhi->addIncoming(V: RHS.getBasePointer(), BB: RHSBlock);
2682	LHS.replaceBasePointer(P: PtrPhi);
2683	LHS.setAlignment(std::min(a: LHS.getAlignment(), b: RHS.getAlignment()));
2684	return LHS;
2685	}
2686
2687	/// Construct an address with the natural alignment of T. If a pointer to T
2688	/// is expected to be signed, the pointer passed to this function must have
2689	/// been signed, and the returned Address will have the pointer authentication
2690	/// information needed to authenticate the signed pointer.
2691	Address makeNaturalAddressForPointer(
2692	llvm::Value *Ptr, QualType T, CharUnits Alignment = CharUnits::Zero(),
2693	bool ForPointeeType = false, LValueBaseInfo *BaseInfo = nullptr,
2694	TBAAAccessInfo *TBAAInfo = nullptr,
2695	KnownNonNull_t IsKnownNonNull = NotKnownNonNull) {
2696	if (Alignment.isZero())
2697	Alignment =
2698	CGM.getNaturalTypeAlignment(T, BaseInfo, TBAAInfo, forPointeeType: ForPointeeType);
2699	return Address(Ptr, ConvertTypeForMem(T), Alignment,
2700	CGM.getPointerAuthInfoForPointeeType(type: T), /*Offset=*/nullptr,
2701	IsKnownNonNull);
2702	}
2703
2704	LValue MakeAddrLValue(Address Addr, QualType T,
2705	AlignmentSource Source = AlignmentSource::Type) {
2706	return MakeAddrLValue(Addr, T, BaseInfo: LValueBaseInfo(Source),
2707	TBAAInfo: CGM.getTBAAAccessInfo(AccessType: T));
2708	}
2709
2710	LValue MakeAddrLValue(Address Addr, QualType T, LValueBaseInfo BaseInfo,
2711	TBAAAccessInfo TBAAInfo) {
2712	return LValue::MakeAddr(Addr, type: T, Context&: getContext(), BaseInfo, TBAAInfo);
2713	}
2714
2715	LValue MakeAddrLValue(llvm::Value *V, QualType T, CharUnits Alignment,
2716	AlignmentSource Source = AlignmentSource::Type) {
2717	return MakeAddrLValue(Addr: makeNaturalAddressForPointer(Ptr: V, T, Alignment), T,
2718	BaseInfo: LValueBaseInfo(Source), TBAAInfo: CGM.getTBAAAccessInfo(AccessType: T));
2719	}
2720
2721	/// Same as MakeAddrLValue above except that the pointer is known to be
2722	/// unsigned.
2723	LValue MakeRawAddrLValue(llvm::Value *V, QualType T, CharUnits Alignment,
2724	AlignmentSource Source = AlignmentSource::Type) {
2725	Address Addr(V, ConvertTypeForMem(T), Alignment);
2726	return LValue::MakeAddr(Addr, type: T, Context&: getContext(), BaseInfo: LValueBaseInfo(Source),
2727	TBAAInfo: CGM.getTBAAAccessInfo(AccessType: T));
2728	}
2729
2730	LValue
2731	MakeAddrLValueWithoutTBAA(Address Addr, QualType T,
2732	AlignmentSource Source = AlignmentSource::Type) {
2733	return LValue::MakeAddr(Addr, type: T, Context&: getContext(), BaseInfo: LValueBaseInfo(Source),
2734	TBAAInfo: TBAAAccessInfo());
2735	}
2736
2737	/// Given a value of type T* that may not be to a complete object, construct
2738	/// an l-value with the natural pointee alignment of T.
2739	LValue MakeNaturalAlignPointeeAddrLValue(llvm::Value *V, QualType T);
2740
2741	LValue
2742	MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T,
2743	KnownNonNull_t IsKnownNonNull = NotKnownNonNull);
2744
2745	/// Same as MakeNaturalAlignPointeeAddrLValue except that the pointer is known
2746	/// to be unsigned.
2747	LValue MakeNaturalAlignPointeeRawAddrLValue(llvm::Value *V, QualType T);
2748
2749	LValue MakeNaturalAlignRawAddrLValue(llvm::Value *V, QualType T);
2750
2751	Address EmitLoadOfReference(LValue RefLVal,
2752	LValueBaseInfo *PointeeBaseInfo = nullptr,
2753	TBAAAccessInfo *PointeeTBAAInfo = nullptr);
2754	LValue EmitLoadOfReferenceLValue(LValue RefLVal);
2755	LValue
2756	EmitLoadOfReferenceLValue(Address RefAddr, QualType RefTy,
2757	AlignmentSource Source = AlignmentSource::Type) {
2758	LValue RefLVal = MakeAddrLValue(Addr: RefAddr, T: RefTy, BaseInfo: LValueBaseInfo(Source),
2759	TBAAInfo: CGM.getTBAAAccessInfo(AccessType: RefTy));
2760	return EmitLoadOfReferenceLValue(RefLVal);
2761	}
2762
2763	/// Load a pointer with type \p PtrTy stored at address \p Ptr.
2764	/// Note that \p PtrTy is the type of the loaded pointer, not the addresses
2765	/// it is loaded from.
2766	Address EmitLoadOfPointer(Address Ptr, const PointerType *PtrTy,
2767	LValueBaseInfo *BaseInfo = nullptr,
2768	TBAAAccessInfo *TBAAInfo = nullptr);
2769	LValue EmitLoadOfPointerLValue(Address Ptr, const PointerType *PtrTy);
2770
2771	private:
2772	struct AllocaTracker {
2773	void Add(llvm::AllocaInst *I) { Allocas.push_back(Elt: I); }
2774	llvm::SmallVector<llvm::AllocaInst *> Take() { return std::move(Allocas); }
2775
2776	private:
2777	llvm::SmallVector<llvm::AllocaInst *> Allocas;
2778	};
2779	AllocaTracker *Allocas = nullptr;
2780
2781	/// CGDecl helper.
2782	void emitStoresForConstant(const VarDecl &D, Address Loc, bool isVolatile,
2783	llvm::Constant *constant, bool IsAutoInit);
2784	/// CGDecl helper.
2785	void emitStoresForZeroInit(const VarDecl &D, Address Loc, bool isVolatile);
2786	/// CGDecl helper.
2787	void emitStoresForPatternInit(const VarDecl &D, Address Loc, bool isVolatile);
2788	/// CGDecl helper.
2789	void emitStoresForInitAfterBZero(llvm::Constant *Init, Address Loc,
2790	bool isVolatile, bool IsAutoInit);
2791
2792	public:
2793	// Captures all the allocas created during the scope of its RAII object.
2794	struct AllocaTrackerRAII {
2795	AllocaTrackerRAII(CodeGenFunction &CGF)
2796	: CGF(CGF), OldTracker(CGF.Allocas) {
2797	CGF.Allocas = &Tracker;
2798	}
2799	~AllocaTrackerRAII() { CGF.Allocas = OldTracker; }
2800
2801	llvm::SmallVector<llvm::AllocaInst *> Take() { return Tracker.Take(); }
2802
2803	private:
2804	CodeGenFunction &CGF;
2805	AllocaTracker *OldTracker;
2806	AllocaTracker Tracker;
2807	};
2808
2809	/// CreateTempAlloca - This creates an alloca and inserts it into the entry
2810	/// block if \p ArraySize is nullptr, otherwise inserts it at the current
2811	/// insertion point of the builder. The caller is responsible for setting an
2812	/// appropriate alignment on
2813	/// the alloca.
2814	///
2815	/// \p ArraySize is the number of array elements to be allocated if it
2816	/// is not nullptr.
2817	///
2818	/// LangAS::Default is the address space of pointers to local variables and
2819	/// temporaries, as exposed in the source language. In certain
2820	/// configurations, this is not the same as the alloca address space, and a
2821	/// cast is needed to lift the pointer from the alloca AS into
2822	/// LangAS::Default. This can happen when the target uses a restricted
2823	/// address space for the stack but the source language requires
2824	/// LangAS::Default to be a generic address space. The latter condition is
2825	/// common for most programming languages; OpenCL is an exception in that
2826	/// LangAS::Default is the private address space, which naturally maps
2827	/// to the stack.
2828	///
2829	/// Because the address of a temporary is often exposed to the program in
2830	/// various ways, this function will perform the cast. The original alloca
2831	/// instruction is returned through \p Alloca if it is not nullptr.
2832	///
2833	/// The cast is not performaed in CreateTempAllocaWithoutCast. This is
2834	/// more efficient if the caller knows that the address will not be exposed.
2835	llvm::AllocaInst *CreateTempAlloca(llvm::Type *Ty, const Twine &Name = "tmp",
2836	llvm::Value *ArraySize = nullptr);
2837
2838	/// CreateTempAlloca - This creates a alloca and inserts it into the entry
2839	/// block. The alloca is casted to the address space of \p UseAddrSpace if
2840	/// necessary.
2841	RawAddress CreateTempAlloca(llvm::Type *Ty, LangAS UseAddrSpace,
2842	CharUnits align, const Twine &Name = "tmp",
2843	llvm::Value *ArraySize = nullptr,
2844	RawAddress *Alloca = nullptr);
2845
2846	/// CreateTempAlloca - This creates a alloca and inserts it into the entry
2847	/// block. The alloca is casted to default address space if necessary.
2848	///
2849	/// FIXME: This version should be removed, and context should provide the
2850	/// context use address space used instead of default.
2851	RawAddress CreateTempAlloca(llvm::Type *Ty, CharUnits align,
2852	const Twine &Name = "tmp",
2853	llvm::Value *ArraySize = nullptr,
2854	RawAddress *Alloca = nullptr) {
2855	return CreateTempAlloca(Ty, UseAddrSpace: LangAS::Default, align, Name, ArraySize,
2856	Alloca);
2857	}
2858
2859	RawAddress CreateTempAllocaWithoutCast(llvm::Type *Ty, CharUnits align,
2860	const Twine &Name = "tmp",
2861	llvm::Value *ArraySize = nullptr);
2862
2863	/// CreateDefaultAlignedTempAlloca - This creates an alloca with the
2864	/// default ABI alignment of the given LLVM type.
2865	///
2866	/// IMPORTANT NOTE: This is *not* generally the right alignment for
2867	/// any given AST type that happens to have been lowered to the
2868	/// given IR type. This should only ever be used for function-local,
2869	/// IR-driven manipulations like saving and restoring a value. Do
2870	/// not hand this address off to arbitrary IRGen routines, and especially
2871	/// do not pass it as an argument to a function that might expect a
2872	/// properly ABI-aligned value.
2873	RawAddress CreateDefaultAlignTempAlloca(llvm::Type *Ty,
2874	const Twine &Name = "tmp");
2875
2876	/// CreateIRTemp - Create a temporary IR object of the given type, with
2877	/// appropriate alignment. This routine should only be used when an temporary
2878	/// value needs to be stored into an alloca (for example, to avoid explicit
2879	/// PHI construction), but the type is the IR type, not the type appropriate
2880	/// for storing in memory.
2881	///
2882	/// That is, this is exactly equivalent to CreateMemTemp, but calling
2883	/// ConvertType instead of ConvertTypeForMem.
2884	RawAddress CreateIRTemp(QualType T, const Twine &Name = "tmp");
2885
2886	/// CreateMemTemp - Create a temporary memory object of the given type, with
2887	/// appropriate alignmen and cast it to the default address space. Returns
2888	/// the original alloca instruction by \p Alloca if it is not nullptr.
2889	RawAddress CreateMemTemp(QualType T, const Twine &Name = "tmp",
2890	RawAddress *Alloca = nullptr);
2891	RawAddress CreateMemTemp(QualType T, CharUnits Align,
2892	const Twine &Name = "tmp",
2893	RawAddress *Alloca = nullptr);
2894
2895	/// CreateMemTemp - Create a temporary memory object of the given type, with
2896	/// appropriate alignmen without casting it to the default address space.
2897	RawAddress CreateMemTempWithoutCast(QualType T, const Twine &Name = "tmp");
2898	RawAddress CreateMemTempWithoutCast(QualType T, CharUnits Align,
2899	const Twine &Name = "tmp");
2900
2901	/// CreateAggTemp - Create a temporary memory object for the given
2902	/// aggregate type.
2903	AggValueSlot CreateAggTemp(QualType T, const Twine &Name = "tmp",
2904	RawAddress *Alloca = nullptr) {
2905	return AggValueSlot::forAddr(
2906	addr: CreateMemTemp(T, Name, Alloca), quals: T.getQualifiers(),
2907	isDestructed: AggValueSlot::IsNotDestructed, needsGC: AggValueSlot::DoesNotNeedGCBarriers,
2908	isAliased: AggValueSlot::IsNotAliased, mayOverlap: AggValueSlot::DoesNotOverlap);
2909	}
2910
2911	/// EvaluateExprAsBool - Perform the usual unary conversions on the specified
2912	/// expression and compare the result against zero, returning an Int1Ty value.
2913	llvm::Value *EvaluateExprAsBool(const Expr *E);
2914
2915	/// Retrieve the implicit cast expression of the rhs in a binary operator
2916	/// expression by passing pointers to Value and QualType
2917	/// This is used for implicit bitfield conversion checks, which
2918	/// must compare with the value before potential truncation.
2919	llvm::Value *EmitWithOriginalRHSBitfieldAssignment(const BinaryOperator *E,
2920	llvm::Value **Previous,
2921	QualType *SrcType);
2922
2923	/// Emit a check that an [implicit] conversion of a bitfield. It is not UB,
2924	/// so we use the value after conversion.
2925	void EmitBitfieldConversionCheck(llvm::Value *Src, QualType SrcType,
2926	llvm::Value *Dst, QualType DstType,
2927	const CGBitFieldInfo &Info,
2928	SourceLocation Loc);
2929
2930	/// EmitIgnoredExpr - Emit an expression in a context which ignores the
2931	/// result.
2932	void EmitIgnoredExpr(const Expr *E);
2933
2934	/// EmitAnyExpr - Emit code to compute the specified expression which can have
2935	/// any type. The result is returned as an RValue struct. If this is an
2936	/// aggregate expression, the aggloc/agglocvolatile arguments indicate where
2937	/// the result should be returned.
2938	///
2939	/// \param ignoreResult True if the resulting value isn't used.
2940	RValue EmitAnyExpr(const Expr *E,
2941	AggValueSlot aggSlot = AggValueSlot::ignored(),
2942	bool ignoreResult = false);
2943
2944	// EmitVAListRef - Emit a "reference" to a va_list; this is either the address
2945	// or the value of the expression, depending on how va_list is defined.
2946	Address EmitVAListRef(const Expr *E);
2947
2948	/// Emit a "reference" to a __builtin_ms_va_list; this is
2949	/// always the value of the expression, because a __builtin_ms_va_list is a
2950	/// pointer to a char.
2951	Address EmitMSVAListRef(const Expr *E);
2952
2953	/// EmitAnyExprToTemp - Similarly to EmitAnyExpr(), however, the result will
2954	/// always be accessible even if no aggregate location is provided.
2955	RValue EmitAnyExprToTemp(const Expr *E);
2956
2957	/// EmitAnyExprToMem - Emits the code necessary to evaluate an
2958	/// arbitrary expression into the given memory location.
2959	void EmitAnyExprToMem(const Expr *E, Address Location, Qualifiers Quals,
2960	bool IsInitializer);
2961
2962	void EmitAnyExprToExn(const Expr *E, Address Addr);
2963
2964	/// EmitInitializationToLValue - Emit an initializer to an LValue.
2965	void EmitInitializationToLValue(
2966	const Expr *E, LValue LV,
2967	AggValueSlot::IsZeroed_t IsZeroed = AggValueSlot::IsNotZeroed);
2968
2969	/// EmitExprAsInit - Emits the code necessary to initialize a
2970	/// location in memory with the given initializer.
2971	void EmitExprAsInit(const Expr *init, const ValueDecl *D, LValue lvalue,
2972	bool capturedByInit);
2973
2974	/// hasVolatileMember - returns true if aggregate type has a volatile
2975	/// member.
2976	bool hasVolatileMember(QualType T) {
2977	if (const RecordType *RT = T->getAs<RecordType>()) {
2978	const RecordDecl *RD = cast<RecordDecl>(Val: RT->getDecl());
2979	return RD->hasVolatileMember();
2980	}
2981	return false;
2982	}
2983
2984	/// Determine whether a return value slot may overlap some other object.
2985	AggValueSlot::Overlap_t getOverlapForReturnValue() {
2986	// FIXME: Assuming no overlap here breaks guaranteed copy elision for base
2987	// class subobjects. These cases may need to be revisited depending on the
2988	// resolution of the relevant core issue.
2989	return AggValueSlot::DoesNotOverlap;
2990	}
2991
2992	/// Determine whether a field initialization may overlap some other object.
2993	AggValueSlot::Overlap_t getOverlapForFieldInit(const FieldDecl *FD);
2994
2995	/// Determine whether a base class initialization may overlap some other
2996	/// object.
2997	AggValueSlot::Overlap_t getOverlapForBaseInit(const CXXRecordDecl *RD,
2998	const CXXRecordDecl *BaseRD,
2999	bool IsVirtual);
3000
3001	/// Emit an aggregate assignment.
3002	void EmitAggregateAssign(LValue Dest, LValue Src, QualType EltTy) {
3003	ApplyAtomGroup Grp(getDebugInfo());
3004	bool IsVolatile = hasVolatileMember(T: EltTy);
3005	EmitAggregateCopy(Dest, Src, EltTy, MayOverlap: AggValueSlot::MayOverlap, isVolatile: IsVolatile);
3006	}
3007
3008	void EmitAggregateCopyCtor(LValue Dest, LValue Src,
3009	AggValueSlot::Overlap_t MayOverlap) {
3010	EmitAggregateCopy(Dest, Src, EltTy: Src.getType(), MayOverlap);
3011	}
3012
3013	/// EmitAggregateCopy - Emit an aggregate copy.
3014	///
3015	/// \param isVolatile \c true iff either the source or the destination is
3016	/// volatile.
3017	/// \param MayOverlap Whether the tail padding of the destination might be
3018	/// occupied by some other object. More efficient code can often be
3019	/// generated if not.
3020	void EmitAggregateCopy(LValue Dest, LValue Src, QualType EltTy,
3021	AggValueSlot::Overlap_t MayOverlap,
3022	bool isVolatile = false);
3023
3024	/// GetAddrOfLocalVar - Return the address of a local variable.
3025	Address GetAddrOfLocalVar(const VarDecl *VD) {
3026	auto it = LocalDeclMap.find(Val: VD);
3027	assert(it != LocalDeclMap.end() &&
3028	"Invalid argument to GetAddrOfLocalVar(), no decl!");
3029	return it->second;
3030	}
3031
3032	/// Given an opaque value expression, return its LValue mapping if it exists,
3033	/// otherwise create one.
3034	LValue getOrCreateOpaqueLValueMapping(const OpaqueValueExpr *e);
3035
3036	/// Given an opaque value expression, return its RValue mapping if it exists,
3037	/// otherwise create one.
3038	RValue getOrCreateOpaqueRValueMapping(const OpaqueValueExpr *e);
3039
3040	/// isOpaqueValueEmitted - Return true if the opaque value expression has
3041	/// already been emitted.
3042	bool isOpaqueValueEmitted(const OpaqueValueExpr *E);
3043
3044	/// Get the index of the current ArrayInitLoopExpr, if any.
3045	llvm::Value *getArrayInitIndex() { return ArrayInitIndex; }
3046
3047	/// getAccessedFieldNo - Given an encoded value and a result number, return
3048	/// the input field number being accessed.
3049	static unsigned getAccessedFieldNo(unsigned Idx, const llvm::Constant *Elts);
3050
3051	llvm::BlockAddress *GetAddrOfLabel(const LabelDecl *L);
3052	llvm::BasicBlock *GetIndirectGotoBlock();
3053
3054	/// Check if \p E is a C++ "this" pointer wrapped in value-preserving casts.
3055	static bool IsWrappedCXXThis(const Expr *E);
3056
3057	/// EmitNullInitialization - Generate code to set a value of the given type to
3058	/// null, If the type contains data member pointers, they will be initialized
3059	/// to -1 in accordance with the Itanium C++ ABI.
3060	void EmitNullInitialization(Address DestPtr, QualType Ty);
3061
3062	/// Emits a call to an LLVM variable-argument intrinsic, either
3063	/// \c llvm.va_start or \c llvm.va_end.
3064	/// \param ArgValue A reference to the \c va_list as emitted by either
3065	/// \c EmitVAListRef or \c EmitMSVAListRef.
3066	/// \param IsStart If \c true, emits a call to \c llvm.va_start; otherwise,
3067	/// calls \c llvm.va_end.
3068	llvm::Value *EmitVAStartEnd(llvm::Value *ArgValue, bool IsStart);
3069
3070	/// Generate code to get an argument from the passed in pointer
3071	/// and update it accordingly.
3072	/// \param VE The \c VAArgExpr for which to generate code.
3073	/// \param VAListAddr Receives a reference to the \c va_list as emitted by
3074	/// either \c EmitVAListRef or \c EmitMSVAListRef.
3075	/// \returns A pointer to the argument.
3076	// FIXME: We should be able to get rid of this method and use the va_arg
3077	// instruction in LLVM instead once it works well enough.
3078	RValue EmitVAArg(VAArgExpr *VE, Address &VAListAddr,
3079	AggValueSlot Slot = AggValueSlot::ignored());
3080
3081	/// emitArrayLength - Compute the length of an array, even if it's a
3082	/// VLA, and drill down to the base element type.
3083	llvm::Value *emitArrayLength(const ArrayType *arrayType, QualType &baseType,
3084	Address &addr);
3085
3086	/// EmitVLASize - Capture all the sizes for the VLA expressions in
3087	/// the given variably-modified type and store them in the VLASizeMap.
3088	///
3089	/// This function can be called with a null (unreachable) insert point.
3090	void EmitVariablyModifiedType(QualType Ty);
3091
3092	struct VlaSizePair {
3093	llvm::Value *NumElts;
3094	QualType Type;
3095
3096	VlaSizePair(llvm::Value *NE, QualType T) : NumElts(NE), Type(T) {}
3097	};
3098
3099	/// Return the number of elements for a single dimension
3100	/// for the given array type.
3101	VlaSizePair getVLAElements1D(const VariableArrayType *vla);
3102	VlaSizePair getVLAElements1D(QualType vla);
3103
3104	/// Returns an LLVM value that corresponds to the size,
3105	/// in non-variably-sized elements, of a variable length array type,
3106	/// plus that largest non-variably-sized element type. Assumes that
3107	/// the type has already been emitted with EmitVariablyModifiedType.
3108	VlaSizePair getVLASize(const VariableArrayType *vla);
3109	VlaSizePair getVLASize(QualType vla);
3110
3111	/// LoadCXXThis - Load the value of 'this'. This function is only valid while
3112	/// generating code for an C++ member function.
3113	llvm::Value *LoadCXXThis() {
3114	assert(CXXThisValue && "no 'this' value for this function");
3115	return CXXThisValue;
3116	}
3117	Address LoadCXXThisAddress();
3118
3119	/// LoadCXXVTT - Load the VTT parameter to base constructors/destructors have
3120	/// virtual bases.
3121	// FIXME: Every place that calls LoadCXXVTT is something
3122	// that needs to be abstracted properly.
3123	llvm::Value *LoadCXXVTT() {
3124	assert(CXXStructorImplicitParamValue && "no VTT value for this function");
3125	return CXXStructorImplicitParamValue;
3126	}
3127
3128	/// GetAddressOfBaseOfCompleteClass - Convert the given pointer to a
3129	/// complete class to the given direct base.
3130	Address GetAddressOfDirectBaseInCompleteClass(Address Value,
3131	const CXXRecordDecl *Derived,
3132	const CXXRecordDecl *Base,
3133	bool BaseIsVirtual);
3134
3135	static bool ShouldNullCheckClassCastValue(const CastExpr *Cast);
3136
3137	/// GetAddressOfBaseClass - This function will add the necessary delta to the
3138	/// load of 'this' and returns address of the base class.
3139	Address GetAddressOfBaseClass(Address Value, const CXXRecordDecl *Derived,
3140	CastExpr::path_const_iterator PathBegin,
3141	CastExpr::path_const_iterator PathEnd,
3142	bool NullCheckValue, SourceLocation Loc);
3143
3144	Address GetAddressOfDerivedClass(Address Value, const CXXRecordDecl *Derived,
3145	CastExpr::path_const_iterator PathBegin,
3146	CastExpr::path_const_iterator PathEnd,
3147	bool NullCheckValue);
3148
3149	/// GetVTTParameter - Return the VTT parameter that should be passed to a
3150	/// base constructor/destructor with virtual bases.
3151	/// FIXME: VTTs are Itanium ABI-specific, so the definition should move
3152	/// to ItaniumCXXABI.cpp together with all the references to VTT.
3153	llvm::Value *GetVTTParameter(GlobalDecl GD, bool ForVirtualBase,
3154	bool Delegating);
3155
3156	void EmitDelegateCXXConstructorCall(const CXXConstructorDecl *Ctor,
3157	CXXCtorType CtorType,
3158	const FunctionArgList &Args,
3159	SourceLocation Loc);
3160	// It's important not to confuse this and the previous function. Delegating
3161	// constructors are the C++0x feature. The constructor delegate optimization
3162	// is used to reduce duplication in the base and complete consturctors where
3163	// they are substantially the same.
3164	void EmitDelegatingCXXConstructorCall(const CXXConstructorDecl *Ctor,
3165	const FunctionArgList &Args);
3166
3167	/// Emit a call to an inheriting constructor (that is, one that invokes a
3168	/// constructor inherited from a base class) by inlining its definition. This
3169	/// is necessary if the ABI does not support forwarding the arguments to the
3170	/// base class constructor (because they're variadic or similar).
3171	void EmitInlinedInheritingCXXConstructorCall(const CXXConstructorDecl *Ctor,
3172	CXXCtorType CtorType,
3173	bool ForVirtualBase,
3174	bool Delegating,
3175	CallArgList &Args);
3176
3177	/// Emit a call to a constructor inherited from a base class, passing the
3178	/// current constructor's arguments along unmodified (without even making
3179	/// a copy).
3180	void EmitInheritedCXXConstructorCall(const CXXConstructorDecl *D,
3181	bool ForVirtualBase, Address This,
3182	bool InheritedFromVBase,
3183	const CXXInheritedCtorInitExpr *E);
3184
3185	void EmitCXXConstructorCall(const CXXConstructorDecl *D, CXXCtorType Type,
3186	bool ForVirtualBase, bool Delegating,
3187	AggValueSlot ThisAVS, const CXXConstructExpr *E);
3188
3189	void EmitCXXConstructorCall(const CXXConstructorDecl *D, CXXCtorType Type,
3190	bool ForVirtualBase, bool Delegating,
3191	Address This, CallArgList &Args,
3192	AggValueSlot::Overlap_t Overlap,
3193	SourceLocation Loc, bool NewPointerIsChecked,
3194	llvm::CallBase **CallOrInvoke = nullptr);
3195
3196	/// Emit assumption load for all bases. Requires to be called only on
3197	/// most-derived class and not under construction of the object.
3198	void EmitVTableAssumptionLoads(const CXXRecordDecl *ClassDecl, Address This);
3199
3200	/// Emit assumption that vptr load == global vtable.
3201	void EmitVTableAssumptionLoad(const VPtr &vptr, Address This);
3202
3203	void EmitSynthesizedCXXCopyCtorCall(const CXXConstructorDecl *D, Address This,
3204	Address Src, const CXXConstructExpr *E);
3205
3206	void EmitCXXAggrConstructorCall(const CXXConstructorDecl *D,
3207	const ArrayType *ArrayTy, Address ArrayPtr,
3208	const CXXConstructExpr *E,
3209	bool NewPointerIsChecked,
3210	bool ZeroInitialization = false);
3211
3212	void EmitCXXAggrConstructorCall(const CXXConstructorDecl *D,
3213	llvm::Value *NumElements, Address ArrayPtr,
3214	const CXXConstructExpr *E,
3215	bool NewPointerIsChecked,
3216	bool ZeroInitialization = false);
3217
3218	static Destroyer destroyCXXObject;
3219
3220	void EmitCXXDestructorCall(const CXXDestructorDecl *D, CXXDtorType Type,
3221	bool ForVirtualBase, bool Delegating, Address This,
3222	QualType ThisTy);
3223
3224	void EmitNewArrayInitializer(const CXXNewExpr *E, QualType elementType,
3225	llvm::Type *ElementTy, Address NewPtr,
3226	llvm::Value *NumElements,
3227	llvm::Value *AllocSizeWithoutCookie);
3228
3229	void EmitCXXTemporary(const CXXTemporary *Temporary, QualType TempType,
3230	Address Ptr);
3231
3232	void EmitSehCppScopeBegin();
3233	void EmitSehCppScopeEnd();
3234	void EmitSehTryScopeBegin();
3235	void EmitSehTryScopeEnd();
3236
3237	llvm::Value *EmitLifetimeStart(llvm::TypeSize Size, llvm::Value *Addr);
3238	void EmitLifetimeEnd(llvm::Value *Size, llvm::Value *Addr);
3239
3240	llvm::Value *EmitCXXNewExpr(const CXXNewExpr *E);
3241	void EmitCXXDeleteExpr(const CXXDeleteExpr *E);
3242
3243	void EmitDeleteCall(const FunctionDecl *DeleteFD, llvm::Value *Ptr,
3244	QualType DeleteTy, llvm::Value *NumElements = nullptr,
3245	CharUnits CookieSize = CharUnits());
3246
3247	RValue EmitBuiltinNewDeleteCall(const FunctionProtoType *Type,
3248	const CallExpr *TheCallExpr, bool IsDelete);
3249
3250	llvm::Value *EmitCXXTypeidExpr(const CXXTypeidExpr *E);
3251	llvm::Value *EmitDynamicCast(Address V, const CXXDynamicCastExpr *DCE);
3252	Address EmitCXXUuidofExpr(const CXXUuidofExpr *E);
3253
3254	/// Situations in which we might emit a check for the suitability of a
3255	/// pointer or glvalue. Needs to be kept in sync with ubsan_handlers.cpp in
3256	/// compiler-rt.
3257	enum TypeCheckKind {
3258	/// Checking the operand of a load. Must be suitably sized and aligned.
3259	TCK_Load,
3260	/// Checking the destination of a store. Must be suitably sized and aligned.
3261	TCK_Store,
3262	/// Checking the bound value in a reference binding. Must be suitably sized
3263	/// and aligned, but is not required to refer to an object (until the
3264	/// reference is used), per core issue 453.
3265	TCK_ReferenceBinding,
3266	/// Checking the object expression in a non-static data member access. Must
3267	/// be an object within its lifetime.
3268	TCK_MemberAccess,
3269	/// Checking the 'this' pointer for a call to a non-static member function.
3270	/// Must be an object within its lifetime.
3271	TCK_MemberCall,
3272	/// Checking the 'this' pointer for a constructor call.
3273	TCK_ConstructorCall,
3274	/// Checking the operand of a static_cast to a derived pointer type. Must be
3275	/// null or an object within its lifetime.
3276	TCK_DowncastPointer,
3277	/// Checking the operand of a static_cast to a derived reference type. Must
3278	/// be an object within its lifetime.
3279	TCK_DowncastReference,
3280	/// Checking the operand of a cast to a base object. Must be suitably sized
3281	/// and aligned.
3282	TCK_Upcast,
3283	/// Checking the operand of a cast to a virtual base object. Must be an
3284	/// object within its lifetime.
3285	TCK_UpcastToVirtualBase,
3286	/// Checking the value assigned to a _Nonnull pointer. Must not be null.
3287	TCK_NonnullAssign,
3288	/// Checking the operand of a dynamic_cast or a typeid expression. Must be
3289	/// null or an object within its lifetime.
3290	TCK_DynamicOperation
3291	};
3292
3293	/// Determine whether the pointer type check \p TCK permits null pointers.
3294	static bool isNullPointerAllowed(TypeCheckKind TCK);
3295
3296	/// Determine whether the pointer type check \p TCK requires a vptr check.
3297	static bool isVptrCheckRequired(TypeCheckKind TCK, QualType Ty);
3298
3299	/// Whether any type-checking sanitizers are enabled. If \c false,
3300	/// calls to EmitTypeCheck can be skipped.
3301	bool sanitizePerformTypeCheck() const;
3302
3303	void EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc, LValue LV,
3304	QualType Type, SanitizerSet SkippedChecks = SanitizerSet(),
3305	llvm::Value *ArraySize = nullptr) {
3306	if (!sanitizePerformTypeCheck())
3307	return;
3308	EmitTypeCheck(TCK, Loc, V: LV.emitRawPointer(CGF&: *this), Type, Alignment: LV.getAlignment(),
3309	SkippedChecks, ArraySize);
3310	}
3311
3312	void EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc, Address Addr,
3313	QualType Type, CharUnits Alignment = CharUnits::Zero(),
3314	SanitizerSet SkippedChecks = SanitizerSet(),
3315	llvm::Value *ArraySize = nullptr) {
3316	if (!sanitizePerformTypeCheck())
3317	return;
3318	EmitTypeCheck(TCK, Loc, V: Addr.emitRawPointer(CGF&: *this), Type, Alignment,
3319	SkippedChecks, ArraySize);
3320	}
3321
3322	/// Emit a check that \p V is the address of storage of the
3323	/// appropriate size and alignment for an object of type \p Type
3324	/// (or if ArraySize is provided, for an array of that bound).
3325	void EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc, llvm::Value *V,
3326	QualType Type, CharUnits Alignment = CharUnits::Zero(),
3327	SanitizerSet SkippedChecks = SanitizerSet(),
3328	llvm::Value *ArraySize = nullptr);
3329
3330	/// Emit a check that \p Base points into an array object, which
3331	/// we can access at index \p Index. \p Accessed should be \c false if we
3332	/// this expression is used as an lvalue, for instance in "&Arr[Idx]".
3333	void EmitBoundsCheck(const Expr *E, const Expr *Base, llvm::Value *Index,
3334	QualType IndexType, bool Accessed);
3335	void EmitBoundsCheckImpl(const Expr *E, llvm::Value *Bound,
3336	llvm::Value *Index, QualType IndexType,
3337	QualType IndexedType, bool Accessed);
3338
3339	/// Returns debug info, with additional annotation if
3340	/// CGM.getCodeGenOpts().SanitizeAnnotateDebugInfo[Ordinal] is enabled for
3341	/// any of the ordinals.
3342	llvm::DILocation *
3343	SanitizerAnnotateDebugInfo(ArrayRef<SanitizerKind::SanitizerOrdinal> Ordinals,
3344	SanitizerHandler Handler);
3345
3346	llvm::Value *GetCountedByFieldExprGEP(const Expr *Base, const FieldDecl *FD,
3347	const FieldDecl *CountDecl);
3348
3349	/// Build an expression accessing the "counted_by" field.
3350	llvm::Value *EmitLoadOfCountedByField(const Expr *Base, const FieldDecl *FD,
3351	const FieldDecl *CountDecl);
3352
3353	// Emit bounds checking for flexible array and pointer members with the
3354	// counted_by attribute.
3355	void EmitCountedByBoundsChecking(const Expr *E, llvm::Value *Idx,
3356	Address Addr, QualType IdxTy,
3357	QualType ArrayTy, bool Accessed,
3358	bool FlexibleArray);
3359
3360	llvm::Value *EmitScalarPrePostIncDec(const UnaryOperator *E, LValue LV,
3361	bool isInc, bool isPre);
3362	ComplexPairTy EmitComplexPrePostIncDec(const UnaryOperator *E, LValue LV,
3363	bool isInc, bool isPre);
3364
3365	/// Converts Location to a DebugLoc, if debug information is enabled.
3366	llvm::DebugLoc SourceLocToDebugLoc(SourceLocation Location);
3367
3368	/// Get the record field index as represented in debug info.
3369	unsigned getDebugInfoFIndex(const RecordDecl *Rec, unsigned FieldIndex);
3370
3371	//===--------------------------------------------------------------------===//
3372	// Declaration Emission
3373	//===--------------------------------------------------------------------===//
3374
3375	/// EmitDecl - Emit a declaration.
3376	///
3377	/// This function can be called with a null (unreachable) insert point.
3378	void EmitDecl(const Decl &D, bool EvaluateConditionDecl = false);
3379
3380	/// EmitVarDecl - Emit a local variable declaration.
3381	///
3382	/// This function can be called with a null (unreachable) insert point.
3383	void EmitVarDecl(const VarDecl &D);
3384
3385	void EmitScalarInit(const Expr *init, const ValueDecl *D, LValue lvalue,
3386	bool capturedByInit);
3387
3388	typedef void SpecialInitFn(CodeGenFunction &Init, const VarDecl &D,
3389	llvm::Value *Address);
3390
3391	/// Determine whether the given initializer is trivial in the sense
3392	/// that it requires no code to be generated.
3393	bool isTrivialInitializer(const Expr *Init);
3394
3395	/// EmitAutoVarDecl - Emit an auto variable declaration.
3396	///
3397	/// This function can be called with a null (unreachable) insert point.
3398	void EmitAutoVarDecl(const VarDecl &D);
3399
3400	class AutoVarEmission {
3401	friend class CodeGenFunction;
3402
3403	const VarDecl *Variable;
3404
3405	/// The address of the alloca for languages with explicit address space
3406	/// (e.g. OpenCL) or alloca casted to generic pointer for address space
3407	/// agnostic languages (e.g. C++). Invalid if the variable was emitted
3408	/// as a global constant.
3409	Address Addr;
3410
3411	llvm::Value *NRVOFlag;
3412
3413	/// True if the variable is a __block variable that is captured by an
3414	/// escaping block.
3415	bool IsEscapingByRef;
3416
3417	/// True if the variable is of aggregate type and has a constant
3418	/// initializer.
3419	bool IsConstantAggregate;
3420
3421	/// Non-null if we should use lifetime annotations.
3422	llvm::Value *SizeForLifetimeMarkers;
3423
3424	/// Address with original alloca instruction. Invalid if the variable was
3425	/// emitted as a global constant.
3426	RawAddress AllocaAddr;
3427
3428	struct Invalid {};
3429	AutoVarEmission(Invalid)
3430	: Variable(nullptr), Addr(Address::invalid()),
3431	AllocaAddr(RawAddress::invalid()) {}
3432
3433	AutoVarEmission(const VarDecl &variable)
3434	: Variable(&variable), Addr(Address::invalid()), NRVOFlag(nullptr),
3435	IsEscapingByRef(false), IsConstantAggregate(false),
3436	SizeForLifetimeMarkers(nullptr), AllocaAddr(RawAddress::invalid()) {}
3437
3438	bool wasEmittedAsGlobal() const { return !Addr.isValid(); }
3439
3440	public:
3441	static AutoVarEmission invalid() { return AutoVarEmission(Invalid()); }
3442
3443	bool useLifetimeMarkers() const {
3444	return SizeForLifetimeMarkers != nullptr;
3445	}
3446	llvm::Value *getSizeForLifetimeMarkers() const {
3447	assert(useLifetimeMarkers());
3448	return SizeForLifetimeMarkers;
3449	}
3450
3451	/// Returns the raw, allocated address, which is not necessarily
3452	/// the address of the object itself. It is casted to default
3453	/// address space for address space agnostic languages.
3454	Address getAllocatedAddress() const { return Addr; }
3455
3456	/// Returns the address for the original alloca instruction.
3457	RawAddress getOriginalAllocatedAddress() const { return AllocaAddr; }
3458
3459	/// Returns the address of the object within this declaration.
3460	/// Note that this does not chase the forwarding pointer for
3461	/// __block decls.
3462	Address getObjectAddress(CodeGenFunction &CGF) const {
3463	if (!IsEscapingByRef)
3464	return Addr;
3465
3466	return CGF.emitBlockByrefAddress(baseAddr: Addr, V: Variable, /*forward*/ followForward: false);
3467	}
3468	};
3469	AutoVarEmission EmitAutoVarAlloca(const VarDecl &var);
3470	void EmitAutoVarInit(const AutoVarEmission &emission);
3471	void EmitAutoVarCleanups(const AutoVarEmission &emission);
3472	void emitAutoVarTypeCleanup(const AutoVarEmission &emission,
3473	QualType::DestructionKind dtorKind);
3474
3475	void MaybeEmitDeferredVarDeclInit(const VarDecl *var);
3476
3477	/// Emits the alloca and debug information for the size expressions for each
3478	/// dimension of an array. It registers the association of its (1-dimensional)
3479	/// QualTypes and size expression's debug node, so that CGDebugInfo can
3480	/// reference this node when creating the DISubrange object to describe the
3481	/// array types.
3482	void EmitAndRegisterVariableArrayDimensions(CGDebugInfo *DI, const VarDecl &D,
3483	bool EmitDebugInfo);
3484
3485	void EmitStaticVarDecl(const VarDecl &D,
3486	llvm::GlobalValue::LinkageTypes Linkage);
3487
3488	class ParamValue {
3489	union {
3490	Address Addr;
3491	llvm::Value *Value;
3492	};
3493
3494	bool IsIndirect;
3495
3496	ParamValue(llvm::Value *V) : Value(V), IsIndirect(false) {}
3497	ParamValue(Address A) : Addr(A), IsIndirect(true) {}
3498
3499	public:
3500	static ParamValue forDirect(llvm::Value *value) {
3501	return ParamValue(value);
3502	}
3503	static ParamValue forIndirect(Address addr) {
3504	assert(!addr.getAlignment().isZero());
3505	return ParamValue(addr);
3506	}
3507
3508	bool isIndirect() const { return IsIndirect; }
3509	llvm::Value *getAnyValue() const {
3510	if (!isIndirect())
3511	return Value;
3512	assert(!Addr.hasOffset() && "unexpected offset");
3513	return Addr.getBasePointer();
3514	}
3515
3516	llvm::Value *getDirectValue() const {
3517	assert(!isIndirect());
3518	return Value;
3519	}
3520
3521	Address getIndirectAddress() const {
3522	assert(isIndirect());
3523	return Addr;
3524	}
3525	};
3526
3527	/// EmitParmDecl - Emit a ParmVarDecl or an ImplicitParamDecl.
3528	void EmitParmDecl(const VarDecl &D, ParamValue Arg, unsigned ArgNo);
3529
3530	/// protectFromPeepholes - Protect a value that we're intending to
3531	/// store to the side, but which will probably be used later, from
3532	/// aggressive peepholing optimizations that might delete it.
3533	///
3534	/// Pass the result to unprotectFromPeepholes to declare that
3535	/// protection is no longer required.
3536	///
3537	/// There's no particular reason why this shouldn't apply to
3538	/// l-values, it's just that no existing peepholes work on pointers.
3539	PeepholeProtection protectFromPeepholes(RValue rvalue);
3540	void unprotectFromPeepholes(PeepholeProtection protection);
3541
3542	void emitAlignmentAssumptionCheck(llvm::Value *Ptr, QualType Ty,
3543	SourceLocation Loc,
3544	SourceLocation AssumptionLoc,
3545	llvm::Value *Alignment,
3546	llvm::Value *OffsetValue,
3547	llvm::Value *TheCheck,
3548	llvm::Instruction *Assumption);
3549
3550	void emitAlignmentAssumption(llvm::Value *PtrValue, QualType Ty,
3551	SourceLocation Loc, SourceLocation AssumptionLoc,
3552	llvm::Value *Alignment,
3553	llvm::Value *OffsetValue = nullptr);
3554
3555	void emitAlignmentAssumption(llvm::Value *PtrValue, const Expr *E,
3556	SourceLocation AssumptionLoc,
3557	llvm::Value *Alignment,
3558	llvm::Value *OffsetValue = nullptr);
3559
3560	//===--------------------------------------------------------------------===//
3561	// Statement Emission
3562	//===--------------------------------------------------------------------===//
3563
3564	/// EmitStopPoint - Emit a debug stoppoint if we are emitting debug info.
3565	void EmitStopPoint(const Stmt *S);
3566
3567	/// EmitStmt - Emit the code for the statement \arg S. It is legal to call
3568	/// this function even if there is no current insertion point.
3569	///
3570	/// This function may clear the current insertion point; callers should use
3571	/// EnsureInsertPoint if they wish to subsequently generate code without first
3572	/// calling EmitBlock, EmitBranch, or EmitStmt.
3573	void EmitStmt(const Stmt *S, ArrayRef<const Attr *> Attrs = {});
3574
3575	/// EmitSimpleStmt - Try to emit a "simple" statement which does not
3576	/// necessarily require an insertion point or debug information; typically
3577	/// because the statement amounts to a jump or a container of other
3578	/// statements.
3579	///
3580	/// \return True if the statement was handled.
3581	bool EmitSimpleStmt(const Stmt *S, ArrayRef<const Attr *> Attrs);
3582
3583	Address EmitCompoundStmt(const CompoundStmt &S, bool GetLast = false,
3584	AggValueSlot AVS = AggValueSlot::ignored());
3585	Address
3586	EmitCompoundStmtWithoutScope(const CompoundStmt &S, bool GetLast = false,
3587	AggValueSlot AVS = AggValueSlot::ignored());
3588
3589	/// EmitLabel - Emit the block for the given label. It is legal to call this
3590	/// function even if there is no current insertion point.
3591	void EmitLabel(const LabelDecl *D); // helper for EmitLabelStmt.
3592
3593	void EmitLabelStmt(const LabelStmt &S);
3594	void EmitAttributedStmt(const AttributedStmt &S);
3595	void EmitGotoStmt(const GotoStmt &S);
3596	void EmitIndirectGotoStmt(const IndirectGotoStmt &S);
3597	void EmitIfStmt(const IfStmt &S);
3598
3599	void EmitWhileStmt(const WhileStmt &S, ArrayRef<const Attr *> Attrs = {});
3600	void EmitDoStmt(const DoStmt &S, ArrayRef<const Attr *> Attrs = {});
3601	void EmitForStmt(const ForStmt &S, ArrayRef<const Attr *> Attrs = {});
3602	void EmitReturnStmt(const ReturnStmt &S);
3603	void EmitDeclStmt(const DeclStmt &S);
3604	void EmitBreakStmt(const BreakStmt &S);
3605	void EmitContinueStmt(const ContinueStmt &S);
3606	void EmitSwitchStmt(const SwitchStmt &S);
3607	void EmitDefaultStmt(const DefaultStmt &S, ArrayRef<const Attr *> Attrs);
3608	void EmitCaseStmt(const CaseStmt &S, ArrayRef<const Attr *> Attrs);
3609	void EmitCaseStmtRange(const CaseStmt &S, ArrayRef<const Attr *> Attrs);
3610	void EmitAsmStmt(const AsmStmt &S);
3611
3612	void EmitObjCForCollectionStmt(const ObjCForCollectionStmt &S);
3613	void EmitObjCAtTryStmt(const ObjCAtTryStmt &S);
3614	void EmitObjCAtThrowStmt(const ObjCAtThrowStmt &S);
3615	void EmitObjCAtSynchronizedStmt(const ObjCAtSynchronizedStmt &S);
3616	void EmitObjCAutoreleasePoolStmt(const ObjCAutoreleasePoolStmt &S);
3617
3618	void EmitCoroutineBody(const CoroutineBodyStmt &S);
3619	void EmitCoreturnStmt(const CoreturnStmt &S);
3620	RValue EmitCoawaitExpr(const CoawaitExpr &E,
3621	AggValueSlot aggSlot = AggValueSlot::ignored(),
3622	bool ignoreResult = false);
3623	LValue EmitCoawaitLValue(const CoawaitExpr *E);
3624	RValue EmitCoyieldExpr(const CoyieldExpr &E,
3625	AggValueSlot aggSlot = AggValueSlot::ignored(),
3626	bool ignoreResult = false);
3627	LValue EmitCoyieldLValue(const CoyieldExpr *E);
3628	RValue EmitCoroutineIntrinsic(const CallExpr *E, unsigned int IID);
3629
3630	void EnterCXXTryStmt(const CXXTryStmt &S, bool IsFnTryBlock = false);
3631	void ExitCXXTryStmt(const CXXTryStmt &S, bool IsFnTryBlock = false);
3632
3633	void EmitCXXTryStmt(const CXXTryStmt &S);
3634	void EmitSEHTryStmt(const SEHTryStmt &S);
3635	void EmitSEHLeaveStmt(const SEHLeaveStmt &S);
3636	void EnterSEHTryStmt(const SEHTryStmt &S);
3637	void ExitSEHTryStmt(const SEHTryStmt &S);
3638	void VolatilizeTryBlocks(llvm::BasicBlock *BB,
3639	llvm::SmallPtrSet<llvm::BasicBlock *, 10> &V);
3640
3641	void pushSEHCleanup(CleanupKind kind, llvm::Function *FinallyFunc);
3642	void startOutlinedSEHHelper(CodeGenFunction &ParentCGF, bool IsFilter,
3643	const Stmt *OutlinedStmt);
3644
3645	llvm::Function *GenerateSEHFilterFunction(CodeGenFunction &ParentCGF,
3646	const SEHExceptStmt &Except);
3647
3648	llvm::Function *GenerateSEHFinallyFunction(CodeGenFunction &ParentCGF,
3649	const SEHFinallyStmt &Finally);
3650
3651	void EmitSEHExceptionCodeSave(CodeGenFunction &ParentCGF,
3652	llvm::Value *ParentFP, llvm::Value *EntryEBP);
3653	llvm::Value *EmitSEHExceptionCode();
3654	llvm::Value *EmitSEHExceptionInfo();
3655	llvm::Value *EmitSEHAbnormalTermination();
3656
3657	/// Emit simple code for OpenMP directives in Simd-only mode.
3658	void EmitSimpleOMPExecutableDirective(const OMPExecutableDirective &D);
3659
3660	/// Scan the outlined statement for captures from the parent function. For
3661	/// each capture, mark the capture as escaped and emit a call to
3662	/// llvm.localrecover. Insert the localrecover result into the LocalDeclMap.
3663	void EmitCapturedLocals(CodeGenFunction &ParentCGF, const Stmt *OutlinedStmt,
3664	bool IsFilter);
3665
3666	/// Recovers the address of a local in a parent function. ParentVar is the
3667	/// address of the variable used in the immediate parent function. It can
3668	/// either be an alloca or a call to llvm.localrecover if there are nested
3669	/// outlined functions. ParentFP is the frame pointer of the outermost parent
3670	/// frame.
3671	Address recoverAddrOfEscapedLocal(CodeGenFunction &ParentCGF,
3672	Address ParentVar, llvm::Value *ParentFP);
3673
3674	void EmitCXXForRangeStmt(const CXXForRangeStmt &S,
3675	ArrayRef<const Attr *> Attrs = {});
3676
3677	/// Controls insertion of cancellation exit blocks in worksharing constructs.
3678	class OMPCancelStackRAII {
3679	CodeGenFunction &CGF;
3680
3681	public:
3682	OMPCancelStackRAII(CodeGenFunction &CGF, OpenMPDirectiveKind Kind,
3683	bool HasCancel)
3684	: CGF(CGF) {
3685	CGF.OMPCancelStack.enter(CGF, Kind, HasCancel);
3686	}
3687	~OMPCancelStackRAII() { CGF.OMPCancelStack.exit(CGF); }
3688	};
3689
3690	/// Returns calculated size of the specified type.
3691	llvm::Value *getTypeSize(QualType Ty);
3692	LValue InitCapturedStruct(const CapturedStmt &S);
3693	llvm::Function *EmitCapturedStmt(const CapturedStmt &S, CapturedRegionKind K);
3694	llvm::Function *GenerateCapturedStmtFunction(const CapturedStmt &S);
3695	Address GenerateCapturedStmtArgument(const CapturedStmt &S);
3696	llvm::Function *GenerateOpenMPCapturedStmtFunction(const CapturedStmt &S,
3697	SourceLocation Loc);
3698	void GenerateOpenMPCapturedVars(const CapturedStmt &S,
3699	SmallVectorImpl<llvm::Value *> &CapturedVars);
3700	void emitOMPSimpleStore(LValue LVal, RValue RVal, QualType RValTy,
3701	SourceLocation Loc);
3702	/// Perform element by element copying of arrays with type \a
3703	/// OriginalType from \a SrcAddr to \a DestAddr using copying procedure
3704	/// generated by \a CopyGen.
3705	///
3706	/// \param DestAddr Address of the destination array.
3707	/// \param SrcAddr Address of the source array.
3708	/// \param OriginalType Type of destination and source arrays.
3709	/// \param CopyGen Copying procedure that copies value of single array element
3710	/// to another single array element.
3711	void EmitOMPAggregateAssign(
3712	Address DestAddr, Address SrcAddr, QualType OriginalType,
3713	const llvm::function_ref<void(Address, Address)> CopyGen);
3714	/// Emit proper copying of data from one variable to another.
3715	///
3716	/// \param OriginalType Original type of the copied variables.
3717	/// \param DestAddr Destination address.
3718	/// \param SrcAddr Source address.
3719	/// \param DestVD Destination variable used in \a CopyExpr (for arrays, has
3720	/// type of the base array element).
3721	/// \param SrcVD Source variable used in \a CopyExpr (for arrays, has type of
3722	/// the base array element).
3723	/// \param Copy Actual copygin expression for copying data from \a SrcVD to \a
3724	/// DestVD.
3725	void EmitOMPCopy(QualType OriginalType, Address DestAddr, Address SrcAddr,
3726	const VarDecl *DestVD, const VarDecl *SrcVD,
3727	const Expr *Copy);
3728	/// Emit atomic update code for constructs: \a X = \a X \a BO \a E or
3729	/// \a X = \a E \a BO \a E.
3730	///
3731	/// \param X Value to be updated.
3732	/// \param E Update value.
3733	/// \param BO Binary operation for update operation.
3734	/// \param IsXLHSInRHSPart true if \a X is LHS in RHS part of the update
3735	/// expression, false otherwise.
3736	/// \param AO Atomic ordering of the generated atomic instructions.
3737	/// \param CommonGen Code generator for complex expressions that cannot be
3738	/// expressed through atomicrmw instruction.
3739	/// \returns <true, OldAtomicValue> if simple 'atomicrmw' instruction was
3740	/// generated, <false, RValue::get(nullptr)> otherwise.
3741	std::pair<bool, RValue> EmitOMPAtomicSimpleUpdateExpr(
3742	LValue X, RValue E, BinaryOperatorKind BO, bool IsXLHSInRHSPart,
3743	llvm::AtomicOrdering AO, SourceLocation Loc,
3744	const llvm::function_ref<RValue(RValue)> CommonGen);
3745	bool EmitOMPFirstprivateClause(const OMPExecutableDirective &D,
3746	OMPPrivateScope &PrivateScope);
3747	void EmitOMPPrivateClause(const OMPExecutableDirective &D,
3748	OMPPrivateScope &PrivateScope);
3749	void EmitOMPUseDevicePtrClause(
3750	const OMPUseDevicePtrClause &C, OMPPrivateScope &PrivateScope,
3751	const llvm::DenseMap<const ValueDecl *, llvm::Value *>
3752	CaptureDeviceAddrMap);
3753	void EmitOMPUseDeviceAddrClause(
3754	const OMPUseDeviceAddrClause &C, OMPPrivateScope &PrivateScope,
3755	const llvm::DenseMap<const ValueDecl *, llvm::Value *>
3756	CaptureDeviceAddrMap);
3757	/// Emit code for copyin clause in \a D directive. The next code is
3758	/// generated at the start of outlined functions for directives:
3759	/// \code
3760	/// threadprivate_var1 = master_threadprivate_var1;
3761	/// operator=(threadprivate_var2, master_threadprivate_var2);
3762	/// ...
3763	/// __kmpc_barrier(&loc, global_tid);
3764	/// \endcode
3765	///
3766	/// \param D OpenMP directive possibly with 'copyin' clause(s).
3767	/// \returns true if at least one copyin variable is found, false otherwise.
3768	bool EmitOMPCopyinClause(const OMPExecutableDirective &D);
3769	/// Emit initial code for lastprivate variables. If some variable is
3770	/// not also firstprivate, then the default initialization is used. Otherwise
3771	/// initialization of this variable is performed by EmitOMPFirstprivateClause
3772	/// method.
3773	///
3774	/// \param D Directive that may have 'lastprivate' directives.
3775	/// \param PrivateScope Private scope for capturing lastprivate variables for
3776	/// proper codegen in internal captured statement.
3777	///
3778	/// \returns true if there is at least one lastprivate variable, false
3779	/// otherwise.
3780	bool EmitOMPLastprivateClauseInit(const OMPExecutableDirective &D,
3781	OMPPrivateScope &PrivateScope);
3782	/// Emit final copying of lastprivate values to original variables at
3783	/// the end of the worksharing or simd directive.
3784	///
3785	/// \param D Directive that has at least one 'lastprivate' directives.
3786	/// \param IsLastIterCond Boolean condition that must be set to 'i1 true' if
3787	/// it is the last iteration of the loop code in associated directive, or to
3788	/// 'i1 false' otherwise. If this item is nullptr, no final check is required.
3789	void EmitOMPLastprivateClauseFinal(const OMPExecutableDirective &D,
3790	bool NoFinals,
3791	llvm::Value *IsLastIterCond = nullptr);
3792	/// Emit initial code for linear clauses.
3793	void EmitOMPLinearClause(const OMPLoopDirective &D,
3794	CodeGenFunction::OMPPrivateScope &PrivateScope);
3795	/// Emit final code for linear clauses.
3796	/// \param CondGen Optional conditional code for final part of codegen for
3797	/// linear clause.
3798	void EmitOMPLinearClauseFinal(
3799	const OMPLoopDirective &D,
3800	const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen);
3801	/// Emit initial code for reduction variables. Creates reduction copies
3802	/// and initializes them with the values according to OpenMP standard.
3803	///
3804	/// \param D Directive (possibly) with the 'reduction' clause.
3805	/// \param PrivateScope Private scope for capturing reduction variables for
3806	/// proper codegen in internal captured statement.
3807	///
3808	void EmitOMPReductionClauseInit(const OMPExecutableDirective &D,
3809	OMPPrivateScope &PrivateScope,
3810	bool ForInscan = false);
3811	/// Emit final update of reduction values to original variables at
3812	/// the end of the directive.
3813	///
3814	/// \param D Directive that has at least one 'reduction' directives.
3815	/// \param ReductionKind The kind of reduction to perform.
3816	void EmitOMPReductionClauseFinal(const OMPExecutableDirective &D,
3817	const OpenMPDirectiveKind ReductionKind);
3818	/// Emit initial code for linear variables. Creates private copies
3819	/// and initializes them with the values according to OpenMP standard.
3820	///
3821	/// \param D Directive (possibly) with the 'linear' clause.
3822	/// \return true if at least one linear variable is found that should be
3823	/// initialized with the value of the original variable, false otherwise.
3824	bool EmitOMPLinearClauseInit(const OMPLoopDirective &D);
3825
3826	typedef const llvm::function_ref<void(CodeGenFunction & /*CGF*/,
3827	llvm::Function * /*OutlinedFn*/,
3828	const OMPTaskDataTy & /*Data*/)>
3829	TaskGenTy;
3830	void EmitOMPTaskBasedDirective(const OMPExecutableDirective &S,
3831	const OpenMPDirectiveKind CapturedRegion,
3832	const RegionCodeGenTy &BodyGen,
3833	const TaskGenTy &TaskGen, OMPTaskDataTy &Data);
3834	struct OMPTargetDataInfo {
3835	Address BasePointersArray = Address::invalid();
3836	Address PointersArray = Address::invalid();
3837	Address SizesArray = Address::invalid();
3838	Address MappersArray = Address::invalid();
3839	unsigned NumberOfTargetItems = 0;
3840	explicit OMPTargetDataInfo() = default;
3841	OMPTargetDataInfo(Address BasePointersArray, Address PointersArray,
3842	Address SizesArray, Address MappersArray,
3843	unsigned NumberOfTargetItems)
3844	: BasePointersArray(BasePointersArray), PointersArray(PointersArray),
3845	SizesArray(SizesArray), MappersArray(MappersArray),
3846	NumberOfTargetItems(NumberOfTargetItems) {}
3847	};
3848	void EmitOMPTargetTaskBasedDirective(const OMPExecutableDirective &S,
3849	const RegionCodeGenTy &BodyGen,
3850	OMPTargetDataInfo &InputInfo);
3851	void processInReduction(const OMPExecutableDirective &S, OMPTaskDataTy &Data,
3852	CodeGenFunction &CGF, const CapturedStmt *CS,
3853	OMPPrivateScope &Scope);
3854	void EmitOMPMetaDirective(const OMPMetaDirective &S);
3855	void EmitOMPParallelDirective(const OMPParallelDirective &S);
3856	void EmitOMPSimdDirective(const OMPSimdDirective &S);
3857	void EmitOMPTileDirective(const OMPTileDirective &S);
3858	void EmitOMPStripeDirective(const OMPStripeDirective &S);
3859	void EmitOMPUnrollDirective(const OMPUnrollDirective &S);
3860	void EmitOMPReverseDirective(const OMPReverseDirective &S);
3861	void EmitOMPInterchangeDirective(const OMPInterchangeDirective &S);
3862	void EmitOMPForDirective(const OMPForDirective &S);
3863	void EmitOMPForSimdDirective(const OMPForSimdDirective &S);
3864	void EmitOMPScopeDirective(const OMPScopeDirective &S);
3865	void EmitOMPSectionsDirective(const OMPSectionsDirective &S);
3866	void EmitOMPSectionDirective(const OMPSectionDirective &S);
3867	void EmitOMPSingleDirective(const OMPSingleDirective &S);
3868	void EmitOMPMasterDirective(const OMPMasterDirective &S);
3869	void EmitOMPMaskedDirective(const OMPMaskedDirective &S);
3870	void EmitOMPCriticalDirective(const OMPCriticalDirective &S);
3871	void EmitOMPParallelForDirective(const OMPParallelForDirective &S);
3872	void EmitOMPParallelForSimdDirective(const OMPParallelForSimdDirective &S);
3873	void EmitOMPParallelSectionsDirective(const OMPParallelSectionsDirective &S);
3874	void EmitOMPParallelMasterDirective(const OMPParallelMasterDirective &S);
3875	void EmitOMPTaskDirective(const OMPTaskDirective &S);
3876	void EmitOMPTaskyieldDirective(const OMPTaskyieldDirective &S);
3877	void EmitOMPErrorDirective(const OMPErrorDirective &S);
3878	void EmitOMPBarrierDirective(const OMPBarrierDirective &S);
3879	void EmitOMPTaskwaitDirective(const OMPTaskwaitDirective &S);
3880	void EmitOMPTaskgroupDirective(const OMPTaskgroupDirective &S);
3881	void EmitOMPFlushDirective(const OMPFlushDirective &S);
3882	void EmitOMPDepobjDirective(const OMPDepobjDirective &S);
3883	void EmitOMPScanDirective(const OMPScanDirective &S);
3884	void EmitOMPOrderedDirective(const OMPOrderedDirective &S);
3885	void EmitOMPAtomicDirective(const OMPAtomicDirective &S);
3886	void EmitOMPTargetDirective(const OMPTargetDirective &S);
3887	void EmitOMPTargetDataDirective(const OMPTargetDataDirective &S);
3888	void EmitOMPTargetEnterDataDirective(const OMPTargetEnterDataDirective &S);
3889	void EmitOMPTargetExitDataDirective(const OMPTargetExitDataDirective &S);
3890	void EmitOMPTargetUpdateDirective(const OMPTargetUpdateDirective &S);
3891	void EmitOMPTargetParallelDirective(const OMPTargetParallelDirective &S);
3892	void
3893	EmitOMPTargetParallelForDirective(const OMPTargetParallelForDirective &S);
3894	void EmitOMPTeamsDirective(const OMPTeamsDirective &S);
3895	void
3896	EmitOMPCancellationPointDirective(const OMPCancellationPointDirective &S);
3897	void EmitOMPCancelDirective(const OMPCancelDirective &S);
3898	void EmitOMPTaskLoopBasedDirective(const OMPLoopDirective &S);
3899	void EmitOMPTaskLoopDirective(const OMPTaskLoopDirective &S);
3900	void EmitOMPTaskLoopSimdDirective(const OMPTaskLoopSimdDirective &S);
3901	void EmitOMPMasterTaskLoopDirective(const OMPMasterTaskLoopDirective &S);
3902	void EmitOMPMaskedTaskLoopDirective(const OMPMaskedTaskLoopDirective &S);
3903	void
3904	EmitOMPMasterTaskLoopSimdDirective(const OMPMasterTaskLoopSimdDirective &S);
3905	void
3906	EmitOMPMaskedTaskLoopSimdDirective(const OMPMaskedTaskLoopSimdDirective &S);
3907	void EmitOMPParallelMasterTaskLoopDirective(
3908	const OMPParallelMasterTaskLoopDirective &S);
3909	void EmitOMPParallelMaskedTaskLoopDirective(
3910	const OMPParallelMaskedTaskLoopDirective &S);
3911	void EmitOMPParallelMasterTaskLoopSimdDirective(
3912	const OMPParallelMasterTaskLoopSimdDirective &S);
3913	void EmitOMPParallelMaskedTaskLoopSimdDirective(
3914	const OMPParallelMaskedTaskLoopSimdDirective &S);
3915	void EmitOMPDistributeDirective(const OMPDistributeDirective &S);
3916	void EmitOMPDistributeParallelForDirective(
3917	const OMPDistributeParallelForDirective &S);
3918	void EmitOMPDistributeParallelForSimdDirective(
3919	const OMPDistributeParallelForSimdDirective &S);
3920	void EmitOMPDistributeSimdDirective(const OMPDistributeSimdDirective &S);
3921	void EmitOMPTargetParallelForSimdDirective(
3922	const OMPTargetParallelForSimdDirective &S);
3923	void EmitOMPTargetSimdDirective(const OMPTargetSimdDirective &S);
3924	void EmitOMPTeamsDistributeDirective(const OMPTeamsDistributeDirective &S);
3925	void
3926	EmitOMPTeamsDistributeSimdDirective(const OMPTeamsDistributeSimdDirective &S);
3927	void EmitOMPTeamsDistributeParallelForSimdDirective(
3928	const OMPTeamsDistributeParallelForSimdDirective &S);
3929	void EmitOMPTeamsDistributeParallelForDirective(
3930	const OMPTeamsDistributeParallelForDirective &S);
3931	void EmitOMPTargetTeamsDirective(const OMPTargetTeamsDirective &S);
3932	void EmitOMPTargetTeamsDistributeDirective(
3933	const OMPTargetTeamsDistributeDirective &S);
3934	void EmitOMPTargetTeamsDistributeParallelForDirective(
3935	const OMPTargetTeamsDistributeParallelForDirective &S);
3936	void EmitOMPTargetTeamsDistributeParallelForSimdDirective(
3937	const OMPTargetTeamsDistributeParallelForSimdDirective &S);
3938	void EmitOMPTargetTeamsDistributeSimdDirective(
3939	const OMPTargetTeamsDistributeSimdDirective &S);
3940	void EmitOMPGenericLoopDirective(const OMPGenericLoopDirective &S);
3941	void EmitOMPParallelGenericLoopDirective(const OMPLoopDirective &S);
3942	void EmitOMPTargetParallelGenericLoopDirective(
3943	const OMPTargetParallelGenericLoopDirective &S);
3944	void EmitOMPTargetTeamsGenericLoopDirective(
3945	const OMPTargetTeamsGenericLoopDirective &S);
3946	void EmitOMPTeamsGenericLoopDirective(const OMPTeamsGenericLoopDirective &S);
3947	void EmitOMPInteropDirective(const OMPInteropDirective &S);
3948	void EmitOMPParallelMaskedDirective(const OMPParallelMaskedDirective &S);
3949	void EmitOMPAssumeDirective(const OMPAssumeDirective &S);
3950
3951	/// Emit device code for the target directive.
3952	static void EmitOMPTargetDeviceFunction(CodeGenModule &CGM,
3953	StringRef ParentName,
3954	const OMPTargetDirective &S);
3955	static void
3956	EmitOMPTargetParallelDeviceFunction(CodeGenModule &CGM, StringRef ParentName,
3957	const OMPTargetParallelDirective &S);
3958	/// Emit device code for the target parallel for directive.
3959	static void EmitOMPTargetParallelForDeviceFunction(
3960	CodeGenModule &CGM, StringRef ParentName,
3961	const OMPTargetParallelForDirective &S);
3962	/// Emit device code for the target parallel for simd directive.
3963	static void EmitOMPTargetParallelForSimdDeviceFunction(
3964	CodeGenModule &CGM, StringRef ParentName,
3965	const OMPTargetParallelForSimdDirective &S);
3966	/// Emit device code for the target teams directive.
3967	static void
3968	EmitOMPTargetTeamsDeviceFunction(CodeGenModule &CGM, StringRef ParentName,
3969	const OMPTargetTeamsDirective &S);
3970	/// Emit device code for the target teams distribute directive.
3971	static void EmitOMPTargetTeamsDistributeDeviceFunction(
3972	CodeGenModule &CGM, StringRef ParentName,
3973	const OMPTargetTeamsDistributeDirective &S);
3974	/// Emit device code for the target teams distribute simd directive.
3975	static void EmitOMPTargetTeamsDistributeSimdDeviceFunction(
3976	CodeGenModule &CGM, StringRef ParentName,
3977	const OMPTargetTeamsDistributeSimdDirective &S);
3978	/// Emit device code for the target simd directive.
3979	static void EmitOMPTargetSimdDeviceFunction(CodeGenModule &CGM,
3980	StringRef ParentName,
3981	const OMPTargetSimdDirective &S);
3982	/// Emit device code for the target teams distribute parallel for simd
3983	/// directive.
3984	static void EmitOMPTargetTeamsDistributeParallelForSimdDeviceFunction(
3985	CodeGenModule &CGM, StringRef ParentName,
3986	const OMPTargetTeamsDistributeParallelForSimdDirective &S);
3987
3988	/// Emit device code for the target teams loop directive.
3989	static void EmitOMPTargetTeamsGenericLoopDeviceFunction(
3990	CodeGenModule &CGM, StringRef ParentName,
3991	const OMPTargetTeamsGenericLoopDirective &S);
3992
3993	/// Emit device code for the target parallel loop directive.
3994	static void EmitOMPTargetParallelGenericLoopDeviceFunction(
3995	CodeGenModule &CGM, StringRef ParentName,
3996	const OMPTargetParallelGenericLoopDirective &S);
3997
3998	static void EmitOMPTargetTeamsDistributeParallelForDeviceFunction(
3999	CodeGenModule &CGM, StringRef ParentName,
4000	const OMPTargetTeamsDistributeParallelForDirective &S);
4001
4002	/// Emit the Stmt \p S and return its topmost canonical loop, if any.
4003	/// TODO: The \p Depth paramter is not yet implemented and must be 1. In the
4004	/// future it is meant to be the number of loops expected in the loop nests
4005	/// (usually specified by the "collapse" clause) that are collapsed to a
4006	/// single loop by this function.
4007	llvm::CanonicalLoopInfo *EmitOMPCollapsedCanonicalLoopNest(const Stmt *S,
4008	int Depth);
4009
4010	/// Emit an OMPCanonicalLoop using the OpenMPIRBuilder.
4011	void EmitOMPCanonicalLoop(const OMPCanonicalLoop *S);
4012
4013	/// Emit inner loop of the worksharing/simd construct.
4014	///
4015	/// \param S Directive, for which the inner loop must be emitted.
4016	/// \param RequiresCleanup true, if directive has some associated private
4017	/// variables.
4018	/// \param LoopCond Bollean condition for loop continuation.
4019	/// \param IncExpr Increment expression for loop control variable.
4020	/// \param BodyGen Generator for the inner body of the inner loop.
4021	/// \param PostIncGen Genrator for post-increment code (required for ordered
4022	/// loop directvies).
4023	void EmitOMPInnerLoop(
4024	const OMPExecutableDirective &S, bool RequiresCleanup,
4025	const Expr *LoopCond, const Expr *IncExpr,
4026	const llvm::function_ref<void(CodeGenFunction &)> BodyGen,
4027	const llvm::function_ref<void(CodeGenFunction &)> PostIncGen);
4028
4029	JumpDest getOMPCancelDestination(OpenMPDirectiveKind Kind);
4030	/// Emit initial code for loop counters of loop-based directives.
4031	void EmitOMPPrivateLoopCounters(const OMPLoopDirective &S,
4032	OMPPrivateScope &LoopScope);
4033
4034	/// Helper for the OpenMP loop directives.
4035	void EmitOMPLoopBody(const OMPLoopDirective &D, JumpDest LoopExit);
4036
4037	/// Emit code for the worksharing loop-based directive.
4038	/// \return true, if this construct has any lastprivate clause, false -
4039	/// otherwise.
4040	bool EmitOMPWorksharingLoop(const OMPLoopDirective &S, Expr *EUB,
4041	const CodeGenLoopBoundsTy &CodeGenLoopBounds,
4042	const CodeGenDispatchBoundsTy &CGDispatchBounds);
4043
4044	/// Emit code for the distribute loop-based directive.
4045	void EmitOMPDistributeLoop(const OMPLoopDirective &S,
4046	const CodeGenLoopTy &CodeGenLoop, Expr *IncExpr);
4047
4048	/// Helpers for the OpenMP loop directives.
4049	void EmitOMPSimdInit(const OMPLoopDirective &D);
4050	void EmitOMPSimdFinal(
4051	const OMPLoopDirective &D,
4052	const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen);
4053
4054	/// Emits the lvalue for the expression with possibly captured variable.
4055	LValue EmitOMPSharedLValue(const Expr *E);
4056
4057	private:
4058	/// Helpers for blocks.
4059	llvm::Value *EmitBlockLiteral(const CGBlockInfo &Info);
4060
4061	/// struct with the values to be passed to the OpenMP loop-related functions
4062	struct OMPLoopArguments {
4063	/// loop lower bound
4064	Address LB = Address::invalid();
4065	/// loop upper bound
4066	Address UB = Address::invalid();
4067	/// loop stride
4068	Address ST = Address::invalid();
4069	/// isLastIteration argument for runtime functions
4070	Address IL = Address::invalid();
4071	/// Chunk value generated by sema
4072	llvm::Value *Chunk = nullptr;
4073	/// EnsureUpperBound
4074	Expr *EUB = nullptr;
4075	/// IncrementExpression
4076	Expr *IncExpr = nullptr;
4077	/// Loop initialization
4078	Expr *Init = nullptr;
4079	/// Loop exit condition
4080	Expr *Cond = nullptr;
4081	/// Update of LB after a whole chunk has been executed
4082	Expr *NextLB = nullptr;
4083	/// Update of UB after a whole chunk has been executed
4084	Expr *NextUB = nullptr;
4085	/// Distinguish between the for distribute and sections
4086	OpenMPDirectiveKind DKind = llvm::omp::OMPD_unknown;
4087	OMPLoopArguments() = default;
4088	OMPLoopArguments(Address LB, Address UB, Address ST, Address IL,
4089	llvm::Value *Chunk = nullptr, Expr *EUB = nullptr,
4090	Expr *IncExpr = nullptr, Expr *Init = nullptr,
4091	Expr *Cond = nullptr, Expr *NextLB = nullptr,
4092	Expr *NextUB = nullptr)
4093	: LB(LB), UB(UB), ST(ST), IL(IL), Chunk(Chunk), EUB(EUB),
4094	IncExpr(IncExpr), Init(Init), Cond(Cond), NextLB(NextLB),
4095	NextUB(NextUB) {}
4096	};
4097	void EmitOMPOuterLoop(bool DynamicOrOrdered, bool IsMonotonic,
4098	const OMPLoopDirective &S, OMPPrivateScope &LoopScope,
4099	const OMPLoopArguments &LoopArgs,
4100	const CodeGenLoopTy &CodeGenLoop,
4101	const CodeGenOrderedTy &CodeGenOrdered);
4102	void EmitOMPForOuterLoop(const OpenMPScheduleTy &ScheduleKind,
4103	bool IsMonotonic, const OMPLoopDirective &S,
4104	OMPPrivateScope &LoopScope, bool Ordered,
4105	const OMPLoopArguments &LoopArgs,
4106	const CodeGenDispatchBoundsTy &CGDispatchBounds);
4107	void EmitOMPDistributeOuterLoop(OpenMPDistScheduleClauseKind ScheduleKind,
4108	const OMPLoopDirective &S,
4109	OMPPrivateScope &LoopScope,
4110	const OMPLoopArguments &LoopArgs,
4111	const CodeGenLoopTy &CodeGenLoopContent);
4112	/// Emit code for sections directive.
4113	void EmitSections(const OMPExecutableDirective &S);
4114
4115	public:
4116	//===--------------------------------------------------------------------===//
4117	// OpenACC Emission
4118	//===--------------------------------------------------------------------===//
4119	void EmitOpenACCComputeConstruct(const OpenACCComputeConstruct &S) {
4120	// TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4121	// simply emitting its structured block, but in the future we will implement
4122	// some sort of IR.
4123	EmitStmt(S: S.getStructuredBlock());
4124	}
4125
4126	void EmitOpenACCLoopConstruct(const OpenACCLoopConstruct &S) {
4127	// TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4128	// simply emitting its loop, but in the future we will implement
4129	// some sort of IR.
4130	EmitStmt(S: S.getLoop());
4131	}
4132
4133	void EmitOpenACCCombinedConstruct(const OpenACCCombinedConstruct &S) {
4134	// TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4135	// simply emitting its loop, but in the future we will implement
4136	// some sort of IR.
4137	EmitStmt(S: S.getLoop());
4138	}
4139
4140	void EmitOpenACCDataConstruct(const OpenACCDataConstruct &S) {
4141	// TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4142	// simply emitting its structured block, but in the future we will implement
4143	// some sort of IR.
4144	EmitStmt(S: S.getStructuredBlock());
4145	}
4146
4147	void EmitOpenACCEnterDataConstruct(const OpenACCEnterDataConstruct &S) {
4148	// TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4149	// but in the future we will implement some sort of IR.
4150	}
4151
4152	void EmitOpenACCExitDataConstruct(const OpenACCExitDataConstruct &S) {
4153	// TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4154	// but in the future we will implement some sort of IR.
4155	}
4156
4157	void EmitOpenACCHostDataConstruct(const OpenACCHostDataConstruct &S) {
4158	// TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4159	// simply emitting its structured block, but in the future we will implement
4160	// some sort of IR.
4161	EmitStmt(S: S.getStructuredBlock());
4162	}
4163
4164	void EmitOpenACCWaitConstruct(const OpenACCWaitConstruct &S) {
4165	// TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4166	// but in the future we will implement some sort of IR.
4167	}
4168
4169	void EmitOpenACCInitConstruct(const OpenACCInitConstruct &S) {
4170	// TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4171	// but in the future we will implement some sort of IR.
4172	}
4173
4174	void EmitOpenACCShutdownConstruct(const OpenACCShutdownConstruct &S) {
4175	// TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4176	// but in the future we will implement some sort of IR.
4177	}
4178
4179	void EmitOpenACCSetConstruct(const OpenACCSetConstruct &S) {
4180	// TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4181	// but in the future we will implement some sort of IR.
4182	}
4183
4184	void EmitOpenACCUpdateConstruct(const OpenACCUpdateConstruct &S) {
4185	// TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4186	// but in the future we will implement some sort of IR.
4187	}
4188
4189	void EmitOpenACCAtomicConstruct(const OpenACCAtomicConstruct &S) {
4190	// TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4191	// simply emitting its associated stmt, but in the future we will implement
4192	// some sort of IR.
4193	EmitStmt(S: S.getAssociatedStmt());
4194	}
4195	void EmitOpenACCCacheConstruct(const OpenACCCacheConstruct &S) {
4196	// TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4197	// but in the future we will implement some sort of IR.
4198	}
4199
4200	//===--------------------------------------------------------------------===//
4201	// LValue Expression Emission
4202	//===--------------------------------------------------------------------===//
4203
4204	/// Create a check that a scalar RValue is non-null.
4205	llvm::Value *EmitNonNullRValueCheck(RValue RV, QualType T);
4206
4207	/// GetUndefRValue - Get an appropriate 'undef' rvalue for the given type.
4208	RValue GetUndefRValue(QualType Ty);
4209
4210	/// EmitUnsupportedRValue - Emit a dummy r-value using the type of E
4211	/// and issue an ErrorUnsupported style diagnostic (using the
4212	/// provided Name).
4213	RValue EmitUnsupportedRValue(const Expr *E, const char *Name);
4214
4215	/// EmitUnsupportedLValue - Emit a dummy l-value using the type of E and issue
4216	/// an ErrorUnsupported style diagnostic (using the provided Name).
4217	LValue EmitUnsupportedLValue(const Expr *E, const char *Name);
4218
4219	/// EmitLValue - Emit code to compute a designator that specifies the location
4220	/// of the expression.
4221	///
4222	/// This can return one of two things: a simple address or a bitfield
4223	/// reference. In either case, the LLVM Value* in the LValue structure is
4224	/// guaranteed to be an LLVM pointer type.
4225	///
4226	/// If this returns a bitfield reference, nothing about the pointee type of
4227	/// the LLVM value is known: For example, it may not be a pointer to an
4228	/// integer.
4229	///
4230	/// If this returns a normal address, and if the lvalue's C type is fixed
4231	/// size, this method guarantees that the returned pointer type will point to
4232	/// an LLVM type of the same size of the lvalue's type. If the lvalue has a
4233	/// variable length type, this is not possible.
4234	///
4235	LValue EmitLValue(const Expr *E,
4236	KnownNonNull_t IsKnownNonNull = NotKnownNonNull);
4237
4238	private:
4239	LValue EmitLValueHelper(const Expr *E, KnownNonNull_t IsKnownNonNull);
4240
4241	public:
4242	/// Same as EmitLValue but additionally we generate checking code to
4243	/// guard against undefined behavior. This is only suitable when we know
4244	/// that the address will be used to access the object.
4245	LValue EmitCheckedLValue(const Expr *E, TypeCheckKind TCK);
4246
4247	RValue convertTempToRValue(Address addr, QualType type, SourceLocation Loc);
4248
4249	void EmitAtomicInit(Expr *E, LValue lvalue);
4250
4251	bool LValueIsSuitableForInlineAtomic(LValue Src);
4252
4253	RValue EmitAtomicLoad(LValue LV, SourceLocation SL,
4254	AggValueSlot Slot = AggValueSlot::ignored());
4255
4256	RValue EmitAtomicLoad(LValue lvalue, SourceLocation loc,
4257	llvm::AtomicOrdering AO, bool IsVolatile = false,
4258	AggValueSlot slot = AggValueSlot::ignored());
4259
4260	void EmitAtomicStore(RValue rvalue, LValue lvalue, bool isInit);
4261
4262	void EmitAtomicStore(RValue rvalue, LValue lvalue, llvm::AtomicOrdering AO,
4263	bool IsVolatile, bool isInit);
4264
4265	std::pair<RValue, llvm::Value *> EmitAtomicCompareExchange(
4266	LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc,
4267	llvm::AtomicOrdering Success =
4268	llvm::AtomicOrdering::SequentiallyConsistent,
4269	llvm::AtomicOrdering Failure =
4270	llvm::AtomicOrdering::SequentiallyConsistent,
4271	bool IsWeak = false, AggValueSlot Slot = AggValueSlot::ignored());
4272
4273	/// Emit an atomicrmw instruction, and applying relevant metadata when
4274	/// applicable.
4275	llvm::AtomicRMWInst *emitAtomicRMWInst(
4276	llvm::AtomicRMWInst::BinOp Op, Address Addr, llvm::Value *Val,
4277	llvm::AtomicOrdering Order = llvm::AtomicOrdering::SequentiallyConsistent,
4278	llvm::SyncScope::ID SSID = llvm::SyncScope::System,
4279	const AtomicExpr *AE = nullptr);
4280
4281	void EmitAtomicUpdate(LValue LVal, llvm::AtomicOrdering AO,
4282	const llvm::function_ref<RValue(RValue)> &UpdateOp,
4283	bool IsVolatile);
4284
4285	/// EmitToMemory - Change a scalar value from its value
4286	/// representation to its in-memory representation.
4287	llvm::Value *EmitToMemory(llvm::Value *Value, QualType Ty);
4288
4289	/// EmitFromMemory - Change a scalar value from its memory
4290	/// representation to its value representation.
4291	llvm::Value *EmitFromMemory(llvm::Value *Value, QualType Ty);
4292
4293	/// Check if the scalar \p Value is within the valid range for the given
4294	/// type \p Ty.
4295	///
4296	/// Returns true if a check is needed (even if the range is unknown).
4297	bool EmitScalarRangeCheck(llvm::Value *Value, QualType Ty,
4298	SourceLocation Loc);
4299
4300	/// EmitLoadOfScalar - Load a scalar value from an address, taking
4301	/// care to appropriately convert from the memory representation to
4302	/// the LLVM value representation.
4303	llvm::Value *EmitLoadOfScalar(Address Addr, bool Volatile, QualType Ty,
4304	SourceLocation Loc,
4305	AlignmentSource Source = AlignmentSource::Type,
4306	bool isNontemporal = false) {
4307	return EmitLoadOfScalar(Addr, Volatile, Ty, Loc, BaseInfo: LValueBaseInfo(Source),
4308	TBAAInfo: CGM.getTBAAAccessInfo(AccessType: Ty), isNontemporal);
4309	}
4310
4311	llvm::Value *EmitLoadOfScalar(Address Addr, bool Volatile, QualType Ty,
4312	SourceLocation Loc, LValueBaseInfo BaseInfo,
4313	TBAAAccessInfo TBAAInfo,
4314	bool isNontemporal = false);
4315
4316	/// EmitLoadOfScalar - Load a scalar value from an address, taking
4317	/// care to appropriately convert from the memory representation to
4318	/// the LLVM value representation. The l-value must be a simple
4319	/// l-value.
4320	llvm::Value *EmitLoadOfScalar(LValue lvalue, SourceLocation Loc);
4321
4322	/// EmitStoreOfScalar - Store a scalar value to an address, taking
4323	/// care to appropriately convert from the memory representation to
4324	/// the LLVM value representation.
4325	void EmitStoreOfScalar(llvm::Value *Value, Address Addr, bool Volatile,
4326	QualType Ty,
4327	AlignmentSource Source = AlignmentSource::Type,
4328	bool isInit = false, bool isNontemporal = false) {
4329	EmitStoreOfScalar(Value, Addr, Volatile, Ty, BaseInfo: LValueBaseInfo(Source),
4330	TBAAInfo: CGM.getTBAAAccessInfo(AccessType: Ty), isInit, isNontemporal);
4331	}
4332
4333	void EmitStoreOfScalar(llvm::Value *Value, Address Addr, bool Volatile,
4334	QualType Ty, LValueBaseInfo BaseInfo,
4335	TBAAAccessInfo TBAAInfo, bool isInit = false,
4336	bool isNontemporal = false);
4337
4338	/// EmitStoreOfScalar - Store a scalar value to an address, taking
4339	/// care to appropriately convert from the memory representation to
4340	/// the LLVM value representation. The l-value must be a simple
4341	/// l-value. The isInit flag indicates whether this is an initialization.
4342	/// If so, atomic qualifiers are ignored and the store is always non-atomic.
4343	void EmitStoreOfScalar(llvm::Value *value, LValue lvalue,
4344	bool isInit = false);
4345
4346	/// EmitLoadOfLValue - Given an expression that represents a value lvalue,
4347	/// this method emits the address of the lvalue, then loads the result as an
4348	/// rvalue, returning the rvalue.
4349	RValue EmitLoadOfLValue(LValue V, SourceLocation Loc);
4350	RValue EmitLoadOfExtVectorElementLValue(LValue V);
4351	RValue EmitLoadOfBitfieldLValue(LValue LV, SourceLocation Loc);
4352	RValue EmitLoadOfGlobalRegLValue(LValue LV);
4353
4354	/// Like EmitLoadOfLValue but also handles complex and aggregate types.
4355	RValue EmitLoadOfAnyValue(LValue V,
4356	AggValueSlot Slot = AggValueSlot::ignored(),
4357	SourceLocation Loc = {});
4358
4359	/// EmitStoreThroughLValue - Store the specified rvalue into the specified
4360	/// lvalue, where both are guaranteed to the have the same type, and that type
4361	/// is 'Ty'.
4362	void EmitStoreThroughLValue(RValue Src, LValue Dst, bool isInit = false);
4363	void EmitStoreThroughExtVectorComponentLValue(RValue Src, LValue Dst);
4364	void EmitStoreThroughGlobalRegLValue(RValue Src, LValue Dst);
4365
4366	/// EmitStoreThroughBitfieldLValue - Store Src into Dst with same constraints
4367	/// as EmitStoreThroughLValue.
4368	///
4369	/// \param Result [out] - If non-null, this will be set to a Value* for the
4370	/// bit-field contents after the store, appropriate for use as the result of
4371	/// an assignment to the bit-field.
4372	void EmitStoreThroughBitfieldLValue(RValue Src, LValue Dst,
4373	llvm::Value **Result = nullptr);
4374
4375	/// Emit an l-value for an assignment (simple or compound) of complex type.
4376	LValue EmitComplexAssignmentLValue(const BinaryOperator *E);
4377	LValue EmitComplexCompoundAssignmentLValue(const CompoundAssignOperator *E);
4378	LValue EmitScalarCompoundAssignWithComplex(const CompoundAssignOperator *E,
4379	llvm::Value *&Result);
4380
4381	// Note: only available for agg return types
4382	LValue EmitBinaryOperatorLValue(const BinaryOperator *E);
4383	LValue EmitCompoundAssignmentLValue(const CompoundAssignOperator *E);
4384	// Note: only available for agg return types
4385	LValue EmitCallExprLValue(const CallExpr *E,
4386	llvm::CallBase **CallOrInvoke = nullptr);
4387	// Note: only available for agg return types
4388	LValue EmitVAArgExprLValue(const VAArgExpr *E);
4389	LValue EmitDeclRefLValue(const DeclRefExpr *E);
4390	LValue EmitStringLiteralLValue(const StringLiteral *E);
4391	LValue EmitObjCEncodeExprLValue(const ObjCEncodeExpr *E);
4392	LValue EmitPredefinedLValue(const PredefinedExpr *E);
4393	LValue EmitUnaryOpLValue(const UnaryOperator *E);
4394	LValue EmitArraySubscriptExpr(const ArraySubscriptExpr *E,
4395	bool Accessed = false);
4396	llvm::Value *EmitMatrixIndexExpr(const Expr *E);
4397	LValue EmitMatrixSubscriptExpr(const MatrixSubscriptExpr *E);
4398	LValue EmitArraySectionExpr(const ArraySectionExpr *E,
4399	bool IsLowerBound = true);
4400	LValue EmitExtVectorElementExpr(const ExtVectorElementExpr *E);
4401	LValue EmitMemberExpr(const MemberExpr *E);
4402	LValue EmitObjCIsaExpr(const ObjCIsaExpr *E);
4403	LValue EmitCompoundLiteralLValue(const CompoundLiteralExpr *E);
4404	LValue EmitInitListLValue(const InitListExpr *E);
4405	void EmitIgnoredConditionalOperator(const AbstractConditionalOperator *E);
4406	LValue EmitConditionalOperatorLValue(const AbstractConditionalOperator *E);
4407	LValue EmitCastLValue(const CastExpr *E);
4408	LValue EmitMaterializeTemporaryExpr(const MaterializeTemporaryExpr *E);
4409	LValue EmitOpaqueValueLValue(const OpaqueValueExpr *e);
4410	LValue EmitHLSLArrayAssignLValue(const BinaryOperator *E);
4411
4412	std::pair<LValue, LValue> EmitHLSLOutArgLValues(const HLSLOutArgExpr *E,
4413	QualType Ty);
4414	LValue EmitHLSLOutArgExpr(const HLSLOutArgExpr *E, CallArgList &Args,
4415	QualType Ty);
4416
4417	Address EmitExtVectorElementLValue(LValue V);
4418
4419	RValue EmitRValueForField(LValue LV, const FieldDecl *FD, SourceLocation Loc);
4420
4421	Address EmitArrayToPointerDecay(const Expr *Array,
4422	LValueBaseInfo *BaseInfo = nullptr,
4423	TBAAAccessInfo *TBAAInfo = nullptr);
4424
4425	class ConstantEmission {
4426	llvm::PointerIntPair<llvm::Constant *, 1, bool> ValueAndIsReference;
4427	ConstantEmission(llvm::Constant *C, bool isReference)
4428	: ValueAndIsReference(C, isReference) {}
4429
4430	public:
4431	ConstantEmission() {}
4432	static ConstantEmission forReference(llvm::Constant *C) {
4433	return ConstantEmission(C, true);
4434	}
4435	static ConstantEmission forValue(llvm::Constant *C) {
4436	return ConstantEmission(C, false);
4437	}
4438
4439	explicit operator bool() const {
4440	return ValueAndIsReference.getOpaqueValue() != nullptr;
4441	}
4442
4443	bool isReference() const { return ValueAndIsReference.getInt(); }
4444	LValue getReferenceLValue(CodeGenFunction &CGF, const Expr *RefExpr) const {
4445	assert(isReference());
4446	return CGF.MakeNaturalAlignAddrLValue(V: ValueAndIsReference.getPointer(),
4447	T: RefExpr->getType());
4448	}
4449
4450	llvm::Constant *getValue() const {
4451	assert(!isReference());
4452	return ValueAndIsReference.getPointer();
4453	}
4454	};
4455
4456	ConstantEmission tryEmitAsConstant(const DeclRefExpr *RefExpr);
4457	ConstantEmission tryEmitAsConstant(const MemberExpr *ME);
4458	llvm::Value *emitScalarConstant(const ConstantEmission &Constant, Expr *E);
4459
4460	RValue EmitPseudoObjectRValue(const PseudoObjectExpr *e,
4461	AggValueSlot slot = AggValueSlot::ignored());
4462	LValue EmitPseudoObjectLValue(const PseudoObjectExpr *e);
4463
4464	void FlattenAccessAndType(
4465	Address Addr, QualType AddrTy,
4466	SmallVectorImpl<std::pair<Address, llvm::Value *>> &AccessList,
4467	SmallVectorImpl<QualType> &FlatTypes);
4468
4469	llvm::Value *EmitIvarOffset(const ObjCInterfaceDecl *Interface,
4470	const ObjCIvarDecl *Ivar);
4471	llvm::Value *EmitIvarOffsetAsPointerDiff(const ObjCInterfaceDecl *Interface,
4472	const ObjCIvarDecl *Ivar);
4473	LValue EmitLValueForField(LValue Base, const FieldDecl *Field,
4474	bool IsInBounds = true);
4475	LValue EmitLValueForLambdaField(const FieldDecl *Field);
4476	LValue EmitLValueForLambdaField(const FieldDecl *Field,
4477	llvm::Value *ThisValue);
4478
4479	/// EmitLValueForFieldInitialization - Like EmitLValueForField, except that
4480	/// if the Field is a reference, this will return the address of the reference
4481	/// and not the address of the value stored in the reference.
4482	LValue EmitLValueForFieldInitialization(LValue Base, const FieldDecl *Field);
4483
4484	LValue EmitLValueForIvar(QualType ObjectTy, llvm::Value *Base,
4485	const ObjCIvarDecl *Ivar, unsigned CVRQualifiers);
4486
4487	LValue EmitCXXConstructLValue(const CXXConstructExpr *E);
4488	LValue EmitCXXBindTemporaryLValue(const CXXBindTemporaryExpr *E);
4489	LValue EmitCXXTypeidLValue(const CXXTypeidExpr *E);
4490	LValue EmitCXXUuidofLValue(const CXXUuidofExpr *E);
4491
4492	LValue EmitObjCMessageExprLValue(const ObjCMessageExpr *E);
4493	LValue EmitObjCIvarRefLValue(const ObjCIvarRefExpr *E);
4494	LValue EmitStmtExprLValue(const StmtExpr *E);
4495	LValue EmitPointerToDataMemberBinaryExpr(const BinaryOperator *E);
4496	LValue EmitObjCSelectorLValue(const ObjCSelectorExpr *E);
4497	void EmitDeclRefExprDbgValue(const DeclRefExpr *E, const APValue &Init);
4498
4499	//===--------------------------------------------------------------------===//
4500	// Scalar Expression Emission
4501	//===--------------------------------------------------------------------===//
4502
4503	/// EmitCall - Generate a call of the given function, expecting the given
4504	/// result type, and using the given argument list which specifies both the
4505	/// LLVM arguments and the types they were derived from.
4506	RValue EmitCall(const CGFunctionInfo &CallInfo, const CGCallee &Callee,
4507	ReturnValueSlot ReturnValue, const CallArgList &Args,
4508	llvm::CallBase **CallOrInvoke, bool IsMustTail,
4509	SourceLocation Loc,
4510	bool IsVirtualFunctionPointerThunk = false);
4511	RValue EmitCall(const CGFunctionInfo &CallInfo, const CGCallee &Callee,
4512	ReturnValueSlot ReturnValue, const CallArgList &Args,
4513	llvm::CallBase **CallOrInvoke = nullptr,
4514	bool IsMustTail = false) {
4515	return EmitCall(CallInfo, Callee, ReturnValue, Args, CallOrInvoke,
4516	IsMustTail, Loc: SourceLocation());
4517	}
4518	RValue EmitCall(QualType FnType, const CGCallee &Callee, const CallExpr *E,
4519	ReturnValueSlot ReturnValue, llvm::Value *Chain = nullptr,
4520	llvm::CallBase **CallOrInvoke = nullptr,
4521	CGFunctionInfo const **ResolvedFnInfo = nullptr);
4522
4523	// If a Call or Invoke instruction was emitted for this CallExpr, this method
4524	// writes the pointer to `CallOrInvoke` if it's not null.
4525	RValue EmitCallExpr(const CallExpr *E,
4526	ReturnValueSlot ReturnValue = ReturnValueSlot(),
4527	llvm::CallBase **CallOrInvoke = nullptr);
4528	RValue EmitSimpleCallExpr(const CallExpr *E, ReturnValueSlot ReturnValue,
4529	llvm::CallBase **CallOrInvoke = nullptr);
4530	CGCallee EmitCallee(const Expr *E);
4531
4532	void checkTargetFeatures(const CallExpr *E, const FunctionDecl *TargetDecl);
4533	void checkTargetFeatures(SourceLocation Loc, const FunctionDecl *TargetDecl);
4534
4535	llvm::CallInst *EmitRuntimeCall(llvm::FunctionCallee callee,
4536	const Twine &name = "");
4537	llvm::CallInst *EmitRuntimeCall(llvm::FunctionCallee callee,
4538	ArrayRef<llvm::Value *> args,
4539	const Twine &name = "");
4540	llvm::CallInst *EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
4541	const Twine &name = "");
4542	llvm::CallInst *EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
4543	ArrayRef<Address> args,
4544	const Twine &name = "");
4545	llvm::CallInst *EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
4546	ArrayRef<llvm::Value *> args,
4547	const Twine &name = "");
4548
4549	SmallVector<llvm::OperandBundleDef, 1>
4550	getBundlesForFunclet(llvm::Value *Callee);
4551
4552	llvm::CallBase *EmitCallOrInvoke(llvm::FunctionCallee Callee,
4553	ArrayRef<llvm::Value *> Args,
4554	const Twine &Name = "");
4555	llvm::CallBase *EmitRuntimeCallOrInvoke(llvm::FunctionCallee callee,
4556	ArrayRef<llvm::Value *> args,
4557	const Twine &name = "");
4558	llvm::CallBase *EmitRuntimeCallOrInvoke(llvm::FunctionCallee callee,
4559	const Twine &name = "");
4560	void EmitNoreturnRuntimeCallOrInvoke(llvm::FunctionCallee callee,
4561	ArrayRef<llvm::Value *> args);
4562
4563	CGCallee BuildAppleKextVirtualCall(const CXXMethodDecl *MD,
4564	NestedNameSpecifier *Qual, llvm::Type *Ty);
4565
4566	CGCallee BuildAppleKextVirtualDestructorCall(const CXXDestructorDecl *DD,
4567	CXXDtorType Type,
4568	const CXXRecordDecl *RD);
4569
4570	bool isPointerKnownNonNull(const Expr *E);
4571	/// Check whether the underlying base pointer is a constant null.
4572	bool isUnderlyingBasePointerConstantNull(const Expr *E);
4573
4574	/// Create the discriminator from the storage address and the entity hash.
4575	llvm::Value *EmitPointerAuthBlendDiscriminator(llvm::Value *StorageAddress,
4576	llvm::Value *Discriminator);
4577	CGPointerAuthInfo EmitPointerAuthInfo(const PointerAuthSchema &Schema,
4578	llvm::Value *StorageAddress,
4579	GlobalDecl SchemaDecl,
4580	QualType SchemaType);
4581
4582	llvm::Value *EmitPointerAuthSign(const CGPointerAuthInfo &Info,
4583	llvm::Value *Pointer);
4584
4585	llvm::Value *EmitPointerAuthAuth(const CGPointerAuthInfo &Info,
4586	llvm::Value *Pointer);
4587
4588	llvm::Value *emitPointerAuthResign(llvm::Value *Pointer, QualType PointerType,
4589	const CGPointerAuthInfo &CurAuthInfo,
4590	const CGPointerAuthInfo &NewAuthInfo,
4591	bool IsKnownNonNull);
4592	llvm::Value *emitPointerAuthResignCall(llvm::Value *Pointer,
4593	const CGPointerAuthInfo &CurInfo,
4594	const CGPointerAuthInfo &NewInfo);
4595
4596	void EmitPointerAuthOperandBundle(
4597	const CGPointerAuthInfo &Info,
4598	SmallVectorImpl<llvm::OperandBundleDef> &Bundles);
4599
4600	CGPointerAuthInfo EmitPointerAuthInfo(PointerAuthQualifier Qualifier,
4601	Address StorageAddress);
4602	llvm::Value *EmitPointerAuthQualify(PointerAuthQualifier Qualifier,
4603	llvm::Value *Pointer, QualType ValueType,
4604	Address StorageAddress,
4605	bool IsKnownNonNull);
4606	llvm::Value *EmitPointerAuthQualify(PointerAuthQualifier Qualifier,
4607	const Expr *PointerExpr,
4608	Address StorageAddress);
4609	llvm::Value *EmitPointerAuthUnqualify(PointerAuthQualifier Qualifier,
4610	llvm::Value *Pointer,
4611	QualType PointerType,
4612	Address StorageAddress,
4613	bool IsKnownNonNull);
4614	void EmitPointerAuthCopy(PointerAuthQualifier Qualifier, QualType Type,
4615	Address DestField, Address SrcField);
4616
4617	std::pair<llvm::Value *, CGPointerAuthInfo>
4618	EmitOrigPointerRValue(const Expr *E);
4619
4620	llvm::Value *authPointerToPointerCast(llvm::Value *ResultPtr,
4621	QualType SourceType, QualType DestType);
4622	Address authPointerToPointerCast(Address Ptr, QualType SourceType,
4623	QualType DestType);
4624
4625	Address getAsNaturalAddressOf(Address Addr, QualType PointeeTy);
4626
4627	llvm::Value *getAsNaturalPointerTo(Address Addr, QualType PointeeType) {
4628	return getAsNaturalAddressOf(Addr, PointeeTy: PointeeType).getBasePointer();
4629	}
4630
4631	// Return the copy constructor name with the prefix "__copy_constructor_"
4632	// removed.
4633	static std::string getNonTrivialCopyConstructorStr(QualType QT,
4634	CharUnits Alignment,
4635	bool IsVolatile,
4636	ASTContext &Ctx);
4637
4638	// Return the destructor name with the prefix "__destructor_" removed.
4639	static std::string getNonTrivialDestructorStr(QualType QT,
4640	CharUnits Alignment,
4641	bool IsVolatile,
4642	ASTContext &Ctx);
4643
4644	// These functions emit calls to the special functions of non-trivial C
4645	// structs.
4646	void defaultInitNonTrivialCStructVar(LValue Dst);
4647	void callCStructDefaultConstructor(LValue Dst);
4648	void callCStructDestructor(LValue Dst);
4649	void callCStructCopyConstructor(LValue Dst, LValue Src);
4650	void callCStructMoveConstructor(LValue Dst, LValue Src);
4651	void callCStructCopyAssignmentOperator(LValue Dst, LValue Src);
4652	void callCStructMoveAssignmentOperator(LValue Dst, LValue Src);
4653
4654	RValue EmitCXXMemberOrOperatorCall(
4655	const CXXMethodDecl *Method, const CGCallee &Callee,
4656	ReturnValueSlot ReturnValue, llvm::Value *This,
4657	llvm::Value *ImplicitParam, QualType ImplicitParamTy, const CallExpr *E,
4658	CallArgList *RtlArgs, llvm::CallBase **CallOrInvoke);
4659	RValue EmitCXXDestructorCall(GlobalDecl Dtor, const CGCallee &Callee,
4660	llvm::Value *This, QualType ThisTy,
4661	llvm::Value *ImplicitParam,
4662	QualType ImplicitParamTy, const CallExpr *E,
4663	llvm::CallBase **CallOrInvoke = nullptr);
4664	RValue EmitCXXMemberCallExpr(const CXXMemberCallExpr *E,
4665	ReturnValueSlot ReturnValue,
4666	llvm::CallBase **CallOrInvoke = nullptr);
4667	RValue EmitCXXMemberOrOperatorMemberCallExpr(
4668	const CallExpr *CE, const CXXMethodDecl *MD, ReturnValueSlot ReturnValue,
4669	bool HasQualifier, NestedNameSpecifier *Qualifier, bool IsArrow,
4670	const Expr *Base, llvm::CallBase **CallOrInvoke);
4671	// Compute the object pointer.
4672	Address EmitCXXMemberDataPointerAddress(
4673	const Expr *E, Address base, llvm::Value *memberPtr,
4674	const MemberPointerType *memberPtrType, bool IsInBounds,
4675	LValueBaseInfo *BaseInfo = nullptr, TBAAAccessInfo *TBAAInfo = nullptr);
4676	RValue EmitCXXMemberPointerCallExpr(const CXXMemberCallExpr *E,
4677	ReturnValueSlot ReturnValue,
4678	llvm::CallBase **CallOrInvoke);
4679
4680	RValue EmitCXXOperatorMemberCallExpr(const CXXOperatorCallExpr *E,
4681	const CXXMethodDecl *MD,
4682	ReturnValueSlot ReturnValue,
4683	llvm::CallBase **CallOrInvoke);
4684	RValue EmitCXXPseudoDestructorExpr(const CXXPseudoDestructorExpr *E);
4685
4686	RValue EmitCUDAKernelCallExpr(const CUDAKernelCallExpr *E,
4687	ReturnValueSlot ReturnValue,
4688	llvm::CallBase **CallOrInvoke);
4689
4690	RValue EmitNVPTXDevicePrintfCallExpr(const CallExpr *E);
4691	RValue EmitAMDGPUDevicePrintfCallExpr(const CallExpr *E);
4692
4693	RValue EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
4694	const CallExpr *E, ReturnValueSlot ReturnValue);
4695
4696	RValue emitRotate(const CallExpr *E, bool IsRotateRight);
4697
4698	/// Emit IR for __builtin_os_log_format.
4699	RValue emitBuiltinOSLogFormat(const CallExpr &E);
4700
4701	/// Emit IR for __builtin_is_aligned.
4702	RValue EmitBuiltinIsAligned(const CallExpr *E);
4703	/// Emit IR for __builtin_align_up/__builtin_align_down.
4704	RValue EmitBuiltinAlignTo(const CallExpr *E, bool AlignUp);
4705
4706	llvm::Function *generateBuiltinOSLogHelperFunction(
4707	const analyze_os_log::OSLogBufferLayout &Layout,
4708	CharUnits BufferAlignment);
4709
4710	RValue EmitBlockCallExpr(const CallExpr *E, ReturnValueSlot ReturnValue,
4711	llvm::CallBase **CallOrInvoke);
4712
4713	/// EmitTargetBuiltinExpr - Emit the given builtin call. Returns 0 if the call
4714	/// is unhandled by the current target.
4715	llvm::Value *EmitTargetBuiltinExpr(unsigned BuiltinID, const CallExpr *E,
4716	ReturnValueSlot ReturnValue);
4717
4718	llvm::Value *
4719	EmitAArch64CompareBuiltinExpr(llvm::Value *Op, llvm::Type *Ty,
4720	const llvm::CmpInst::Predicate Pred,
4721	const llvm::Twine &Name = "");
4722	llvm::Value *EmitARMBuiltinExpr(unsigned BuiltinID, const CallExpr *E,
4723	ReturnValueSlot ReturnValue,
4724	llvm::Triple::ArchType Arch);
4725	llvm::Value *EmitARMMVEBuiltinExpr(unsigned BuiltinID, const CallExpr *E,
4726	ReturnValueSlot ReturnValue,
4727	llvm::Triple::ArchType Arch);
4728	llvm::Value *EmitARMCDEBuiltinExpr(unsigned BuiltinID, const CallExpr *E,
4729	ReturnValueSlot ReturnValue,
4730	llvm::Triple::ArchType Arch);
4731	llvm::Value *EmitCMSEClearRecord(llvm::Value *V, llvm::IntegerType *ITy,
4732	QualType RTy);
4733	llvm::Value *EmitCMSEClearRecord(llvm::Value *V, llvm::ArrayType *ATy,
4734	QualType RTy);
4735
4736	llvm::Value *
4737	EmitCommonNeonBuiltinExpr(unsigned BuiltinID, unsigned LLVMIntrinsic,
4738	unsigned AltLLVMIntrinsic, const char *NameHint,
4739	unsigned Modifier, const CallExpr *E,
4740	SmallVectorImpl<llvm::Value *> &Ops, Address PtrOp0,
4741	Address PtrOp1, llvm::Triple::ArchType Arch);
4742
4743	llvm::Function *LookupNeonLLVMIntrinsic(unsigned IntrinsicID,
4744	unsigned Modifier, llvm::Type *ArgTy,
4745	const CallExpr *E);
4746	llvm::Value *EmitNeonCall(llvm::Function *F,
4747	SmallVectorImpl<llvm::Value *> &O, const char *name,
4748	unsigned shift = 0, bool rightshift = false);
4749	llvm::Value *EmitFP8NeonCall(unsigned IID, ArrayRef<llvm::Type *> Tys,
4750	SmallVectorImpl<llvm::Value *> &O,
4751	const CallExpr *E, const char *name);
4752	llvm::Value *EmitFP8NeonCvtCall(unsigned IID, llvm::Type *Ty0,
4753	llvm::Type *Ty1, bool Extract,
4754	SmallVectorImpl<llvm::Value *> &Ops,
4755	const CallExpr *E, const char *name);
4756	llvm::Value *EmitFP8NeonFDOTCall(unsigned IID, bool ExtendLaneArg,
4757	llvm::Type *RetTy,
4758	SmallVectorImpl<llvm::Value *> &Ops,
4759	const CallExpr *E, const char *name);
4760	llvm::Value *EmitFP8NeonFMLACall(unsigned IID, bool ExtendLaneArg,
4761	llvm::Type *RetTy,
4762	SmallVectorImpl<llvm::Value *> &Ops,
4763	const CallExpr *E, const char *name);
4764	llvm::Value *EmitNeonSplat(llvm::Value *V, llvm::Constant *Idx,
4765	const llvm::ElementCount &Count);
4766	llvm::Value *EmitNeonSplat(llvm::Value *V, llvm::Constant *Idx);
4767	llvm::Value *EmitNeonShiftVector(llvm::Value *V, llvm::Type *Ty,
4768	bool negateForRightShift);
4769	llvm::Value *EmitNeonRShiftImm(llvm::Value *Vec, llvm::Value *Amt,
4770	llvm::Type *Ty, bool usgn, const char *name);
4771	llvm::Value *vectorWrapScalar16(llvm::Value *Op);
4772	/// SVEBuiltinMemEltTy - Returns the memory element type for this memory
4773	/// access builtin. Only required if it can't be inferred from the base
4774	/// pointer operand.
4775	llvm::Type *SVEBuiltinMemEltTy(const SVETypeFlags &TypeFlags);
4776
4777	SmallVector<llvm::Type *, 2>
4778	getSVEOverloadTypes(const SVETypeFlags &TypeFlags, llvm::Type *ReturnType,
4779	ArrayRef<llvm::Value *> Ops);
4780	llvm::Type *getEltType(const SVETypeFlags &TypeFlags);
4781	llvm::ScalableVectorType *getSVEType(const SVETypeFlags &TypeFlags);
4782	llvm::ScalableVectorType *getSVEPredType(const SVETypeFlags &TypeFlags);
4783	llvm::Value *EmitSVETupleSetOrGet(const SVETypeFlags &TypeFlags,
4784	ArrayRef<llvm::Value *> Ops);
4785	llvm::Value *EmitSVETupleCreate(const SVETypeFlags &TypeFlags,
4786	llvm::Type *ReturnType,
4787	ArrayRef<llvm::Value *> Ops);
4788	llvm::Value *EmitSVEAllTruePred(const SVETypeFlags &TypeFlags);
4789	llvm::Value *EmitSVEDupX(llvm::Value *Scalar);
4790	llvm::Value *EmitSVEDupX(llvm::Value *Scalar, llvm::Type *Ty);
4791	llvm::Value *EmitSVEReinterpret(llvm::Value *Val, llvm::Type *Ty);
4792	llvm::Value *EmitSVEPMull(const SVETypeFlags &TypeFlags,
4793	llvm::SmallVectorImpl<llvm::Value *> &Ops,
4794	unsigned BuiltinID);
4795	llvm::Value *EmitSVEMovl(const SVETypeFlags &TypeFlags,
4796	llvm::ArrayRef<llvm::Value *> Ops,
4797	unsigned BuiltinID);
4798	llvm::Value *EmitSVEPredicateCast(llvm::Value *Pred,
4799	llvm::ScalableVectorType *VTy);
4800	llvm::Value *EmitSVEPredicateTupleCast(llvm::Value *PredTuple,
4801	llvm::StructType *Ty);
4802	llvm::Value *EmitSVEGatherLoad(const SVETypeFlags &TypeFlags,
4803	llvm::SmallVectorImpl<llvm::Value *> &Ops,
4804	unsigned IntID);
4805	llvm::Value *EmitSVEScatterStore(const SVETypeFlags &TypeFlags,
4806	llvm::SmallVectorImpl<llvm::Value *> &Ops,
4807	unsigned IntID);
4808	llvm::Value *EmitSVEMaskedLoad(const CallExpr *, llvm::Type *ReturnTy,
4809	SmallVectorImpl<llvm::Value *> &Ops,
4810	unsigned BuiltinID, bool IsZExtReturn);
4811	llvm::Value *EmitSVEMaskedStore(const CallExpr *,
4812	SmallVectorImpl<llvm::Value *> &Ops,
4813	unsigned BuiltinID);
4814	llvm::Value *EmitSVEPrefetchLoad(const SVETypeFlags &TypeFlags,
4815	SmallVectorImpl<llvm::Value *> &Ops,
4816	unsigned BuiltinID);
4817	llvm::Value *EmitSVEGatherPrefetch(const SVETypeFlags &TypeFlags,
4818	SmallVectorImpl<llvm::Value *> &Ops,
4819	unsigned IntID);
4820	llvm::Value *EmitSVEStructLoad(const SVETypeFlags &TypeFlags,
4821	SmallVectorImpl<llvm::Value *> &Ops,
4822	unsigned IntID);
4823	llvm::Value *EmitSVEStructStore(const SVETypeFlags &TypeFlags,
4824	SmallVectorImpl<llvm::Value *> &Ops,
4825	unsigned IntID);
4826	llvm::Value *EmitAArch64SVEBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
4827
4828	llvm::Value *EmitSMELd1St1(const SVETypeFlags &TypeFlags,
4829	llvm::SmallVectorImpl<llvm::Value *> &Ops,
4830	unsigned IntID);
4831	llvm::Value *EmitSMEReadWrite(const SVETypeFlags &TypeFlags,
4832	llvm::SmallVectorImpl<llvm::Value *> &Ops,
4833	unsigned IntID);
4834	llvm::Value *EmitSMEZero(const SVETypeFlags &TypeFlags,
4835	llvm::SmallVectorImpl<llvm::Value *> &Ops,
4836	unsigned IntID);
4837	llvm::Value *EmitSMELdrStr(const SVETypeFlags &TypeFlags,
4838	llvm::SmallVectorImpl<llvm::Value *> &Ops,
4839	unsigned IntID);
4840
4841	void GetAArch64SVEProcessedOperands(unsigned BuiltinID, const CallExpr *E,
4842	SmallVectorImpl<llvm::Value *> &Ops,
4843	SVETypeFlags TypeFlags);
4844
4845	llvm::Value *EmitAArch64SMEBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
4846
4847	llvm::Value *EmitAArch64BuiltinExpr(unsigned BuiltinID, const CallExpr *E,
4848	llvm::Triple::ArchType Arch);
4849	llvm::Value *EmitBPFBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
4850
4851	llvm::Value *BuildVector(ArrayRef<llvm::Value *> Ops);
4852	llvm::Value *EmitX86BuiltinExpr(unsigned BuiltinID, const CallExpr *E);
4853	llvm::Value *EmitPPCBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
4854	llvm::Value *EmitAMDGPUBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
4855	llvm::Value *EmitHLSLBuiltinExpr(unsigned BuiltinID, const CallExpr *E,
4856	ReturnValueSlot ReturnValue);
4857
4858	// Returns a builtin function that the SPIR-V backend will expand into a spec
4859	// constant.
4860	llvm::Function *
4861	getSpecConstantFunction(const clang::QualType &SpecConstantType);
4862
4863	llvm::Value *EmitDirectXBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
4864	llvm::Value *EmitSPIRVBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
4865	llvm::Value *EmitScalarOrConstFoldImmArg(unsigned ICEArguments, unsigned Idx,
4866	const CallExpr *E);
4867	llvm::Value *EmitSystemZBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
4868	llvm::Value *EmitNVPTXBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
4869	llvm::Value *EmitWebAssemblyBuiltinExpr(unsigned BuiltinID,
4870	const CallExpr *E);
4871	llvm::Value *EmitHexagonBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
4872	llvm::Value *EmitRISCVBuiltinExpr(unsigned BuiltinID, const CallExpr *E,
4873	ReturnValueSlot ReturnValue);
4874
4875	llvm::Value *EmitRISCVCpuSupports(const CallExpr *E);
4876	llvm::Value *EmitRISCVCpuSupports(ArrayRef<StringRef> FeaturesStrs);
4877	llvm::Value *EmitRISCVCpuInit();
4878	llvm::Value *EmitRISCVCpuIs(const CallExpr *E);
4879	llvm::Value *EmitRISCVCpuIs(StringRef CPUStr);
4880
4881	void AddAMDGPUFenceAddressSpaceMMRA(llvm::Instruction *Inst,
4882	const CallExpr *E);
4883	void ProcessOrderScopeAMDGCN(llvm::Value *Order, llvm::Value *Scope,
4884	llvm::AtomicOrdering &AO,
4885	llvm::SyncScope::ID &SSID);
4886
4887	enum class MSVCIntrin;
4888	llvm::Value *EmitMSVCBuiltinExpr(MSVCIntrin BuiltinID, const CallExpr *E);
4889
4890	llvm::Value *EmitBuiltinAvailable(const VersionTuple &Version);
4891
4892	llvm::Value *EmitObjCProtocolExpr(const ObjCProtocolExpr *E);
4893	llvm::Value *EmitObjCStringLiteral(const ObjCStringLiteral *E);
4894	llvm::Value *EmitObjCBoxedExpr(const ObjCBoxedExpr *E);
4895	llvm::Value *EmitObjCArrayLiteral(const ObjCArrayLiteral *E);
4896	llvm::Value *EmitObjCDictionaryLiteral(const ObjCDictionaryLiteral *E);
4897	llvm::Value *
4898	EmitObjCCollectionLiteral(const Expr *E,
4899	const ObjCMethodDecl *MethodWithObjects);
4900	llvm::Value *EmitObjCSelectorExpr(const ObjCSelectorExpr *E);
4901	RValue EmitObjCMessageExpr(const ObjCMessageExpr *E,
4902	ReturnValueSlot Return = ReturnValueSlot());
4903
4904	/// Retrieves the default cleanup kind for an ARC cleanup.
4905	/// Except under -fobjc-arc-eh, ARC cleanups are normal-only.
4906	CleanupKind getARCCleanupKind() {
4907	return CGM.getCodeGenOpts().ObjCAutoRefCountExceptions ? NormalAndEHCleanup
4908	: NormalCleanup;
4909	}
4910
4911	// ARC primitives.
4912	void EmitARCInitWeak(Address addr, llvm::Value *value);
4913	void EmitARCDestroyWeak(Address addr);
4914	llvm::Value *EmitARCLoadWeak(Address addr);
4915	llvm::Value *EmitARCLoadWeakRetained(Address addr);
4916	llvm::Value *EmitARCStoreWeak(Address addr, llvm::Value *value, bool ignored);
4917	void emitARCCopyAssignWeak(QualType Ty, Address DstAddr, Address SrcAddr);
4918	void emitARCMoveAssignWeak(QualType Ty, Address DstAddr, Address SrcAddr);
4919	void EmitARCCopyWeak(Address dst, Address src);
4920	void EmitARCMoveWeak(Address dst, Address src);
4921	llvm::Value *EmitARCRetainAutorelease(QualType type, llvm::Value *value);
4922	llvm::Value *EmitARCRetainAutoreleaseNonBlock(llvm::Value *value);
4923	llvm::Value *EmitARCStoreStrong(LValue lvalue, llvm::Value *value,
4924	bool resultIgnored);
4925	llvm::Value *EmitARCStoreStrongCall(Address addr, llvm::Value *value,
4926	bool resultIgnored);
4927	llvm::Value *EmitARCRetain(QualType type, llvm::Value *value);
4928	llvm::Value *EmitARCRetainNonBlock(llvm::Value *value);
4929	llvm::Value *EmitARCRetainBlock(llvm::Value *value, bool mandatory);
4930	void EmitARCDestroyStrong(Address addr, ARCPreciseLifetime_t precise);
4931	void EmitARCRelease(llvm::Value *value, ARCPreciseLifetime_t precise);
4932	llvm::Value *EmitARCAutorelease(llvm::Value *value);
4933	llvm::Value *EmitARCAutoreleaseReturnValue(llvm::Value *value);
4934	llvm::Value *EmitARCRetainAutoreleaseReturnValue(llvm::Value *value);
4935	llvm::Value *EmitARCRetainAutoreleasedReturnValue(llvm::Value *value);
4936	llvm::Value *EmitARCUnsafeClaimAutoreleasedReturnValue(llvm::Value *value);
4937
4938	llvm::Value *EmitObjCAutorelease(llvm::Value *value, llvm::Type *returnType);
4939	llvm::Value *EmitObjCRetainNonBlock(llvm::Value *value,
4940	llvm::Type *returnType);
4941	void EmitObjCRelease(llvm::Value *value, ARCPreciseLifetime_t precise);
4942
4943	std::pair<LValue, llvm::Value *>
4944	EmitARCStoreAutoreleasing(const BinaryOperator *e);
4945	std::pair<LValue, llvm::Value *> EmitARCStoreStrong(const BinaryOperator *e,
4946	bool ignored);
4947	std::pair<LValue, llvm::Value *>
4948	EmitARCStoreUnsafeUnretained(const BinaryOperator *e, bool ignored);
4949
4950	llvm::Value *EmitObjCAlloc(llvm::Value *value, llvm::Type *returnType);
4951	llvm::Value *EmitObjCAllocWithZone(llvm::Value *value,
4952	llvm::Type *returnType);
4953	llvm::Value *EmitObjCAllocInit(llvm::Value *value, llvm::Type *resultType);
4954
4955	llvm::Value *EmitObjCThrowOperand(const Expr *expr);
4956	llvm::Value *EmitObjCConsumeObject(QualType T, llvm::Value *Ptr);
4957	llvm::Value *EmitObjCExtendObjectLifetime(QualType T, llvm::Value *Ptr);
4958
4959	llvm::Value *EmitARCExtendBlockObject(const Expr *expr);
4960	llvm::Value *EmitARCReclaimReturnedObject(const Expr *e,
4961	bool allowUnsafeClaim);
4962	llvm::Value *EmitARCRetainScalarExpr(const Expr *expr);
4963	llvm::Value *EmitARCRetainAutoreleaseScalarExpr(const Expr *expr);
4964	llvm::Value *EmitARCUnsafeUnretainedScalarExpr(const Expr *expr);
4965
4966	void EmitARCIntrinsicUse(ArrayRef<llvm::Value *> values);
4967
4968	void EmitARCNoopIntrinsicUse(ArrayRef<llvm::Value *> values);
4969
4970	static Destroyer destroyARCStrongImprecise;
4971	static Destroyer destroyARCStrongPrecise;
4972	static Destroyer destroyARCWeak;
4973	static Destroyer emitARCIntrinsicUse;
4974	static Destroyer destroyNonTrivialCStruct;
4975
4976	void EmitObjCAutoreleasePoolPop(llvm::Value *Ptr);
4977	llvm::Value *EmitObjCAutoreleasePoolPush();
4978	llvm::Value *EmitObjCMRRAutoreleasePoolPush();
4979	void EmitObjCAutoreleasePoolCleanup(llvm::Value *Ptr);
4980	void EmitObjCMRRAutoreleasePoolPop(llvm::Value *Ptr);
4981
4982	/// Emits a reference binding to the passed in expression.
4983	RValue EmitReferenceBindingToExpr(const Expr *E);
4984
4985	//===--------------------------------------------------------------------===//
4986	// Expression Emission
4987	//===--------------------------------------------------------------------===//
4988
4989	// Expressions are broken into three classes: scalar, complex, aggregate.
4990
4991	/// EmitScalarExpr - Emit the computation of the specified expression of LLVM
4992	/// scalar type, returning the result.
4993	llvm::Value *EmitScalarExpr(const Expr *E, bool IgnoreResultAssign = false);
4994
4995	/// Emit a conversion from the specified type to the specified destination
4996	/// type, both of which are LLVM scalar types.
4997	llvm::Value *EmitScalarConversion(llvm::Value *Src, QualType SrcTy,
4998	QualType DstTy, SourceLocation Loc);
4999
5000	/// Emit a conversion from the specified complex type to the specified
5001	/// destination type, where the destination type is an LLVM scalar type.
5002	llvm::Value *EmitComplexToScalarConversion(ComplexPairTy Src, QualType SrcTy,
5003	QualType DstTy,
5004	SourceLocation Loc);
5005
5006	/// EmitAggExpr - Emit the computation of the specified expression
5007	/// of aggregate type. The result is computed into the given slot,
5008	/// which may be null to indicate that the value is not needed.
5009	void EmitAggExpr(const Expr *E, AggValueSlot AS);
5010
5011	/// EmitAggExprToLValue - Emit the computation of the specified expression of
5012	/// aggregate type into a temporary LValue.
5013	LValue EmitAggExprToLValue(const Expr *E);
5014
5015	enum ExprValueKind { EVK_RValue, EVK_NonRValue };
5016
5017	/// EmitAggFinalDestCopy - Emit copy of the specified aggregate into
5018	/// destination address.
5019	void EmitAggFinalDestCopy(QualType Type, AggValueSlot Dest, const LValue &Src,
5020	ExprValueKind SrcKind);
5021
5022	/// Create a store to \arg DstPtr from \arg Src, truncating the stored value
5023	/// to at most \arg DstSize bytes.
5024	void CreateCoercedStore(llvm::Value *Src, Address Dst, llvm::TypeSize DstSize,
5025	bool DstIsVolatile);
5026
5027	/// EmitExtendGCLifetime - Given a pointer to an Objective-C object,
5028	/// make sure it survives garbage collection until this point.
5029	void EmitExtendGCLifetime(llvm::Value *object);
5030
5031	/// EmitComplexExpr - Emit the computation of the specified expression of
5032	/// complex type, returning the result.
5033	ComplexPairTy EmitComplexExpr(const Expr *E, bool IgnoreReal = false,
5034	bool IgnoreImag = false);
5035
5036	/// EmitComplexExprIntoLValue - Emit the given expression of complex
5037	/// type and place its result into the specified l-value.
5038	void EmitComplexExprIntoLValue(const Expr *E, LValue dest, bool isInit);
5039
5040	/// EmitStoreOfComplex - Store a complex number into the specified l-value.
5041	void EmitStoreOfComplex(ComplexPairTy V, LValue dest, bool isInit);
5042
5043	/// EmitLoadOfComplex - Load a complex number from the specified l-value.
5044	ComplexPairTy EmitLoadOfComplex(LValue src, SourceLocation loc);
5045
5046	ComplexPairTy EmitPromotedComplexExpr(const Expr *E, QualType PromotionType);
5047	llvm::Value *EmitPromotedScalarExpr(const Expr *E, QualType PromotionType);
5048	ComplexPairTy EmitPromotedValue(ComplexPairTy result, QualType PromotionType);
5049	ComplexPairTy EmitUnPromotedValue(ComplexPairTy result,
5050	QualType PromotionType);
5051
5052	Address emitAddrOfRealComponent(Address complex, QualType complexType);
5053	Address emitAddrOfImagComponent(Address complex, QualType complexType);
5054
5055	/// AddInitializerToStaticVarDecl - Add the initializer for 'D' to the
5056	/// global variable that has already been created for it. If the initializer
5057	/// has a different type than GV does, this may free GV and return a different
5058	/// one. Otherwise it just returns GV.
5059	llvm::GlobalVariable *AddInitializerToStaticVarDecl(const VarDecl &D,
5060	llvm::GlobalVariable *GV);
5061
5062	// Emit an @llvm.invariant.start call for the given memory region.
5063	void EmitInvariantStart(llvm::Constant *Addr, CharUnits Size);
5064
5065	/// EmitCXXGlobalVarDeclInit - Create the initializer for a C++
5066	/// variable with global storage.
5067	void EmitCXXGlobalVarDeclInit(const VarDecl &D, llvm::GlobalVariable *GV,
5068	bool PerformInit);
5069
5070	llvm::Constant *createAtExitStub(const VarDecl &VD, llvm::FunctionCallee Dtor,
5071	llvm::Constant *Addr);
5072
5073	llvm::Function *createTLSAtExitStub(const VarDecl &VD,
5074	llvm::FunctionCallee Dtor,
5075	llvm::Constant *Addr,
5076	llvm::FunctionCallee &AtExit);
5077
5078	/// Call atexit() with a function that passes the given argument to
5079	/// the given function.
5080	void registerGlobalDtorWithAtExit(const VarDecl &D, llvm::FunctionCallee fn,
5081	llvm::Constant *addr);
5082
5083	/// Registers the dtor using 'llvm.global_dtors' for platforms that do not
5084	/// support an 'atexit()' function.
5085	void registerGlobalDtorWithLLVM(const VarDecl &D, llvm::FunctionCallee fn,
5086	llvm::Constant *addr);
5087
5088	/// Call atexit() with function dtorStub.
5089	void registerGlobalDtorWithAtExit(llvm::Constant *dtorStub);
5090
5091	/// Call unatexit() with function dtorStub.
5092	llvm::Value *unregisterGlobalDtorWithUnAtExit(llvm::Constant *dtorStub);
5093
5094	/// Emit code in this function to perform a guarded variable
5095	/// initialization. Guarded initializations are used when it's not
5096	/// possible to prove that an initialization will be done exactly
5097	/// once, e.g. with a static local variable or a static data member
5098	/// of a class template.
5099	void EmitCXXGuardedInit(const VarDecl &D, llvm::GlobalVariable *DeclPtr,
5100	bool PerformInit);
5101
5102	enum class GuardKind { VariableGuard, TlsGuard };
5103
5104	/// Emit a branch to select whether or not to perform guarded initialization.
5105	void EmitCXXGuardedInitBranch(llvm::Value *NeedsInit,
5106	llvm::BasicBlock *InitBlock,
5107	llvm::BasicBlock *NoInitBlock, GuardKind Kind,
5108	const VarDecl *D);
5109
5110	/// GenerateCXXGlobalInitFunc - Generates code for initializing global
5111	/// variables.
5112	void
5113	GenerateCXXGlobalInitFunc(llvm::Function *Fn,
5114	ArrayRef<llvm::Function *> CXXThreadLocals,
5115	ConstantAddress Guard = ConstantAddress::invalid());
5116
5117	/// GenerateCXXGlobalCleanUpFunc - Generates code for cleaning up global
5118	/// variables.
5119	void GenerateCXXGlobalCleanUpFunc(
5120	llvm::Function *Fn,
5121	ArrayRef<std::tuple<llvm::FunctionType *, llvm::WeakTrackingVH,
5122	llvm::Constant *>>
5123	DtorsOrStermFinalizers);
5124
5125	void GenerateCXXGlobalVarDeclInitFunc(llvm::Function *Fn, const VarDecl *D,
5126	llvm::GlobalVariable *Addr,
5127	bool PerformInit);
5128
5129	void EmitCXXConstructExpr(const CXXConstructExpr *E, AggValueSlot Dest);
5130
5131	void EmitSynthesizedCXXCopyCtor(Address Dest, Address Src, const Expr *Exp);
5132
5133	void EmitCXXThrowExpr(const CXXThrowExpr *E, bool KeepInsertionPoint = true);
5134
5135	RValue EmitAtomicExpr(AtomicExpr *E);
5136
5137	void EmitFakeUse(Address Addr);
5138
5139	//===--------------------------------------------------------------------===//
5140	// Annotations Emission
5141	//===--------------------------------------------------------------------===//
5142
5143	/// Emit an annotation call (intrinsic).
5144	llvm::Value *EmitAnnotationCall(llvm::Function *AnnotationFn,
5145	llvm::Value *AnnotatedVal,
5146	StringRef AnnotationStr,
5147	SourceLocation Location,
5148	const AnnotateAttr *Attr);
5149
5150	/// Emit local annotations for the local variable V, declared by D.
5151	void EmitVarAnnotations(const VarDecl *D, llvm::Value *V);
5152
5153	/// Emit field annotations for the given field & value. Returns the
5154	/// annotation result.
5155	Address EmitFieldAnnotations(const FieldDecl *D, Address V);
5156
5157	//===--------------------------------------------------------------------===//
5158	// Internal Helpers
5159	//===--------------------------------------------------------------------===//
5160
5161	/// ContainsLabel - Return true if the statement contains a label in it. If
5162	/// this statement is not executed normally, it not containing a label means
5163	/// that we can just remove the code.
5164	static bool ContainsLabel(const Stmt *S, bool IgnoreCaseStmts = false);
5165
5166	/// containsBreak - Return true if the statement contains a break out of it.
5167	/// If the statement (recursively) contains a switch or loop with a break
5168	/// inside of it, this is fine.
5169	static bool containsBreak(const Stmt *S);
5170
5171	/// Determine if the given statement might introduce a declaration into the
5172	/// current scope, by being a (possibly-labelled) DeclStmt.
5173	static bool mightAddDeclToScope(const Stmt *S);
5174
5175	/// ConstantFoldsToSimpleInteger - If the specified expression does not fold
5176	/// to a constant, or if it does but contains a label, return false. If it
5177	/// constant folds return true and set the boolean result in Result.
5178	bool ConstantFoldsToSimpleInteger(const Expr *Cond, bool &Result,
5179	bool AllowLabels = false);
5180
5181	/// ConstantFoldsToSimpleInteger - If the specified expression does not fold
5182	/// to a constant, or if it does but contains a label, return false. If it
5183	/// constant folds return true and set the folded value.
5184	bool ConstantFoldsToSimpleInteger(const Expr *Cond, llvm::APSInt &Result,
5185	bool AllowLabels = false);
5186
5187	/// Ignore parentheses and logical-NOT to track conditions consistently.
5188	static const Expr *stripCond(const Expr *C);
5189
5190	/// isInstrumentedCondition - Determine whether the given condition is an
5191	/// instrumentable condition (i.e. no "&&" or "||").
5192	static bool isInstrumentedCondition(const Expr *C);
5193
5194	/// EmitBranchToCounterBlock - Emit a conditional branch to a new block that
5195	/// increments a profile counter based on the semantics of the given logical
5196	/// operator opcode. This is used to instrument branch condition coverage
5197	/// for logical operators.
5198	void EmitBranchToCounterBlock(const Expr *Cond, BinaryOperator::Opcode LOp,
5199	llvm::BasicBlock *TrueBlock,
5200	llvm::BasicBlock *FalseBlock,
5201	uint64_t TrueCount = 0,
5202	Stmt::Likelihood LH = Stmt::LH_None,
5203	const Expr *CntrIdx = nullptr);
5204
5205	/// EmitBranchOnBoolExpr - Emit a branch on a boolean condition (e.g. for an
5206	/// if statement) to the specified blocks. Based on the condition, this might
5207	/// try to simplify the codegen of the conditional based on the branch.
5208	/// TrueCount should be the number of times we expect the condition to
5209	/// evaluate to true based on PGO data.
5210	void EmitBranchOnBoolExpr(const Expr *Cond, llvm::BasicBlock *TrueBlock,
5211	llvm::BasicBlock *FalseBlock, uint64_t TrueCount,
5212	Stmt::Likelihood LH = Stmt::LH_None,
5213	const Expr *ConditionalOp = nullptr,
5214	const VarDecl *ConditionalDecl = nullptr);
5215
5216	/// Given an assignment `*LHS = RHS`, emit a test that checks if \p RHS is
5217	/// nonnull, if \p LHS is marked _Nonnull.
5218	void EmitNullabilityCheck(LValue LHS, llvm::Value *RHS, SourceLocation Loc);
5219
5220	/// An enumeration which makes it easier to specify whether or not an
5221	/// operation is a subtraction.
5222	enum { NotSubtraction = false, IsSubtraction = true };
5223
5224	/// Same as IRBuilder::CreateInBoundsGEP, but additionally emits a check to
5225	/// detect undefined behavior when the pointer overflow sanitizer is enabled.
5226	/// \p SignedIndices indicates whether any of the GEP indices are signed.
5227	/// \p IsSubtraction indicates whether the expression used to form the GEP
5228	/// is a subtraction.
5229	llvm::Value *EmitCheckedInBoundsGEP(llvm::Type *ElemTy, llvm::Value *Ptr,
5230	ArrayRef<llvm::Value *> IdxList,
5231	bool SignedIndices, bool IsSubtraction,
5232	SourceLocation Loc,
5233	const Twine &Name = "");
5234
5235	Address EmitCheckedInBoundsGEP(Address Addr, ArrayRef<llvm::Value *> IdxList,
5236	llvm::Type *elementType, bool SignedIndices,
5237	bool IsSubtraction, SourceLocation Loc,
5238	CharUnits Align, const Twine &Name = "");
5239
5240	/// Specifies which type of sanitizer check to apply when handling a
5241	/// particular builtin.
5242	enum BuiltinCheckKind {
5243	BCK_CTZPassedZero,
5244	BCK_CLZPassedZero,
5245	BCK_AssumePassedFalse,
5246	};
5247
5248	/// Emits an argument for a call to a builtin. If the builtin sanitizer is
5249	/// enabled, a runtime check specified by \p Kind is also emitted.
5250	llvm::Value *EmitCheckedArgForBuiltin(const Expr *E, BuiltinCheckKind Kind);
5251
5252	/// Emits an argument for a call to a `__builtin_assume`. If the builtin
5253	/// sanitizer is enabled, a runtime check is also emitted.
5254	llvm::Value *EmitCheckedArgForAssume(const Expr *E);
5255
5256	/// Emit a description of a type in a format suitable for passing to
5257	/// a runtime sanitizer handler.
5258	llvm::Constant *EmitCheckTypeDescriptor(QualType T);
5259
5260	/// Convert a value into a format suitable for passing to a runtime
5261	/// sanitizer handler.
5262	llvm::Value *EmitCheckValue(llvm::Value *V);
5263
5264	/// Emit a description of a source location in a format suitable for
5265	/// passing to a runtime sanitizer handler.
5266	llvm::Constant *EmitCheckSourceLocation(SourceLocation Loc);
5267
5268	void EmitKCFIOperandBundle(const CGCallee &Callee,
5269	SmallVectorImpl<llvm::OperandBundleDef> &Bundles);
5270
5271	/// Create a basic block that will either trap or call a handler function in
5272	/// the UBSan runtime with the provided arguments, and create a conditional
5273	/// branch to it.
5274	void
5275	EmitCheck(ArrayRef<std::pair<llvm::Value *, SanitizerKind::SanitizerOrdinal>>
5276	Checked,
5277	SanitizerHandler Check, ArrayRef<llvm::Constant *> StaticArgs,
5278	ArrayRef<llvm::Value *> DynamicArgs);
5279
5280	/// Emit a slow path cross-DSO CFI check which calls __cfi_slowpath
5281	/// if Cond if false.
5282	void EmitCfiSlowPathCheck(SanitizerKind::SanitizerOrdinal Ordinal,
5283	llvm::Value *Cond, llvm::ConstantInt *TypeId,
5284	llvm::Value *Ptr,
5285	ArrayRef<llvm::Constant *> StaticArgs);
5286
5287	/// Emit a reached-unreachable diagnostic if \p Loc is valid and runtime
5288	/// checking is enabled. Otherwise, just emit an unreachable instruction.
5289	void EmitUnreachable(SourceLocation Loc);
5290
5291	/// Create a basic block that will call the trap intrinsic, and emit a
5292	/// conditional branch to it, for the -ftrapv checks.
5293	void EmitTrapCheck(llvm::Value *Checked, SanitizerHandler CheckHandlerID,
5294	bool NoMerge = false);
5295
5296	/// Emit a call to trap or debugtrap and attach function attribute
5297	/// "trap-func-name" if specified.
5298	llvm::CallInst *EmitTrapCall(llvm::Intrinsic::ID IntrID);
5299
5300	/// Emit a stub for the cross-DSO CFI check function.
5301	void EmitCfiCheckStub();
5302
5303	/// Emit a cross-DSO CFI failure handling function.
5304	void EmitCfiCheckFail();
5305
5306	/// Create a check for a function parameter that may potentially be
5307	/// declared as non-null.
5308	void EmitNonNullArgCheck(RValue RV, QualType ArgType, SourceLocation ArgLoc,
5309	AbstractCallee AC, unsigned ParmNum);
5310
5311	void EmitNonNullArgCheck(Address Addr, QualType ArgType,
5312	SourceLocation ArgLoc, AbstractCallee AC,
5313	unsigned ParmNum);
5314
5315	/// EmitWriteback - Emit callbacks for function.
5316	void EmitWritebacks(const CallArgList &Args);
5317
5318	/// EmitCallArg - Emit a single call argument.
5319	void EmitCallArg(CallArgList &args, const Expr *E, QualType ArgType);
5320
5321	/// EmitDelegateCallArg - We are performing a delegate call; that
5322	/// is, the current function is delegating to another one. Produce
5323	/// a r-value suitable for passing the given parameter.
5324	void EmitDelegateCallArg(CallArgList &args, const VarDecl *param,
5325	SourceLocation loc);
5326
5327	/// SetFPAccuracy - Set the minimum required accuracy of the given floating
5328	/// point operation, expressed as the maximum relative error in ulp.
5329	void SetFPAccuracy(llvm::Value *Val, float Accuracy);
5330
5331	/// Set the minimum required accuracy of the given sqrt operation
5332	/// based on CodeGenOpts.
5333	void SetSqrtFPAccuracy(llvm::Value *Val);
5334
5335	/// Set the minimum required accuracy of the given sqrt operation based on
5336	/// CodeGenOpts.
5337	void SetDivFPAccuracy(llvm::Value *Val);
5338
5339	/// Set the codegen fast-math flags.
5340	void SetFastMathFlags(FPOptions FPFeatures);
5341
5342	// Truncate or extend a boolean vector to the requested number of elements.
5343	llvm::Value *emitBoolVecConversion(llvm::Value *SrcVec,
5344	unsigned NumElementsDst,
5345	const llvm::Twine &Name = "");
5346
5347	void maybeAttachRangeForLoad(llvm::LoadInst *Load, QualType Ty,
5348	SourceLocation Loc);
5349
5350	private:
5351	// Emits a convergence_loop instruction for the given |BB|, with |ParentToken|
5352	// as it's parent convergence instr.
5353	llvm::ConvergenceControlInst *emitConvergenceLoopToken(llvm::BasicBlock *BB);
5354
5355	// Adds a convergence_ctrl token with |ParentToken| as parent convergence
5356	// instr to the call |Input|.
5357	llvm::CallBase *addConvergenceControlToken(llvm::CallBase *Input);
5358
5359	// Find the convergence_entry instruction |F|, or emits ones if none exists.
5360	// Returns the convergence instruction.
5361	llvm::ConvergenceControlInst *
5362	getOrEmitConvergenceEntryToken(llvm::Function *F);
5363
5364	private:
5365	llvm::MDNode *getRangeForLoadFromType(QualType Ty);
5366	void EmitReturnOfRValue(RValue RV, QualType Ty);
5367
5368	void deferPlaceholderReplacement(llvm::Instruction *Old, llvm::Value *New);
5369
5370	llvm::SmallVector<std::pair<llvm::WeakTrackingVH, llvm::Value *>, 4>
5371	DeferredReplacements;
5372
5373	/// Set the address of a local variable.
5374	void setAddrOfLocalVar(const VarDecl *VD, Address Addr) {
5375	assert(!LocalDeclMap.count(VD) && "Decl already exists in LocalDeclMap!");
5376	LocalDeclMap.insert(KV: {VD, Addr});
5377	}
5378
5379	/// ExpandTypeFromArgs - Reconstruct a structure of type \arg Ty
5380	/// from function arguments into \arg Dst. See ABIArgInfo::Expand.
5381	///
5382	/// \param AI - The first function argument of the expansion.
5383	void ExpandTypeFromArgs(QualType Ty, LValue Dst,
5384	llvm::Function::arg_iterator &AI);
5385
5386	/// ExpandTypeToArgs - Expand an CallArg \arg Arg, with the LLVM type for \arg
5387	/// Ty, into individual arguments on the provided vector \arg IRCallArgs,
5388	/// starting at index \arg IRCallArgPos. See ABIArgInfo::Expand.
5389	void ExpandTypeToArgs(QualType Ty, CallArg Arg, llvm::FunctionType *IRFuncTy,
5390	SmallVectorImpl<llvm::Value *> &IRCallArgs,
5391	unsigned &IRCallArgPos);
5392
5393	std::pair<llvm::Value *, llvm::Type *>
5394	EmitAsmInput(const TargetInfo::ConstraintInfo &Info, const Expr *InputExpr,
5395	std::string &ConstraintStr);
5396
5397	std::pair<llvm::Value *, llvm::Type *>
5398	EmitAsmInputLValue(const TargetInfo::ConstraintInfo &Info, LValue InputValue,
5399	QualType InputType, std::string &ConstraintStr,
5400	SourceLocation Loc);
5401
5402	/// Attempts to statically evaluate the object size of E. If that
5403	/// fails, emits code to figure the size of E out for us. This is
5404	/// pass_object_size aware.
5405	///
5406	/// If EmittedExpr is non-null, this will use that instead of re-emitting E.
5407	llvm::Value *evaluateOrEmitBuiltinObjectSize(const Expr *E, unsigned Type,
5408	llvm::IntegerType *ResType,
5409	llvm::Value *EmittedE,
5410	bool IsDynamic);
5411
5412	/// Emits the size of E, as required by __builtin_object_size. This
5413	/// function is aware of pass_object_size parameters, and will act accordingly
5414	/// if E is a parameter with the pass_object_size attribute.
5415	llvm::Value *emitBuiltinObjectSize(const Expr *E, unsigned Type,
5416	llvm::IntegerType *ResType,
5417	llvm::Value *EmittedE, bool IsDynamic);
5418
5419	llvm::Value *emitCountedBySize(const Expr *E, llvm::Value *EmittedE,
5420	unsigned Type, llvm::IntegerType *ResType);
5421
5422	llvm::Value *emitCountedByMemberSize(const MemberExpr *E, const Expr *Idx,
5423	llvm::Value *EmittedE,
5424	QualType CastedArrayElementTy,
5425	unsigned Type,
5426	llvm::IntegerType *ResType);
5427
5428	llvm::Value *emitCountedByPointerSize(const ImplicitCastExpr *E,
5429	const Expr *Idx, llvm::Value *EmittedE,
5430	QualType CastedArrayElementTy,
5431	unsigned Type,
5432	llvm::IntegerType *ResType);
5433
5434	void emitZeroOrPatternForAutoVarInit(QualType type, const VarDecl &D,
5435	Address Loc);
5436
5437	public:
5438	enum class EvaluationOrder {
5439	///! No language constraints on evaluation order.
5440	Default,
5441	///! Language semantics require left-to-right evaluation.
5442	ForceLeftToRight,
5443	///! Language semantics require right-to-left evaluation.
5444	ForceRightToLeft
5445	};
5446
5447	// Wrapper for function prototype sources. Wraps either a FunctionProtoType or
5448	// an ObjCMethodDecl.
5449	struct PrototypeWrapper {
5450	llvm::PointerUnion<const FunctionProtoType *, const ObjCMethodDecl *> P;
5451
5452	PrototypeWrapper(const FunctionProtoType *FT) : P(FT) {}
5453	PrototypeWrapper(const ObjCMethodDecl *MD) : P(MD) {}
5454	};
5455
5456	void EmitCallArgs(CallArgList &Args, PrototypeWrapper Prototype,
5457	llvm::iterator_range<CallExpr::const_arg_iterator> ArgRange,
5458	AbstractCallee AC = AbstractCallee(),
5459	unsigned ParamsToSkip = 0,
5460	EvaluationOrder Order = EvaluationOrder::Default);
5461
5462	/// EmitPointerWithAlignment - Given an expression with a pointer type,
5463	/// emit the value and compute our best estimate of the alignment of the
5464	/// pointee.
5465	///
5466	/// \param BaseInfo - If non-null, this will be initialized with
5467	/// information about the source of the alignment and the may-alias
5468	/// attribute. Note that this function will conservatively fall back on
5469	/// the type when it doesn't recognize the expression and may-alias will
5470	/// be set to false.
5471	///
5472	/// One reasonable way to use this information is when there's a language
5473	/// guarantee that the pointer must be aligned to some stricter value, and
5474	/// we're simply trying to ensure that sufficiently obvious uses of under-
5475	/// aligned objects don't get miscompiled; for example, a placement new
5476	/// into the address of a local variable. In such a case, it's quite
5477	/// reasonable to just ignore the returned alignment when it isn't from an
5478	/// explicit source.
5479	Address
5480	EmitPointerWithAlignment(const Expr *Addr, LValueBaseInfo *BaseInfo = nullptr,
5481	TBAAAccessInfo *TBAAInfo = nullptr,
5482	KnownNonNull_t IsKnownNonNull = NotKnownNonNull);
5483
5484	/// If \p E references a parameter with pass_object_size info or a constant
5485	/// array size modifier, emit the object size divided by the size of \p EltTy.
5486	/// Otherwise return null.
5487	llvm::Value *LoadPassedObjectSize(const Expr *E, QualType EltTy);
5488
5489	void EmitSanitizerStatReport(llvm::SanitizerStatKind SSK);
5490
5491	struct FMVResolverOption {
5492	llvm::Function *Function;
5493	llvm::SmallVector<StringRef, 8> Features;
5494	std::optional<StringRef> Architecture;
5495
5496	FMVResolverOption(llvm::Function *F, ArrayRef<StringRef> Feats,
5497	std::optional<StringRef> Arch = std::nullopt)
5498	: Function(F), Features(Feats), Architecture(Arch) {}
5499	};
5500
5501	// Emits the body of a multiversion function's resolver. Assumes that the
5502	// options are already sorted in the proper order, with the 'default' option
5503	// last (if it exists).
5504	void EmitMultiVersionResolver(llvm::Function *Resolver,
5505	ArrayRef<FMVResolverOption> Options);
5506	void EmitX86MultiVersionResolver(llvm::Function *Resolver,
5507	ArrayRef<FMVResolverOption> Options);
5508	void EmitAArch64MultiVersionResolver(llvm::Function *Resolver,
5509	ArrayRef<FMVResolverOption> Options);
5510	void EmitRISCVMultiVersionResolver(llvm::Function *Resolver,
5511	ArrayRef<FMVResolverOption> Options);
5512
5513	private:
5514	QualType getVarArgType(const Expr *Arg);
5515
5516	void EmitDeclMetadata();
5517
5518	BlockByrefHelpers *buildByrefHelpers(llvm::StructType &byrefType,
5519	const AutoVarEmission &emission);
5520
5521	void AddObjCARCExceptionMetadata(llvm::Instruction *Inst);
5522
5523	llvm::Value *GetValueForARMHint(unsigned BuiltinID);
5524	llvm::Value *EmitX86CpuIs(const CallExpr *E);
5525	llvm::Value *EmitX86CpuIs(StringRef CPUStr);
5526	llvm::Value *EmitX86CpuSupports(const CallExpr *E);
5527	llvm::Value *EmitX86CpuSupports(ArrayRef<StringRef> FeatureStrs);
5528	llvm::Value *EmitX86CpuSupports(std::array<uint32_t, 4> FeatureMask);
5529	llvm::Value *EmitX86CpuInit();
5530	llvm::Value *FormX86ResolverCondition(const FMVResolverOption &RO);
5531	llvm::Value *EmitAArch64CpuInit();
5532	llvm::Value *FormAArch64ResolverCondition(const FMVResolverOption &RO);
5533	llvm::Value *EmitAArch64CpuSupports(const CallExpr *E);
5534	llvm::Value *EmitAArch64CpuSupports(ArrayRef<StringRef> FeatureStrs);
5535	};
5536
5537	inline DominatingLLVMValue::saved_type
5538	DominatingLLVMValue::save(CodeGenFunction &CGF, llvm::Value *value) {
5539	if (!needsSaving(value))
5540	return saved_type(value, false);
5541
5542	// Otherwise, we need an alloca.
5543	auto align = CharUnits::fromQuantity(
5544	Quantity: CGF.CGM.getDataLayout().getPrefTypeAlign(Ty: value->getType()));
5545	Address alloca =
5546	CGF.CreateTempAlloca(Ty: value->getType(), align, Name: "cond-cleanup.save");
5547	CGF.Builder.CreateStore(Val: value, Addr: alloca);
5548
5549	return saved_type(alloca.emitRawPointer(CGF), true);
5550	}
5551
5552	inline llvm::Value *DominatingLLVMValue::restore(CodeGenFunction &CGF,
5553	saved_type value) {
5554	// If the value says it wasn't saved, trust that it's still dominating.
5555	if (!value.getInt())
5556	return value.getPointer();
5557
5558	// Otherwise, it should be an alloca instruction, as set up in save().
5559	auto alloca = cast<llvm::AllocaInst>(Val: value.getPointer());
5560	return CGF.Builder.CreateAlignedLoad(Ty: alloca->getAllocatedType(), Ptr: alloca,
5561	Align: alloca->getAlign());
5562	}
5563
5564	} // end namespace CodeGen
5565
5566	// Map the LangOption for floating point exception behavior into
5567	// the corresponding enum in the IR.
5568	llvm::fp::ExceptionBehavior
5569	ToConstrainedExceptMD(LangOptions::FPExceptionModeKind Kind);
5570	} // end namespace clang
5571
5572	#endif
5573