1	//===-- CodeGenFunction.h - Per-Function state for LLVM CodeGen -*- C++ -*-===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This is the internal per-function state used for llvm translation.
10	//
11	//===----------------------------------------------------------------------===//
12
13	#ifndef LLVM_CLANG_LIB_CODEGEN_CODEGENFUNCTION_H
14	#define LLVM_CLANG_LIB_CODEGEN_CODEGENFUNCTION_H
15
16	#include "CGBuilder.h"
17	#include "CGDebugInfo.h"
18	#include "CGLoopInfo.h"
19	#include "CGValue.h"
20	#include "CodeGenModule.h"
21	#include "CodeGenPGO.h"
22	#include "EHScopeStack.h"
23	#include "VarBypassDetector.h"
24	#include "clang/AST/CharUnits.h"
25	#include "clang/AST/CurrentSourceLocExprScope.h"
26	#include "clang/AST/ExprCXX.h"
27	#include "clang/AST/ExprObjC.h"
28	#include "clang/AST/ExprOpenMP.h"
29	#include "clang/AST/StmtOpenACC.h"
30	#include "clang/AST/StmtOpenMP.h"
31	#include "clang/AST/Type.h"
32	#include "clang/Basic/ABI.h"
33	#include "clang/Basic/CapturedStmt.h"
34	#include "clang/Basic/CodeGenOptions.h"
35	#include "clang/Basic/OpenMPKinds.h"
36	#include "clang/Basic/TargetInfo.h"
37	#include "llvm/ADT/ArrayRef.h"
38	#include "llvm/ADT/DenseMap.h"
39	#include "llvm/ADT/MapVector.h"
40	#include "llvm/ADT/SmallVector.h"
41	#include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
42	#include "llvm/IR/ValueHandle.h"
43	#include "llvm/Support/Debug.h"
44	#include "llvm/Transforms/Utils/SanitizerStats.h"
45	#include <optional>
46
47	namespace llvm {
48	class BasicBlock;
49	class LLVMContext;
50	class MDNode;
51	class SwitchInst;
52	class Twine;
53	class Value;
54	class CanonicalLoopInfo;
55	}
56
57	namespace clang {
58	class ASTContext;
59	class CXXDestructorDecl;
60	class CXXForRangeStmt;
61	class CXXTryStmt;
62	class Decl;
63	class LabelDecl;
64	class FunctionDecl;
65	class FunctionProtoType;
66	class LabelStmt;
67	class ObjCContainerDecl;
68	class ObjCInterfaceDecl;
69	class ObjCIvarDecl;
70	class ObjCMethodDecl;
71	class ObjCImplementationDecl;
72	class ObjCPropertyImplDecl;
73	class TargetInfo;
74	class VarDecl;
75	class ObjCForCollectionStmt;
76	class ObjCAtTryStmt;
77	class ObjCAtThrowStmt;
78	class ObjCAtSynchronizedStmt;
79	class ObjCAutoreleasePoolStmt;
80	class OMPUseDevicePtrClause;
81	class OMPUseDeviceAddrClause;
82	class SVETypeFlags;
83	class OMPExecutableDirective;
84
85	namespace analyze_os_log {
86	class OSLogBufferLayout;
87	}
88
89	namespace CodeGen {
90	class CodeGenTypes;
91	class CGCallee;
92	class CGFunctionInfo;
93	class CGBlockInfo;
94	class CGCXXABI;
95	class BlockByrefHelpers;
96	class BlockByrefInfo;
97	class BlockFieldFlags;
98	class RegionCodeGenTy;
99	class TargetCodeGenInfo;
100	struct OMPTaskDataTy;
101	struct CGCoroData;
102
103	/// The kind of evaluation to perform on values of a particular
104	/// type. Basically, is the code in CGExprScalar, CGExprComplex, or
105	/// CGExprAgg?
106	///
107	/// TODO: should vectors maybe be split out into their own thing?
108	enum TypeEvaluationKind {
109	TEK_Scalar,
110	TEK_Complex,
111	TEK_Aggregate
112	};
113
114	#define LIST_SANITIZER_CHECKS \
115	SANITIZER_CHECK(AddOverflow, add_overflow, 0) \
116	SANITIZER_CHECK(BuiltinUnreachable, builtin_unreachable, 0) \
117	SANITIZER_CHECK(CFICheckFail, cfi_check_fail, 0) \
118	SANITIZER_CHECK(DivremOverflow, divrem_overflow, 0) \
119	SANITIZER_CHECK(DynamicTypeCacheMiss, dynamic_type_cache_miss, 0) \
120	SANITIZER_CHECK(FloatCastOverflow, float_cast_overflow, 0) \
121	SANITIZER_CHECK(FunctionTypeMismatch, function_type_mismatch, 0) \
122	SANITIZER_CHECK(ImplicitConversion, implicit_conversion, 0) \
123	SANITIZER_CHECK(InvalidBuiltin, invalid_builtin, 0) \
124	SANITIZER_CHECK(InvalidObjCCast, invalid_objc_cast, 0) \
125	SANITIZER_CHECK(LoadInvalidValue, load_invalid_value, 0) \
126	SANITIZER_CHECK(MissingReturn, missing_return, 0) \
127	SANITIZER_CHECK(MulOverflow, mul_overflow, 0) \
128	SANITIZER_CHECK(NegateOverflow, negate_overflow, 0) \
129	SANITIZER_CHECK(NullabilityArg, nullability_arg, 0) \
130	SANITIZER_CHECK(NullabilityReturn, nullability_return, 1) \
131	SANITIZER_CHECK(NonnullArg, nonnull_arg, 0) \
132	SANITIZER_CHECK(NonnullReturn, nonnull_return, 1) \
133	SANITIZER_CHECK(OutOfBounds, out_of_bounds, 0) \
134	SANITIZER_CHECK(PointerOverflow, pointer_overflow, 0) \
135	SANITIZER_CHECK(ShiftOutOfBounds, shift_out_of_bounds, 0) \
136	SANITIZER_CHECK(SubOverflow, sub_overflow, 0) \
137	SANITIZER_CHECK(TypeMismatch, type_mismatch, 1) \
138	SANITIZER_CHECK(AlignmentAssumption, alignment_assumption, 0) \
139	SANITIZER_CHECK(VLABoundNotPositive, vla_bound_not_positive, 0)
140
141	enum SanitizerHandler {
142	#define SANITIZER_CHECK(Enum, Name, Version) Enum,
143	LIST_SANITIZER_CHECKS
144	#undef SANITIZER_CHECK
145	};
146
147	/// Helper class with most of the code for saving a value for a
148	/// conditional expression cleanup.
149	struct DominatingLLVMValue {
150	typedef llvm::PointerIntPair<llvm::Value*, 1, bool> saved_type;
151
152	/// Answer whether the given value needs extra work to be saved.
153	static bool needsSaving(llvm::Value *value) {
154	if (!value)
155	return false;
156
157	// If it's not an instruction, we don't need to save.
158	if (!isa<llvm::Instruction>(Val: value)) return false;
159
160	// If it's an instruction in the entry block, we don't need to save.
161	llvm::BasicBlock *block = cast<llvm::Instruction>(Val: value)->getParent();
162	return (block != &block->getParent()->getEntryBlock());
163	}
164
165	static saved_type save(CodeGenFunction &CGF, llvm::Value *value);
166	static llvm::Value *restore(CodeGenFunction &CGF, saved_type value);
167	};
168
169	/// A partial specialization of DominatingValue for llvm::Values that
170	/// might be llvm::Instructions.
171	template <class T> struct DominatingPointer<T,true> : DominatingLLVMValue {
172	typedef T *type;
173	static type restore(CodeGenFunction &CGF, saved_type value) {
174	return static_cast<T*>(DominatingLLVMValue::restore(CGF, value));
175	}
176	};
177
178	/// A specialization of DominatingValue for Address.
179	template <> struct DominatingValue<Address> {
180	typedef Address type;
181
182	struct saved_type {
183	DominatingLLVMValue::saved_type BasePtr;
184	llvm::Type *ElementType;
185	CharUnits Alignment;
186	DominatingLLVMValue::saved_type Offset;
187	llvm::PointerType *EffectiveType;
188	};
189
190	static bool needsSaving(type value) {
191	if (DominatingLLVMValue::needsSaving(value: value.getBasePointer()) ||
192	DominatingLLVMValue::needsSaving(value: value.getOffset()))
193	return true;
194	return false;
195	}
196	static saved_type save(CodeGenFunction &CGF, type value) {
197	return {.BasePtr: DominatingLLVMValue::save(CGF, value: value.getBasePointer()),
198	.ElementType: value.getElementType(), .Alignment: value.getAlignment(),
199	.Offset: DominatingLLVMValue::save(CGF, value: value.getOffset()), .EffectiveType: value.getType()};
200	}
201	static type restore(CodeGenFunction &CGF, saved_type value) {
202	return Address(DominatingLLVMValue::restore(CGF, value: value.BasePtr),
203	value.ElementType, value.Alignment,
204	DominatingLLVMValue::restore(CGF, value: value.Offset));
205	}
206	};
207
208	/// A specialization of DominatingValue for RValue.
209	template <> struct DominatingValue<RValue> {
210	typedef RValue type;
211	class saved_type {
212	enum Kind { ScalarLiteral, ScalarAddress, AggregateLiteral,
213	AggregateAddress, ComplexAddress };
214	union {
215	struct {
216	DominatingLLVMValue::saved_type first, second;
217	} Vals;
218	DominatingValue<Address>::saved_type AggregateAddr;
219	};
220	LLVM_PREFERRED_TYPE(Kind)
221	unsigned K : 3;
222	unsigned IsVolatile : 1;
223
224	saved_type(DominatingLLVMValue::saved_type Val1, unsigned K)
225	: Vals{.first: Val1, .second: DominatingLLVMValue::saved_type()}, K(K) {}
226
227	saved_type(DominatingLLVMValue::saved_type Val1,
228	DominatingLLVMValue::saved_type Val2)
229	: Vals{.first: Val1, .second: Val2}, K(ComplexAddress) {}
230
231	saved_type(DominatingValue<Address>::saved_type AggregateAddr,
232	bool IsVolatile, unsigned K)
233	: AggregateAddr(AggregateAddr), K(K) {}
234
235	public:
236	static bool needsSaving(RValue value);
237	static saved_type save(CodeGenFunction &CGF, RValue value);
238	RValue restore(CodeGenFunction &CGF);
239
240	// implementations in CGCleanup.cpp
241	};
242
243	static bool needsSaving(type value) {
244	return saved_type::needsSaving(value);
245	}
246	static saved_type save(CodeGenFunction &CGF, type value) {
247	return saved_type::save(CGF, value);
248	}
249	static type restore(CodeGenFunction &CGF, saved_type value) {
250	return value.restore(CGF);
251	}
252	};
253
254	/// CodeGenFunction - This class organizes the per-function state that is used
255	/// while generating LLVM code.
256	class CodeGenFunction : public CodeGenTypeCache {
257	CodeGenFunction(const CodeGenFunction &) = delete;
258	void operator=(const CodeGenFunction &) = delete;
259
260	friend class CGCXXABI;
261	public:
262	/// A jump destination is an abstract label, branching to which may
263	/// require a jump out through normal cleanups.
264	struct JumpDest {
265	JumpDest() : Block(nullptr), Index(0) {}
266	JumpDest(llvm::BasicBlock *Block, EHScopeStack::stable_iterator Depth,
267	unsigned Index)
268	: Block(Block), ScopeDepth(Depth), Index(Index) {}
269
270	bool isValid() const { return Block != nullptr; }
271	llvm::BasicBlock *getBlock() const { return Block; }
272	EHScopeStack::stable_iterator getScopeDepth() const { return ScopeDepth; }
273	unsigned getDestIndex() const { return Index; }
274
275	// This should be used cautiously.
276	void setScopeDepth(EHScopeStack::stable_iterator depth) {
277	ScopeDepth = depth;
278	}
279
280	private:
281	llvm::BasicBlock *Block;
282	EHScopeStack::stable_iterator ScopeDepth;
283	unsigned Index;
284	};
285
286	CodeGenModule &CGM; // Per-module state.
287	const TargetInfo &Target;
288
289	// For EH/SEH outlined funclets, this field points to parent's CGF
290	CodeGenFunction *ParentCGF = nullptr;
291
292	typedef std::pair<llvm::Value *, llvm::Value *> ComplexPairTy;
293	LoopInfoStack LoopStack;
294	CGBuilderTy Builder;
295
296	// Stores variables for which we can't generate correct lifetime markers
297	// because of jumps.
298	VarBypassDetector Bypasses;
299
300	/// List of recently emitted OMPCanonicalLoops.
301	///
302	/// Since OMPCanonicalLoops are nested inside other statements (in particular
303	/// CapturedStmt generated by OMPExecutableDirective and non-perfectly nested
304	/// loops), we cannot directly call OMPEmitOMPCanonicalLoop and receive its
305	/// llvm::CanonicalLoopInfo. Instead, we call EmitStmt and any
306	/// OMPEmitOMPCanonicalLoop called by it will add its CanonicalLoopInfo to
307	/// this stack when done. Entering a new loop requires clearing this list; it
308	/// either means we start parsing a new loop nest (in which case the previous
309	/// loop nest goes out of scope) or a second loop in the same level in which
310	/// case it would be ambiguous into which of the two (or more) loops the loop
311	/// nest would extend.
312	SmallVector<llvm::CanonicalLoopInfo *, 4> OMPLoopNestStack;
313
314	/// Stack to track the Logical Operator recursion nest for MC/DC.
315	SmallVector<const BinaryOperator *, 16> MCDCLogOpStack;
316
317	/// Number of nested loop to be consumed by the last surrounding
318	/// loop-associated directive.
319	int ExpectedOMPLoopDepth = 0;
320
321	// CodeGen lambda for loops and support for ordered clause
322	typedef llvm::function_ref<void(CodeGenFunction &, const OMPLoopDirective &,
323	JumpDest)>
324	CodeGenLoopTy;
325	typedef llvm::function_ref<void(CodeGenFunction &, SourceLocation,
326	const unsigned, const bool)>
327	CodeGenOrderedTy;
328
329	// Codegen lambda for loop bounds in worksharing loop constructs
330	typedef llvm::function_ref<std::pair<LValue, LValue>(
331	CodeGenFunction &, const OMPExecutableDirective &S)>
332	CodeGenLoopBoundsTy;
333
334	// Codegen lambda for loop bounds in dispatch-based loop implementation
335	typedef llvm::function_ref<std::pair<llvm::Value *, llvm::Value *>(
336	CodeGenFunction &, const OMPExecutableDirective &S, Address LB,
337	Address UB)>
338	CodeGenDispatchBoundsTy;
339
340	/// CGBuilder insert helper. This function is called after an
341	/// instruction is created using Builder.
342	void InsertHelper(llvm::Instruction *I, const llvm::Twine &Name,
343	llvm::BasicBlock *BB,
344	llvm::BasicBlock::iterator InsertPt) const;
345
346	/// CurFuncDecl - Holds the Decl for the current outermost
347	/// non-closure context.
348	const Decl *CurFuncDecl = nullptr;
349	/// CurCodeDecl - This is the inner-most code context, which includes blocks.
350	const Decl *CurCodeDecl = nullptr;
351	const CGFunctionInfo *CurFnInfo = nullptr;
352	QualType FnRetTy;
353	llvm::Function *CurFn = nullptr;
354
355	/// Save Parameter Decl for coroutine.
356	llvm::SmallVector<const ParmVarDecl *, 4> FnArgs;
357
358	// Holds coroutine data if the current function is a coroutine. We use a
359	// wrapper to manage its lifetime, so that we don't have to define CGCoroData
360	// in this header.
361	struct CGCoroInfo {
362	std::unique_ptr<CGCoroData> Data;
363	bool InSuspendBlock = false;
364	CGCoroInfo();
365	~CGCoroInfo();
366	};
367	CGCoroInfo CurCoro;
368
369	bool isCoroutine() const {
370	return CurCoro.Data != nullptr;
371	}
372
373	bool inSuspendBlock() const {
374	return isCoroutine() && CurCoro.InSuspendBlock;
375	}
376
377	// Holds FramePtr for await_suspend wrapper generation,
378	// so that __builtin_coro_frame call can be lowered
379	// directly to value of its second argument
380	struct AwaitSuspendWrapperInfo {
381	llvm::Value *FramePtr = nullptr;
382	};
383	AwaitSuspendWrapperInfo CurAwaitSuspendWrapper;
384
385	// Generates wrapper function for `llvm.coro.await.suspend.*` intrinisics.
386	// It encapsulates SuspendExpr in a function, to separate it's body
387	// from the main coroutine to avoid miscompilations. Intrinisic
388	// is lowered to this function call in CoroSplit pass
389	// Function signature is:
390	// <type> __await_suspend_wrapper_<name>(ptr %awaiter, ptr %hdl)
391	// where type is one of (void, i1, ptr)
392	llvm::Function *generateAwaitSuspendWrapper(Twine const &CoroName,
393	Twine const &SuspendPointName,
394	CoroutineSuspendExpr const &S);
395
396	/// CurGD - The GlobalDecl for the current function being compiled.
397	GlobalDecl CurGD;
398
399	/// PrologueCleanupDepth - The cleanup depth enclosing all the
400	/// cleanups associated with the parameters.
401	EHScopeStack::stable_iterator PrologueCleanupDepth;
402
403	/// ReturnBlock - Unified return block.
404	JumpDest ReturnBlock;
405
406	/// ReturnValue - The temporary alloca to hold the return
407	/// value. This is invalid iff the function has no return value.
408	Address ReturnValue = Address::invalid();
409
410	/// ReturnValuePointer - The temporary alloca to hold a pointer to sret.
411	/// This is invalid if sret is not in use.
412	Address ReturnValuePointer = Address::invalid();
413
414	/// If a return statement is being visited, this holds the return statment's
415	/// result expression.
416	const Expr *RetExpr = nullptr;
417
418	/// Return true if a label was seen in the current scope.
419	bool hasLabelBeenSeenInCurrentScope() const {
420	if (CurLexicalScope)
421	return CurLexicalScope->hasLabels();
422	return !LabelMap.empty();
423	}
424
425	/// AllocaInsertPoint - This is an instruction in the entry block before which
426	/// we prefer to insert allocas.
427	llvm::AssertingVH<llvm::Instruction> AllocaInsertPt;
428
429	private:
430	/// PostAllocaInsertPt - This is a place in the prologue where code can be
431	/// inserted that will be dominated by all the static allocas. This helps
432	/// achieve two things:
433	/// 1. Contiguity of all static allocas (within the prologue) is maintained.
434	/// 2. All other prologue code (which are dominated by static allocas) do
435	/// appear in the source order immediately after all static allocas.
436	///
437	/// PostAllocaInsertPt will be lazily created when it is *really* required.
438	llvm::AssertingVH<llvm::Instruction> PostAllocaInsertPt = nullptr;
439
440	public:
441	/// Return PostAllocaInsertPt. If it is not yet created, then insert it
442	/// immediately after AllocaInsertPt.
443	llvm::Instruction *getPostAllocaInsertPoint() {
444	if (!PostAllocaInsertPt) {
445	assert(AllocaInsertPt &&
446	"Expected static alloca insertion point at function prologue");
447	assert(AllocaInsertPt->getParent()->isEntryBlock() &&
448	"EBB should be entry block of the current code gen function");
449	PostAllocaInsertPt = AllocaInsertPt->clone();
450	PostAllocaInsertPt->setName("postallocapt");
451	PostAllocaInsertPt->insertAfter(InsertPos: AllocaInsertPt);
452	}
453
454	return PostAllocaInsertPt;
455	}
456
457	/// API for captured statement code generation.
458	class CGCapturedStmtInfo {
459	public:
460	explicit CGCapturedStmtInfo(CapturedRegionKind K = CR_Default)
461	: Kind(K), ThisValue(nullptr), CXXThisFieldDecl(nullptr) {}
462	explicit CGCapturedStmtInfo(const CapturedStmt &S,
463	CapturedRegionKind K = CR_Default)
464	: Kind(K), ThisValue(nullptr), CXXThisFieldDecl(nullptr) {
465
466	RecordDecl::field_iterator Field =
467	S.getCapturedRecordDecl()->field_begin();
468	for (CapturedStmt::const_capture_iterator I = S.capture_begin(),
469	E = S.capture_end();
470	I != E; ++I, ++Field) {
471	if (I->capturesThis())
472	CXXThisFieldDecl = *Field;
473	else if (I->capturesVariable())
474	CaptureFields[I->getCapturedVar()->getCanonicalDecl()] = *Field;
475	else if (I->capturesVariableByCopy())
476	CaptureFields[I->getCapturedVar()->getCanonicalDecl()] = *Field;
477	}
478	}
479
480	virtual ~CGCapturedStmtInfo();
481
482	CapturedRegionKind getKind() const { return Kind; }
483
484	virtual void setContextValue(llvm::Value *V) { ThisValue = V; }
485	// Retrieve the value of the context parameter.
486	virtual llvm::Value *getContextValue() const { return ThisValue; }
487
488	/// Lookup the captured field decl for a variable.
489	virtual const FieldDecl *lookup(const VarDecl *VD) const {
490	return CaptureFields.lookup(Val: VD->getCanonicalDecl());
491	}
492
493	bool isCXXThisExprCaptured() const { return getThisFieldDecl() != nullptr; }
494	virtual FieldDecl *getThisFieldDecl() const { return CXXThisFieldDecl; }
495
496	static bool classof(const CGCapturedStmtInfo *) {
497	return true;
498	}
499
500	/// Emit the captured statement body.
501	virtual void EmitBody(CodeGenFunction &CGF, const Stmt *S) {
502	CGF.incrementProfileCounter(S);
503	CGF.EmitStmt(S);
504	}
505
506	/// Get the name of the capture helper.
507	virtual StringRef getHelperName() const { return "__captured_stmt"; }
508
509	/// Get the CaptureFields
510	llvm::SmallDenseMap<const VarDecl *, FieldDecl *> getCaptureFields() {
511	return CaptureFields;
512	}
513
514	private:
515	/// The kind of captured statement being generated.
516	CapturedRegionKind Kind;
517
518	/// Keep the map between VarDecl and FieldDecl.
519	llvm::SmallDenseMap<const VarDecl *, FieldDecl *> CaptureFields;
520
521	/// The base address of the captured record, passed in as the first
522	/// argument of the parallel region function.
523	llvm::Value *ThisValue;
524
525	/// Captured 'this' type.
526	FieldDecl *CXXThisFieldDecl;
527	};
528	CGCapturedStmtInfo *CapturedStmtInfo = nullptr;
529
530	/// RAII for correct setting/restoring of CapturedStmtInfo.
531	class CGCapturedStmtRAII {
532	private:
533	CodeGenFunction &CGF;
534	CGCapturedStmtInfo *PrevCapturedStmtInfo;
535	public:
536	CGCapturedStmtRAII(CodeGenFunction &CGF,
537	CGCapturedStmtInfo *NewCapturedStmtInfo)
538	: CGF(CGF), PrevCapturedStmtInfo(CGF.CapturedStmtInfo) {
539	CGF.CapturedStmtInfo = NewCapturedStmtInfo;
540	}
541	~CGCapturedStmtRAII() { CGF.CapturedStmtInfo = PrevCapturedStmtInfo; }
542	};
543
544	/// An abstract representation of regular/ObjC call/message targets.
545	class AbstractCallee {
546	/// The function declaration of the callee.
547	const Decl *CalleeDecl;
548
549	public:
550	AbstractCallee() : CalleeDecl(nullptr) {}
551	AbstractCallee(const FunctionDecl *FD) : CalleeDecl(FD) {}
552	AbstractCallee(const ObjCMethodDecl *OMD) : CalleeDecl(OMD) {}
553	bool hasFunctionDecl() const {
554	return isa_and_nonnull<FunctionDecl>(Val: CalleeDecl);
555	}
556	const Decl *getDecl() const { return CalleeDecl; }
557	unsigned getNumParams() const {
558	if (const auto *FD = dyn_cast<FunctionDecl>(Val: CalleeDecl))
559	return FD->getNumParams();
560	return cast<ObjCMethodDecl>(Val: CalleeDecl)->param_size();
561	}
562	const ParmVarDecl *getParamDecl(unsigned I) const {
563	if (const auto *FD = dyn_cast<FunctionDecl>(Val: CalleeDecl))
564	return FD->getParamDecl(i: I);
565	return *(cast<ObjCMethodDecl>(Val: CalleeDecl)->param_begin() + I);
566	}
567	};
568
569	/// Sanitizers enabled for this function.
570	SanitizerSet SanOpts;
571
572	/// True if CodeGen currently emits code implementing sanitizer checks.
573	bool IsSanitizerScope = false;
574
575	/// RAII object to set/unset CodeGenFunction::IsSanitizerScope.
576	class SanitizerScope {
577	CodeGenFunction *CGF;
578	public:
579	SanitizerScope(CodeGenFunction *CGF);
580	~SanitizerScope();
581	};
582
583	/// In C++, whether we are code generating a thunk. This controls whether we
584	/// should emit cleanups.
585	bool CurFuncIsThunk = false;
586
587	/// In ARC, whether we should autorelease the return value.
588	bool AutoreleaseResult = false;
589
590	/// Whether we processed a Microsoft-style asm block during CodeGen. These can
591	/// potentially set the return value.
592	bool SawAsmBlock = false;
593
594	GlobalDecl CurSEHParent;
595
596	/// True if the current function is an outlined SEH helper. This can be a
597	/// finally block or filter expression.
598	bool IsOutlinedSEHHelper = false;
599
600	/// True if CodeGen currently emits code inside presereved access index
601	/// region.
602	bool IsInPreservedAIRegion = false;
603
604	/// True if the current statement has nomerge attribute.
605	bool InNoMergeAttributedStmt = false;
606
607	/// True if the current statement has noinline attribute.
608	bool InNoInlineAttributedStmt = false;
609
610	/// True if the current statement has always_inline attribute.
611	bool InAlwaysInlineAttributedStmt = false;
612
613	// The CallExpr within the current statement that the musttail attribute
614	// applies to. nullptr if there is no 'musttail' on the current statement.
615	const CallExpr *MustTailCall = nullptr;
616
617	/// Returns true if a function must make progress, which means the
618	/// mustprogress attribute can be added.
619	bool checkIfFunctionMustProgress() {
620	if (CGM.getCodeGenOpts().getFiniteLoops() ==
621	CodeGenOptions::FiniteLoopsKind::Never)
622	return false;
623
624	// C++11 and later guarantees that a thread eventually will do one of the
625	// following (C++11 [intro.multithread]p24 and C++17 [intro.progress]p1):
626	// - terminate,
627	// - make a call to a library I/O function,
628	// - perform an access through a volatile glvalue, or
629	// - perform a synchronization operation or an atomic operation.
630	//
631	// Hence each function is 'mustprogress' in C++11 or later.
632	return getLangOpts().CPlusPlus11;
633	}
634
635	/// Returns true if a loop must make progress, which means the mustprogress
636	/// attribute can be added. \p HasConstantCond indicates whether the branch
637	/// condition is a known constant.
638	bool checkIfLoopMustProgress(bool HasConstantCond) {
639	if (CGM.getCodeGenOpts().getFiniteLoops() ==
640	CodeGenOptions::FiniteLoopsKind::Always)
641	return true;
642	if (CGM.getCodeGenOpts().getFiniteLoops() ==
643	CodeGenOptions::FiniteLoopsKind::Never)
644	return false;
645
646	// If the containing function must make progress, loops also must make
647	// progress (as in C++11 and later).
648	if (checkIfFunctionMustProgress())
649	return true;
650
651	// Now apply rules for plain C (see 6.8.5.6 in C11).
652	// Loops with constant conditions do not have to make progress in any C
653	// version.
654	if (HasConstantCond)
655	return false;
656
657	// Loops with non-constant conditions must make progress in C11 and later.
658	return getLangOpts().C11;
659	}
660
661	const CodeGen::CGBlockInfo *BlockInfo = nullptr;
662	llvm::Value *BlockPointer = nullptr;
663
664	llvm::DenseMap<const ValueDecl *, FieldDecl *> LambdaCaptureFields;
665	FieldDecl *LambdaThisCaptureField = nullptr;
666
667	/// A mapping from NRVO variables to the flags used to indicate
668	/// when the NRVO has been applied to this variable.
669	llvm::DenseMap<const VarDecl *, llvm::Value *> NRVOFlags;
670
671	EHScopeStack EHStack;
672	llvm::SmallVector<char, 256> LifetimeExtendedCleanupStack;
673	llvm::SmallVector<const JumpDest *, 2> SEHTryEpilogueStack;
674
675	llvm::Instruction *CurrentFuncletPad = nullptr;
676
677	class CallLifetimeEnd final : public EHScopeStack::Cleanup {
678	bool isRedundantBeforeReturn() override { return true; }
679
680	llvm::Value *Addr;
681	llvm::Value *Size;
682
683	public:
684	CallLifetimeEnd(RawAddress addr, llvm::Value *size)
685	: Addr(addr.getPointer()), Size(size) {}
686
687	void Emit(CodeGenFunction &CGF, Flags flags) override {
688	CGF.EmitLifetimeEnd(Size, Addr);
689	}
690	};
691
692	/// Header for data within LifetimeExtendedCleanupStack.
693	struct LifetimeExtendedCleanupHeader {
694	/// The size of the following cleanup object.
695	unsigned Size;
696	/// The kind of cleanup to push.
697	LLVM_PREFERRED_TYPE(CleanupKind)
698	unsigned Kind : 31;
699	/// Whether this is a conditional cleanup.
700	LLVM_PREFERRED_TYPE(bool)
701	unsigned IsConditional : 1;
702
703	size_t getSize() const { return Size; }
704	CleanupKind getKind() const { return (CleanupKind)Kind; }
705	bool isConditional() const { return IsConditional; }
706	};
707
708	/// i32s containing the indexes of the cleanup destinations.
709	RawAddress NormalCleanupDest = RawAddress::invalid();
710
711	unsigned NextCleanupDestIndex = 1;
712
713	/// EHResumeBlock - Unified block containing a call to llvm.eh.resume.
714	llvm::BasicBlock *EHResumeBlock = nullptr;
715
716	/// The exception slot. All landing pads write the current exception pointer
717	/// into this alloca.
718	llvm::Value *ExceptionSlot = nullptr;
719
720	/// The selector slot. Under the MandatoryCleanup model, all landing pads
721	/// write the current selector value into this alloca.
722	llvm::AllocaInst *EHSelectorSlot = nullptr;
723
724	/// A stack of exception code slots. Entering an __except block pushes a slot
725	/// on the stack and leaving pops one. The __exception_code() intrinsic loads
726	/// a value from the top of the stack.
727	SmallVector<Address, 1> SEHCodeSlotStack;
728
729	/// Value returned by __exception_info intrinsic.
730	llvm::Value *SEHInfo = nullptr;
731
732	/// Emits a landing pad for the current EH stack.
733	llvm::BasicBlock *EmitLandingPad();
734
735	llvm::BasicBlock *getInvokeDestImpl();
736
737	/// Parent loop-based directive for scan directive.
738	const OMPExecutableDirective *OMPParentLoopDirectiveForScan = nullptr;
739	llvm::BasicBlock *OMPBeforeScanBlock = nullptr;
740	llvm::BasicBlock *OMPAfterScanBlock = nullptr;
741	llvm::BasicBlock *OMPScanExitBlock = nullptr;
742	llvm::BasicBlock *OMPScanDispatch = nullptr;
743	bool OMPFirstScanLoop = false;
744
745	/// Manages parent directive for scan directives.
746	class ParentLoopDirectiveForScanRegion {
747	CodeGenFunction &CGF;
748	const OMPExecutableDirective *ParentLoopDirectiveForScan;
749
750	public:
751	ParentLoopDirectiveForScanRegion(
752	CodeGenFunction &CGF,
753	const OMPExecutableDirective &ParentLoopDirectiveForScan)
754	: CGF(CGF),
755	ParentLoopDirectiveForScan(CGF.OMPParentLoopDirectiveForScan) {
756	CGF.OMPParentLoopDirectiveForScan = &ParentLoopDirectiveForScan;
757	}
758	~ParentLoopDirectiveForScanRegion() {
759	CGF.OMPParentLoopDirectiveForScan = ParentLoopDirectiveForScan;
760	}
761	};
762
763	template <class T>
764	typename DominatingValue<T>::saved_type saveValueInCond(T value) {
765	return DominatingValue<T>::save(*this, value);
766	}
767
768	class CGFPOptionsRAII {
769	public:
770	CGFPOptionsRAII(CodeGenFunction &CGF, FPOptions FPFeatures);
771	CGFPOptionsRAII(CodeGenFunction &CGF, const Expr *E);
772	~CGFPOptionsRAII();
773
774	private:
775	void ConstructorHelper(FPOptions FPFeatures);
776	CodeGenFunction &CGF;
777	FPOptions OldFPFeatures;
778	llvm::fp::ExceptionBehavior OldExcept;
779	llvm::RoundingMode OldRounding;
780	std::optional<CGBuilderTy::FastMathFlagGuard> FMFGuard;
781	};
782	FPOptions CurFPFeatures;
783
784	public:
785	/// ObjCEHValueStack - Stack of Objective-C exception values, used for
786	/// rethrows.
787	SmallVector<llvm::Value*, 8> ObjCEHValueStack;
788
789	/// A class controlling the emission of a finally block.
790	class FinallyInfo {
791	/// Where the catchall's edge through the cleanup should go.
792	JumpDest RethrowDest;
793
794	/// A function to call to enter the catch.
795	llvm::FunctionCallee BeginCatchFn;
796
797	/// An i1 variable indicating whether or not the @finally is
798	/// running for an exception.
799	llvm::AllocaInst *ForEHVar = nullptr;
800
801	/// An i8* variable into which the exception pointer to rethrow
802	/// has been saved.
803	llvm::AllocaInst *SavedExnVar = nullptr;
804
805	public:
806	void enter(CodeGenFunction &CGF, const Stmt *Finally,
807	llvm::FunctionCallee beginCatchFn,
808	llvm::FunctionCallee endCatchFn, llvm::FunctionCallee rethrowFn);
809	void exit(CodeGenFunction &CGF);
810	};
811
812	/// Returns true inside SEH __try blocks.
813	bool isSEHTryScope() const { return !SEHTryEpilogueStack.empty(); }
814
815	/// Returns true while emitting a cleanuppad.
816	bool isCleanupPadScope() const {
817	return CurrentFuncletPad && isa<llvm::CleanupPadInst>(Val: CurrentFuncletPad);
818	}
819
820	/// pushFullExprCleanup - Push a cleanup to be run at the end of the
821	/// current full-expression. Safe against the possibility that
822	/// we're currently inside a conditionally-evaluated expression.
823	template <class T, class... As>
824	void pushFullExprCleanup(CleanupKind kind, As... A) {
825	// If we're not in a conditional branch, or if none of the
826	// arguments requires saving, then use the unconditional cleanup.
827	if (!isInConditionalBranch())
828	return EHStack.pushCleanup<T>(kind, A...);
829
830	// Stash values in a tuple so we can guarantee the order of saves.
831	typedef std::tuple<typename DominatingValue<As>::saved_type...> SavedTuple;
832	SavedTuple Saved{saveValueInCond(A)...};
833
834	typedef EHScopeStack::ConditionalCleanup<T, As...> CleanupType;
835	EHStack.pushCleanupTuple<CleanupType>(kind, Saved);
836	initFullExprCleanup();
837	}
838
839	/// Queue a cleanup to be pushed after finishing the current full-expression,
840	/// potentially with an active flag.
841	template <class T, class... As>
842	void pushCleanupAfterFullExpr(CleanupKind Kind, As... A) {
843	if (!isInConditionalBranch())
844	return pushCleanupAfterFullExprWithActiveFlag<T>(
845	Kind, RawAddress::invalid(), A...);
846
847	RawAddress ActiveFlag = createCleanupActiveFlag();
848	assert(!DominatingValue<Address>::needsSaving(ActiveFlag) &&
849	"cleanup active flag should never need saving");
850
851	typedef std::tuple<typename DominatingValue<As>::saved_type...> SavedTuple;
852	SavedTuple Saved{saveValueInCond(A)...};
853
854	typedef EHScopeStack::ConditionalCleanup<T, As...> CleanupType;
855	pushCleanupAfterFullExprWithActiveFlag<CleanupType>(Kind, ActiveFlag, Saved);
856	}
857
858	template <class T, class... As>
859	void pushCleanupAfterFullExprWithActiveFlag(CleanupKind Kind,
860	RawAddress ActiveFlag, As... A) {
861	LifetimeExtendedCleanupHeader Header = {.Size: sizeof(T), .Kind: Kind,
862	.IsConditional: ActiveFlag.isValid()};
863
864	size_t OldSize = LifetimeExtendedCleanupStack.size();
865	LifetimeExtendedCleanupStack.resize(
866	N: LifetimeExtendedCleanupStack.size() + sizeof(Header) + Header.Size +
867	(Header.IsConditional ? sizeof(ActiveFlag) : 0));
868
869	static_assert(sizeof(Header) % alignof(T) == 0,
870	"Cleanup will be allocated on misaligned address");
871	char *Buffer = &LifetimeExtendedCleanupStack[OldSize];
872	new (Buffer) LifetimeExtendedCleanupHeader(Header);
873	new (Buffer + sizeof(Header)) T(A...);
874	if (Header.IsConditional)
875	new (Buffer + sizeof(Header) + sizeof(T)) RawAddress(ActiveFlag);
876	}
877
878	/// Set up the last cleanup that was pushed as a conditional
879	/// full-expression cleanup.
880	void initFullExprCleanup() {
881	initFullExprCleanupWithFlag(ActiveFlag: createCleanupActiveFlag());
882	}
883
884	void initFullExprCleanupWithFlag(RawAddress ActiveFlag);
885	RawAddress createCleanupActiveFlag();
886
887	/// PushDestructorCleanup - Push a cleanup to call the
888	/// complete-object destructor of an object of the given type at the
889	/// given address. Does nothing if T is not a C++ class type with a
890	/// non-trivial destructor.
891	void PushDestructorCleanup(QualType T, Address Addr);
892
893	/// PushDestructorCleanup - Push a cleanup to call the
894	/// complete-object variant of the given destructor on the object at
895	/// the given address.
896	void PushDestructorCleanup(const CXXDestructorDecl *Dtor, QualType T,
897	Address Addr);
898
899	/// PopCleanupBlock - Will pop the cleanup entry on the stack and
900	/// process all branch fixups.
901	void PopCleanupBlock(bool FallThroughIsBranchThrough = false);
902
903	/// DeactivateCleanupBlock - Deactivates the given cleanup block.
904	/// The block cannot be reactivated. Pops it if it's the top of the
905	/// stack.
906	///
907	/// \param DominatingIP - An instruction which is known to
908	/// dominate the current IP (if set) and which lies along
909	/// all paths of execution between the current IP and the
910	/// the point at which the cleanup comes into scope.
911	void DeactivateCleanupBlock(EHScopeStack::stable_iterator Cleanup,
912	llvm::Instruction *DominatingIP);
913
914	/// ActivateCleanupBlock - Activates an initially-inactive cleanup.
915	/// Cannot be used to resurrect a deactivated cleanup.
916	///
917	/// \param DominatingIP - An instruction which is known to
918	/// dominate the current IP (if set) and which lies along
919	/// all paths of execution between the current IP and the
920	/// the point at which the cleanup comes into scope.
921	void ActivateCleanupBlock(EHScopeStack::stable_iterator Cleanup,
922	llvm::Instruction *DominatingIP);
923
924	/// Enters a new scope for capturing cleanups, all of which
925	/// will be executed once the scope is exited.
926	class RunCleanupsScope {
927	EHScopeStack::stable_iterator CleanupStackDepth, OldCleanupScopeDepth;
928	size_t LifetimeExtendedCleanupStackSize;
929	bool OldDidCallStackSave;
930	protected:
931	bool PerformCleanup;
932	private:
933
934	RunCleanupsScope(const RunCleanupsScope &) = delete;
935	void operator=(const RunCleanupsScope &) = delete;
936
937	protected:
938	CodeGenFunction& CGF;
939
940	public:
941	/// Enter a new cleanup scope.
942	explicit RunCleanupsScope(CodeGenFunction &CGF)
943	: PerformCleanup(true), CGF(CGF)
944	{
945	CleanupStackDepth = CGF.EHStack.stable_begin();
946	LifetimeExtendedCleanupStackSize =
947	CGF.LifetimeExtendedCleanupStack.size();
948	OldDidCallStackSave = CGF.DidCallStackSave;
949	CGF.DidCallStackSave = false;
950	OldCleanupScopeDepth = CGF.CurrentCleanupScopeDepth;
951	CGF.CurrentCleanupScopeDepth = CleanupStackDepth;
952	}
953
954	/// Exit this cleanup scope, emitting any accumulated cleanups.
955	~RunCleanupsScope() {
956	if (PerformCleanup)
957	ForceCleanup();
958	}
959
960	/// Determine whether this scope requires any cleanups.
961	bool requiresCleanups() const {
962	return CGF.EHStack.stable_begin() != CleanupStackDepth;
963	}
964
965	/// Force the emission of cleanups now, instead of waiting
966	/// until this object is destroyed.
967	/// \param ValuesToReload - A list of values that need to be available at
968	/// the insertion point after cleanup emission. If cleanup emission created
969	/// a shared cleanup block, these value pointers will be rewritten.
970	/// Otherwise, they not will be modified.
971	void ForceCleanup(std::initializer_list<llvm::Value**> ValuesToReload = {}) {
972	assert(PerformCleanup && "Already forced cleanup");
973	CGF.DidCallStackSave = OldDidCallStackSave;
974	CGF.PopCleanupBlocks(OldCleanupStackSize: CleanupStackDepth, OldLifetimeExtendedStackSize: LifetimeExtendedCleanupStackSize,
975	ValuesToReload);
976	PerformCleanup = false;
977	CGF.CurrentCleanupScopeDepth = OldCleanupScopeDepth;
978	}
979	};
980
981	// Cleanup stack depth of the RunCleanupsScope that was pushed most recently.
982	EHScopeStack::stable_iterator CurrentCleanupScopeDepth =
983	EHScopeStack::stable_end();
984
985	class LexicalScope : public RunCleanupsScope {
986	SourceRange Range;
987	SmallVector<const LabelDecl*, 4> Labels;
988	LexicalScope *ParentScope;
989
990	LexicalScope(const LexicalScope &) = delete;
991	void operator=(const LexicalScope &) = delete;
992
993	public:
994	/// Enter a new cleanup scope.
995	explicit LexicalScope(CodeGenFunction &CGF, SourceRange Range)
996	: RunCleanupsScope(CGF), Range(Range), ParentScope(CGF.CurLexicalScope) {
997	CGF.CurLexicalScope = this;
998	if (CGDebugInfo *DI = CGF.getDebugInfo())
999	DI->EmitLexicalBlockStart(Builder&: CGF.Builder, Loc: Range.getBegin());
1000	}
1001
1002	void addLabel(const LabelDecl *label) {
1003	assert(PerformCleanup && "adding label to dead scope?");
1004	Labels.push_back(Elt: label);
1005	}
1006
1007	/// Exit this cleanup scope, emitting any accumulated
1008	/// cleanups.
1009	~LexicalScope() {
1010	if (CGDebugInfo *DI = CGF.getDebugInfo())
1011	DI->EmitLexicalBlockEnd(Builder&: CGF.Builder, Loc: Range.getEnd());
1012
1013	// If we should perform a cleanup, force them now. Note that
1014	// this ends the cleanup scope before rescoping any labels.
1015	if (PerformCleanup) {
1016	ApplyDebugLocation DL(CGF, Range.getEnd());
1017	ForceCleanup();
1018	}
1019	}
1020
1021	/// Force the emission of cleanups now, instead of waiting
1022	/// until this object is destroyed.
1023	void ForceCleanup() {
1024	CGF.CurLexicalScope = ParentScope;
1025	RunCleanupsScope::ForceCleanup();
1026
1027	if (!Labels.empty())
1028	rescopeLabels();
1029	}
1030
1031	bool hasLabels() const {
1032	return !Labels.empty();
1033	}
1034
1035	void rescopeLabels();
1036	};
1037
1038	typedef llvm::DenseMap<const Decl *, Address> DeclMapTy;
1039
1040	/// The class used to assign some variables some temporarily addresses.
1041	class OMPMapVars {
1042	DeclMapTy SavedLocals;
1043	DeclMapTy SavedTempAddresses;
1044	OMPMapVars(const OMPMapVars &) = delete;
1045	void operator=(const OMPMapVars &) = delete;
1046
1047	public:
1048	explicit OMPMapVars() = default;
1049	~OMPMapVars() {
1050	assert(SavedLocals.empty() && "Did not restored original addresses.");
1051	};
1052
1053	/// Sets the address of the variable \p LocalVD to be \p TempAddr in
1054	/// function \p CGF.
1055	/// \return true if at least one variable was set already, false otherwise.
1056	bool setVarAddr(CodeGenFunction &CGF, const VarDecl *LocalVD,
1057	Address TempAddr) {
1058	LocalVD = LocalVD->getCanonicalDecl();
1059	// Only save it once.
1060	if (SavedLocals.count(LocalVD)) return false;
1061
1062	// Copy the existing local entry to SavedLocals.
1063	auto it = CGF.LocalDeclMap.find(LocalVD);
1064	if (it != CGF.LocalDeclMap.end())
1065	SavedLocals.try_emplace(LocalVD, it->second);
1066	else
1067	SavedLocals.try_emplace(LocalVD, Address::invalid());
1068
1069	// Generate the private entry.
1070	QualType VarTy = LocalVD->getType();
1071	if (VarTy->isReferenceType()) {
1072	Address Temp = CGF.CreateMemTemp(T: VarTy);
1073	CGF.Builder.CreateStore(Val: TempAddr.emitRawPointer(CGF), Addr: Temp);
1074	TempAddr = Temp;
1075	}
1076	SavedTempAddresses.try_emplace(LocalVD, TempAddr);
1077
1078	return true;
1079	}
1080
1081	/// Applies new addresses to the list of the variables.
1082	/// \return true if at least one variable is using new address, false
1083	/// otherwise.
1084	bool apply(CodeGenFunction &CGF) {
1085	copyInto(Src: SavedTempAddresses, Dest&: CGF.LocalDeclMap);
1086	SavedTempAddresses.clear();
1087	return !SavedLocals.empty();
1088	}
1089
1090	/// Restores original addresses of the variables.
1091	void restore(CodeGenFunction &CGF) {
1092	if (!SavedLocals.empty()) {
1093	copyInto(Src: SavedLocals, Dest&: CGF.LocalDeclMap);
1094	SavedLocals.clear();
1095	}
1096	}
1097
1098	private:
1099	/// Copy all the entries in the source map over the corresponding
1100	/// entries in the destination, which must exist.
1101	static void copyInto(const DeclMapTy &Src, DeclMapTy &Dest) {
1102	for (auto &Pair : Src) {
1103	if (!Pair.second.isValid()) {
1104	Dest.erase(Val: Pair.first);
1105	continue;
1106	}
1107
1108	auto I = Dest.find(Val: Pair.first);
1109	if (I != Dest.end())
1110	I->second = Pair.second;
1111	else
1112	Dest.insert(KV: Pair);
1113	}
1114	}
1115	};
1116
1117	/// The scope used to remap some variables as private in the OpenMP loop body
1118	/// (or other captured region emitted without outlining), and to restore old
1119	/// vars back on exit.
1120	class OMPPrivateScope : public RunCleanupsScope {
1121	OMPMapVars MappedVars;
1122	OMPPrivateScope(const OMPPrivateScope &) = delete;
1123	void operator=(const OMPPrivateScope &) = delete;
1124
1125	public:
1126	/// Enter a new OpenMP private scope.
1127	explicit OMPPrivateScope(CodeGenFunction &CGF) : RunCleanupsScope(CGF) {}
1128
1129	/// Registers \p LocalVD variable as a private with \p Addr as the address
1130	/// of the corresponding private variable. \p
1131	/// PrivateGen is the address of the generated private variable.
1132	/// \return true if the variable is registered as private, false if it has
1133	/// been privatized already.
1134	bool addPrivate(const VarDecl *LocalVD, Address Addr) {
1135	assert(PerformCleanup && "adding private to dead scope");
1136	return MappedVars.setVarAddr(CGF, LocalVD, TempAddr: Addr);
1137	}
1138
1139	/// Privatizes local variables previously registered as private.
1140	/// Registration is separate from the actual privatization to allow
1141	/// initializers use values of the original variables, not the private one.
1142	/// This is important, for example, if the private variable is a class
1143	/// variable initialized by a constructor that references other private
1144	/// variables. But at initialization original variables must be used, not
1145	/// private copies.
1146	/// \return true if at least one variable was privatized, false otherwise.
1147	bool Privatize() { return MappedVars.apply(CGF); }
1148
1149	void ForceCleanup() {
1150	RunCleanupsScope::ForceCleanup();
1151	restoreMap();
1152	}
1153
1154	/// Exit scope - all the mapped variables are restored.
1155	~OMPPrivateScope() {
1156	if (PerformCleanup)
1157	ForceCleanup();
1158	}
1159
1160	/// Checks if the global variable is captured in current function.
1161	bool isGlobalVarCaptured(const VarDecl *VD) const {
1162	VD = VD->getCanonicalDecl();
1163	return !VD->isLocalVarDeclOrParm() && CGF.LocalDeclMap.count(VD) > 0;
1164	}
1165
1166	/// Restore all mapped variables w/o clean up. This is usefully when we want
1167	/// to reference the original variables but don't want the clean up because
1168	/// that could emit lifetime end too early, causing backend issue #56913.
1169	void restoreMap() { MappedVars.restore(CGF); }
1170	};
1171
1172	/// Save/restore original map of previously emitted local vars in case when we
1173	/// need to duplicate emission of the same code several times in the same
1174	/// function for OpenMP code.
1175	class OMPLocalDeclMapRAII {
1176	CodeGenFunction &CGF;
1177	DeclMapTy SavedMap;
1178
1179	public:
1180	OMPLocalDeclMapRAII(CodeGenFunction &CGF)
1181	: CGF(CGF), SavedMap(CGF.LocalDeclMap) {}
1182	~OMPLocalDeclMapRAII() { SavedMap.swap(RHS&: CGF.LocalDeclMap); }
1183	};
1184
1185	/// Takes the old cleanup stack size and emits the cleanup blocks
1186	/// that have been added.
1187	void
1188	PopCleanupBlocks(EHScopeStack::stable_iterator OldCleanupStackSize,
1189	std::initializer_list<llvm::Value **> ValuesToReload = {});
1190
1191	/// Takes the old cleanup stack size and emits the cleanup blocks
1192	/// that have been added, then adds all lifetime-extended cleanups from
1193	/// the given position to the stack.
1194	void
1195	PopCleanupBlocks(EHScopeStack::stable_iterator OldCleanupStackSize,
1196	size_t OldLifetimeExtendedStackSize,
1197	std::initializer_list<llvm::Value **> ValuesToReload = {});
1198
1199	void ResolveBranchFixups(llvm::BasicBlock *Target);
1200
1201	/// The given basic block lies in the current EH scope, but may be a
1202	/// target of a potentially scope-crossing jump; get a stable handle
1203	/// to which we can perform this jump later.
1204	JumpDest getJumpDestInCurrentScope(llvm::BasicBlock *Target) {
1205	return JumpDest(Target,
1206	EHStack.getInnermostNormalCleanup(),
1207	NextCleanupDestIndex++);
1208	}
1209
1210	/// The given basic block lies in the current EH scope, but may be a
1211	/// target of a potentially scope-crossing jump; get a stable handle
1212	/// to which we can perform this jump later.
1213	JumpDest getJumpDestInCurrentScope(StringRef Name = StringRef()) {
1214	return getJumpDestInCurrentScope(Target: createBasicBlock(name: Name));
1215	}
1216
1217	/// EmitBranchThroughCleanup - Emit a branch from the current insert
1218	/// block through the normal cleanup handling code (if any) and then
1219	/// on to \arg Dest.
1220	void EmitBranchThroughCleanup(JumpDest Dest);
1221
1222	/// isObviouslyBranchWithoutCleanups - Return true if a branch to the
1223	/// specified destination obviously has no cleanups to run. 'false' is always
1224	/// a conservatively correct answer for this method.
1225	bool isObviouslyBranchWithoutCleanups(JumpDest Dest) const;
1226
1227	/// popCatchScope - Pops the catch scope at the top of the EHScope
1228	/// stack, emitting any required code (other than the catch handlers
1229	/// themselves).
1230	void popCatchScope();
1231
1232	llvm::BasicBlock *getEHResumeBlock(bool isCleanup);
1233	llvm::BasicBlock *getEHDispatchBlock(EHScopeStack::stable_iterator scope);
1234	llvm::BasicBlock *
1235	getFuncletEHDispatchBlock(EHScopeStack::stable_iterator scope);
1236
1237	/// An object to manage conditionally-evaluated expressions.
1238	class ConditionalEvaluation {
1239	llvm::BasicBlock *StartBB;
1240
1241	public:
1242	ConditionalEvaluation(CodeGenFunction &CGF)
1243	: StartBB(CGF.Builder.GetInsertBlock()) {}
1244
1245	void begin(CodeGenFunction &CGF) {
1246	assert(CGF.OutermostConditional != this);
1247	if (!CGF.OutermostConditional)
1248	CGF.OutermostConditional = this;
1249	}
1250
1251	void end(CodeGenFunction &CGF) {
1252	assert(CGF.OutermostConditional != nullptr);
1253	if (CGF.OutermostConditional == this)
1254	CGF.OutermostConditional = nullptr;
1255	}
1256
1257	/// Returns a block which will be executed prior to each
1258	/// evaluation of the conditional code.
1259	llvm::BasicBlock *getStartingBlock() const {
1260	return StartBB;
1261	}
1262	};
1263
1264	/// isInConditionalBranch - Return true if we're currently emitting
1265	/// one branch or the other of a conditional expression.
1266	bool isInConditionalBranch() const { return OutermostConditional != nullptr; }
1267
1268	void setBeforeOutermostConditional(llvm::Value *value, Address addr,
1269	CodeGenFunction &CGF) {
1270	assert(isInConditionalBranch());
1271	llvm::BasicBlock *block = OutermostConditional->getStartingBlock();
1272	auto store =
1273	new llvm::StoreInst(value, addr.emitRawPointer(CGF), &block->back());
1274	store->setAlignment(addr.getAlignment().getAsAlign());
1275	}
1276
1277	/// An RAII object to record that we're evaluating a statement
1278	/// expression.
1279	class StmtExprEvaluation {
1280	CodeGenFunction &CGF;
1281
1282	/// We have to save the outermost conditional: cleanups in a
1283	/// statement expression aren't conditional just because the
1284	/// StmtExpr is.
1285	ConditionalEvaluation *SavedOutermostConditional;
1286
1287	public:
1288	StmtExprEvaluation(CodeGenFunction &CGF)
1289	: CGF(CGF), SavedOutermostConditional(CGF.OutermostConditional) {
1290	CGF.OutermostConditional = nullptr;
1291	}
1292
1293	~StmtExprEvaluation() {
1294	CGF.OutermostConditional = SavedOutermostConditional;
1295	CGF.EnsureInsertPoint();
1296	}
1297	};
1298
1299	/// An object which temporarily prevents a value from being
1300	/// destroyed by aggressive peephole optimizations that assume that
1301	/// all uses of a value have been realized in the IR.
1302	class PeepholeProtection {
1303	llvm::Instruction *Inst = nullptr;
1304	friend class CodeGenFunction;
1305
1306	public:
1307	PeepholeProtection() = default;
1308	};
1309
1310	/// A non-RAII class containing all the information about a bound
1311	/// opaque value. OpaqueValueMapping, below, is a RAII wrapper for
1312	/// this which makes individual mappings very simple; using this
1313	/// class directly is useful when you have a variable number of
1314	/// opaque values or don't want the RAII functionality for some
1315	/// reason.
1316	class OpaqueValueMappingData {
1317	const OpaqueValueExpr *OpaqueValue;
1318	bool BoundLValue;
1319	CodeGenFunction::PeepholeProtection Protection;
1320
1321	OpaqueValueMappingData(const OpaqueValueExpr *ov,
1322	bool boundLValue)
1323	: OpaqueValue(ov), BoundLValue(boundLValue) {}
1324	public:
1325	OpaqueValueMappingData() : OpaqueValue(nullptr) {}
1326
1327	static bool shouldBindAsLValue(const Expr *expr) {
1328	// gl-values should be bound as l-values for obvious reasons.
1329	// Records should be bound as l-values because IR generation
1330	// always keeps them in memory. Expressions of function type
1331	// act exactly like l-values but are formally required to be
1332	// r-values in C.
1333	return expr->isGLValue() ||
1334	expr->getType()->isFunctionType() ||
1335	hasAggregateEvaluationKind(T: expr->getType());
1336	}
1337
1338	static OpaqueValueMappingData bind(CodeGenFunction &CGF,
1339	const OpaqueValueExpr *ov,
1340	const Expr *e) {
1341	if (shouldBindAsLValue(ov))
1342	return bind(CGF, ov, lv: CGF.EmitLValue(E: e));
1343	return bind(CGF, ov, rv: CGF.EmitAnyExpr(E: e));
1344	}
1345
1346	static OpaqueValueMappingData bind(CodeGenFunction &CGF,
1347	const OpaqueValueExpr *ov,
1348	const LValue &lv) {
1349	assert(shouldBindAsLValue(ov));
1350	CGF.OpaqueLValues.insert(KV: std::make_pair(x&: ov, y: lv));
1351	return OpaqueValueMappingData(ov, true);
1352	}
1353
1354	static OpaqueValueMappingData bind(CodeGenFunction &CGF,
1355	const OpaqueValueExpr *ov,
1356	const RValue &rv) {
1357	assert(!shouldBindAsLValue(ov));
1358	CGF.OpaqueRValues.insert(KV: std::make_pair(x&: ov, y: rv));
1359
1360	OpaqueValueMappingData data(ov, false);
1361
1362	// Work around an extremely aggressive peephole optimization in
1363	// EmitScalarConversion which assumes that all other uses of a
1364	// value are extant.
1365	data.Protection = CGF.protectFromPeepholes(rvalue: rv);
1366
1367	return data;
1368	}
1369
1370	bool isValid() const { return OpaqueValue != nullptr; }
1371	void clear() { OpaqueValue = nullptr; }
1372
1373	void unbind(CodeGenFunction &CGF) {
1374	assert(OpaqueValue && "no data to unbind!");
1375
1376	if (BoundLValue) {
1377	CGF.OpaqueLValues.erase(Val: OpaqueValue);
1378	} else {
1379	CGF.OpaqueRValues.erase(Val: OpaqueValue);
1380	CGF.unprotectFromPeepholes(protection: Protection);
1381	}
1382	}
1383	};
1384
1385	/// An RAII object to set (and then clear) a mapping for an OpaqueValueExpr.
1386	class OpaqueValueMapping {
1387	CodeGenFunction &CGF;
1388	OpaqueValueMappingData Data;
1389
1390	public:
1391	static bool shouldBindAsLValue(const Expr *expr) {
1392	return OpaqueValueMappingData::shouldBindAsLValue(expr);
1393	}
1394
1395	/// Build the opaque value mapping for the given conditional
1396	/// operator if it's the GNU ?: extension. This is a common
1397	/// enough pattern that the convenience operator is really
1398	/// helpful.
1399	///
1400	OpaqueValueMapping(CodeGenFunction &CGF,
1401	const AbstractConditionalOperator *op) : CGF(CGF) {
1402	if (isa<ConditionalOperator>(Val: op))
1403	// Leave Data empty.
1404	return;
1405
1406	const BinaryConditionalOperator *e = cast<BinaryConditionalOperator>(Val: op);
1407	Data = OpaqueValueMappingData::bind(CGF, ov: e->getOpaqueValue(),
1408	e: e->getCommon());
1409	}
1410
1411	/// Build the opaque value mapping for an OpaqueValueExpr whose source
1412	/// expression is set to the expression the OVE represents.
1413	OpaqueValueMapping(CodeGenFunction &CGF, const OpaqueValueExpr *OV)
1414	: CGF(CGF) {
1415	if (OV) {
1416	assert(OV->getSourceExpr() && "wrong form of OpaqueValueMapping used "
1417	"for OVE with no source expression");
1418	Data = OpaqueValueMappingData::bind(CGF, ov: OV, e: OV->getSourceExpr());
1419	}
1420	}
1421
1422	OpaqueValueMapping(CodeGenFunction &CGF,
1423	const OpaqueValueExpr *opaqueValue,
1424	LValue lvalue)
1425	: CGF(CGF), Data(OpaqueValueMappingData::bind(CGF, ov: opaqueValue, lv: lvalue)) {
1426	}
1427
1428	OpaqueValueMapping(CodeGenFunction &CGF,
1429	const OpaqueValueExpr *opaqueValue,
1430	RValue rvalue)
1431	: CGF(CGF), Data(OpaqueValueMappingData::bind(CGF, ov: opaqueValue, rv: rvalue)) {
1432	}
1433
1434	void pop() {
1435	Data.unbind(CGF);
1436	Data.clear();
1437	}
1438
1439	~OpaqueValueMapping() {
1440	if (Data.isValid()) Data.unbind(CGF);
1441	}
1442	};
1443
1444	private:
1445	CGDebugInfo *DebugInfo;
1446	/// Used to create unique names for artificial VLA size debug info variables.
1447	unsigned VLAExprCounter = 0;
1448	bool DisableDebugInfo = false;
1449
1450	/// DidCallStackSave - Whether llvm.stacksave has been called. Used to avoid
1451	/// calling llvm.stacksave for multiple VLAs in the same scope.
1452	bool DidCallStackSave = false;
1453
1454	/// IndirectBranch - The first time an indirect goto is seen we create a block
1455	/// with an indirect branch. Every time we see the address of a label taken,
1456	/// we add the label to the indirect goto. Every subsequent indirect goto is
1457	/// codegen'd as a jump to the IndirectBranch's basic block.
1458	llvm::IndirectBrInst *IndirectBranch = nullptr;
1459
1460	/// LocalDeclMap - This keeps track of the LLVM allocas or globals for local C
1461	/// decls.
1462	DeclMapTy LocalDeclMap;
1463
1464	// Keep track of the cleanups for callee-destructed parameters pushed to the
1465	// cleanup stack so that they can be deactivated later.
1466	llvm::DenseMap<const ParmVarDecl *, EHScopeStack::stable_iterator>
1467	CalleeDestructedParamCleanups;
1468
1469	/// SizeArguments - If a ParmVarDecl had the pass_object_size attribute, this
1470	/// will contain a mapping from said ParmVarDecl to its implicit "object_size"
1471	/// parameter.
1472	llvm::SmallDenseMap<const ParmVarDecl *, const ImplicitParamDecl *, 2>
1473	SizeArguments;
1474
1475	/// Track escaped local variables with auto storage. Used during SEH
1476	/// outlining to produce a call to llvm.localescape.
1477	llvm::DenseMap<llvm::AllocaInst *, int> EscapedLocals;
1478
1479	/// LabelMap - This keeps track of the LLVM basic block for each C label.
1480	llvm::DenseMap<const LabelDecl*, JumpDest> LabelMap;
1481
1482	// BreakContinueStack - This keeps track of where break and continue
1483	// statements should jump to.
1484	struct BreakContinue {
1485	BreakContinue(JumpDest Break, JumpDest Continue)
1486	: BreakBlock(Break), ContinueBlock(Continue) {}
1487
1488	JumpDest BreakBlock;
1489	JumpDest ContinueBlock;
1490	};
1491	SmallVector<BreakContinue, 8> BreakContinueStack;
1492
1493	/// Handles cancellation exit points in OpenMP-related constructs.
1494	class OpenMPCancelExitStack {
1495	/// Tracks cancellation exit point and join point for cancel-related exit
1496	/// and normal exit.
1497	struct CancelExit {
1498	CancelExit() = default;
1499	CancelExit(OpenMPDirectiveKind Kind, JumpDest ExitBlock,
1500	JumpDest ContBlock)
1501	: Kind(Kind), ExitBlock(ExitBlock), ContBlock(ContBlock) {}
1502	OpenMPDirectiveKind Kind = llvm::omp::OMPD_unknown;
1503	/// true if the exit block has been emitted already by the special
1504	/// emitExit() call, false if the default codegen is used.
1505	bool HasBeenEmitted = false;
1506	JumpDest ExitBlock;
1507	JumpDest ContBlock;
1508	};
1509
1510	SmallVector<CancelExit, 8> Stack;
1511
1512	public:
1513	OpenMPCancelExitStack() : Stack(1) {}
1514	~OpenMPCancelExitStack() = default;
1515	/// Fetches the exit block for the current OpenMP construct.
1516	JumpDest getExitBlock() const { return Stack.back().ExitBlock; }
1517	/// Emits exit block with special codegen procedure specific for the related
1518	/// OpenMP construct + emits code for normal construct cleanup.
1519	void emitExit(CodeGenFunction &CGF, OpenMPDirectiveKind Kind,
1520	const llvm::function_ref<void(CodeGenFunction &)> CodeGen) {
1521	if (Stack.back().Kind == Kind && getExitBlock().isValid()) {
1522	assert(CGF.getOMPCancelDestination(Kind).isValid());
1523	assert(CGF.HaveInsertPoint());
1524	assert(!Stack.back().HasBeenEmitted);
1525	auto IP = CGF.Builder.saveAndClearIP();
1526	CGF.EmitBlock(BB: Stack.back().ExitBlock.getBlock());
1527	CodeGen(CGF);
1528	CGF.EmitBranch(Block: Stack.back().ContBlock.getBlock());
1529	CGF.Builder.restoreIP(IP);
1530	Stack.back().HasBeenEmitted = true;
1531	}
1532	CodeGen(CGF);
1533	}
1534	/// Enter the cancel supporting \a Kind construct.
1535	/// \param Kind OpenMP directive that supports cancel constructs.
1536	/// \param HasCancel true, if the construct has inner cancel directive,
1537	/// false otherwise.
1538	void enter(CodeGenFunction &CGF, OpenMPDirectiveKind Kind, bool HasCancel) {
1539	Stack.push_back(Elt: {Kind,
1540	HasCancel ? CGF.getJumpDestInCurrentScope(Name: "cancel.exit")
1541	: JumpDest(),
1542	HasCancel ? CGF.getJumpDestInCurrentScope(Name: "cancel.cont")
1543	: JumpDest()});
1544	}
1545	/// Emits default exit point for the cancel construct (if the special one
1546	/// has not be used) + join point for cancel/normal exits.
1547	void exit(CodeGenFunction &CGF) {
1548	if (getExitBlock().isValid()) {
1549	assert(CGF.getOMPCancelDestination(Stack.back().Kind).isValid());
1550	bool HaveIP = CGF.HaveInsertPoint();
1551	if (!Stack.back().HasBeenEmitted) {
1552	if (HaveIP)
1553	CGF.EmitBranchThroughCleanup(Dest: Stack.back().ContBlock);
1554	CGF.EmitBlock(BB: Stack.back().ExitBlock.getBlock());
1555	CGF.EmitBranchThroughCleanup(Dest: Stack.back().ContBlock);
1556	}
1557	CGF.EmitBlock(BB: Stack.back().ContBlock.getBlock());
1558	if (!HaveIP) {
1559	CGF.Builder.CreateUnreachable();
1560	CGF.Builder.ClearInsertionPoint();
1561	}
1562	}
1563	Stack.pop_back();
1564	}
1565	};
1566	OpenMPCancelExitStack OMPCancelStack;
1567
1568	/// Lower the Likelihood knowledge about the \p Cond via llvm.expect intrin.
1569	llvm::Value *emitCondLikelihoodViaExpectIntrinsic(llvm::Value *Cond,
1570	Stmt::Likelihood LH);
1571
1572	CodeGenPGO PGO;
1573
1574	/// Bitmap used by MC/DC to track condition outcomes of a boolean expression.
1575	Address MCDCCondBitmapAddr = Address::invalid();
1576
1577	/// Calculate branch weights appropriate for PGO data
1578	llvm::MDNode *createProfileWeights(uint64_t TrueCount,
1579	uint64_t FalseCount) const;
1580	llvm::MDNode *createProfileWeights(ArrayRef<uint64_t> Weights) const;
1581	llvm::MDNode *createProfileWeightsForLoop(const Stmt *Cond,
1582	uint64_t LoopCount) const;
1583
1584	public:
1585	/// Increment the profiler's counter for the given statement by \p StepV.
1586	/// If \p StepV is null, the default increment is 1.
1587	void incrementProfileCounter(const Stmt *S, llvm::Value *StepV = nullptr) {
1588	if (CGM.getCodeGenOpts().hasProfileClangInstr() &&
1589	!CurFn->hasFnAttribute(llvm::Attribute::NoProfile) &&
1590	!CurFn->hasFnAttribute(llvm::Attribute::SkipProfile))
1591	PGO.emitCounterSetOrIncrement(Builder, S, StepV);
1592	PGO.setCurrentStmt(S);
1593	}
1594
1595	bool isMCDCCoverageEnabled() const {
1596	return (CGM.getCodeGenOpts().hasProfileClangInstr() &&
1597	CGM.getCodeGenOpts().MCDCCoverage &&
1598	!CurFn->hasFnAttribute(llvm::Attribute::NoProfile));
1599	}
1600
1601	/// Allocate a temp value on the stack that MCDC can use to track condition
1602	/// results.
1603	void maybeCreateMCDCCondBitmap() {
1604	if (isMCDCCoverageEnabled()) {
1605	PGO.emitMCDCParameters(Builder);
1606	MCDCCondBitmapAddr =
1607	CreateIRTemp(T: getContext().UnsignedIntTy, Name: "mcdc.addr");
1608	}
1609	}
1610
1611	bool isBinaryLogicalOp(const Expr *E) const {
1612	const BinaryOperator *BOp = dyn_cast<BinaryOperator>(Val: E->IgnoreParens());
1613	return (BOp && BOp->isLogicalOp());
1614	}
1615
1616	/// Zero-init the MCDC temp value.
1617	void maybeResetMCDCCondBitmap(const Expr *E) {
1618	if (isMCDCCoverageEnabled() && isBinaryLogicalOp(E)) {
1619	PGO.emitMCDCCondBitmapReset(Builder, S: E, MCDCCondBitmapAddr);
1620	PGO.setCurrentStmt(E);
1621	}
1622	}
1623
1624	/// Increment the profiler's counter for the given expression by \p StepV.
1625	/// If \p StepV is null, the default increment is 1.
1626	void maybeUpdateMCDCTestVectorBitmap(const Expr *E) {
1627	if (isMCDCCoverageEnabled() && isBinaryLogicalOp(E)) {
1628	PGO.emitMCDCTestVectorBitmapUpdate(Builder, S: E, MCDCCondBitmapAddr, CGF&: *this);
1629	PGO.setCurrentStmt(E);
1630	}
1631	}
1632
1633	/// Update the MCDC temp value with the condition's evaluated result.
1634	void maybeUpdateMCDCCondBitmap(const Expr *E, llvm::Value *Val) {
1635	if (isMCDCCoverageEnabled()) {
1636	PGO.emitMCDCCondBitmapUpdate(Builder, S: E, MCDCCondBitmapAddr, Val, CGF&: *this);
1637	PGO.setCurrentStmt(E);
1638	}
1639	}
1640
1641	/// Get the profiler's count for the given statement.
1642	uint64_t getProfileCount(const Stmt *S) {
1643	return PGO.getStmtCount(S).value_or(u: 0);
1644	}
1645
1646	/// Set the profiler's current count.
1647	void setCurrentProfileCount(uint64_t Count) {
1648	PGO.setCurrentRegionCount(Count);
1649	}
1650
1651	/// Get the profiler's current count. This is generally the count for the most
1652	/// recently incremented counter.
1653	uint64_t getCurrentProfileCount() {
1654	return PGO.getCurrentRegionCount();
1655	}
1656
1657	private:
1658
1659	/// SwitchInsn - This is nearest current switch instruction. It is null if
1660	/// current context is not in a switch.
1661	llvm::SwitchInst *SwitchInsn = nullptr;
1662	/// The branch weights of SwitchInsn when doing instrumentation based PGO.
1663	SmallVector<uint64_t, 16> *SwitchWeights = nullptr;
1664
1665	/// The likelihood attributes of the SwitchCase.
1666	SmallVector<Stmt::Likelihood, 16> *SwitchLikelihood = nullptr;
1667
1668	/// CaseRangeBlock - This block holds if condition check for last case
1669	/// statement range in current switch instruction.
1670	llvm::BasicBlock *CaseRangeBlock = nullptr;
1671
1672	/// OpaqueLValues - Keeps track of the current set of opaque value
1673	/// expressions.
1674	llvm::DenseMap<const OpaqueValueExpr *, LValue> OpaqueLValues;
1675	llvm::DenseMap<const OpaqueValueExpr *, RValue> OpaqueRValues;
1676
1677	// VLASizeMap - This keeps track of the associated size for each VLA type.
1678	// We track this by the size expression rather than the type itself because
1679	// in certain situations, like a const qualifier applied to an VLA typedef,
1680	// multiple VLA types can share the same size expression.
1681	// FIXME: Maybe this could be a stack of maps that is pushed/popped as we
1682	// enter/leave scopes.
1683	llvm::DenseMap<const Expr*, llvm::Value*> VLASizeMap;
1684
1685	/// A block containing a single 'unreachable' instruction. Created
1686	/// lazily by getUnreachableBlock().
1687	llvm::BasicBlock *UnreachableBlock = nullptr;
1688
1689	/// Counts of the number return expressions in the function.
1690	unsigned NumReturnExprs = 0;
1691
1692	/// Count the number of simple (constant) return expressions in the function.
1693	unsigned NumSimpleReturnExprs = 0;
1694
1695	/// The last regular (non-return) debug location (breakpoint) in the function.
1696	SourceLocation LastStopPoint;
1697
1698	public:
1699	/// Source location information about the default argument or member
1700	/// initializer expression we're evaluating, if any.
1701	CurrentSourceLocExprScope CurSourceLocExprScope;
1702	using SourceLocExprScopeGuard =
1703	CurrentSourceLocExprScope::SourceLocExprScopeGuard;
1704
1705	/// A scope within which we are constructing the fields of an object which
1706	/// might use a CXXDefaultInitExpr. This stashes away a 'this' value to use
1707	/// if we need to evaluate a CXXDefaultInitExpr within the evaluation.
1708	class FieldConstructionScope {
1709	public:
1710	FieldConstructionScope(CodeGenFunction &CGF, Address This)
1711	: CGF(CGF), OldCXXDefaultInitExprThis(CGF.CXXDefaultInitExprThis) {
1712	CGF.CXXDefaultInitExprThis = This;
1713	}
1714	~FieldConstructionScope() {
1715	CGF.CXXDefaultInitExprThis = OldCXXDefaultInitExprThis;
1716	}
1717
1718	private:
1719	CodeGenFunction &CGF;
1720	Address OldCXXDefaultInitExprThis;
1721	};
1722
1723	/// The scope of a CXXDefaultInitExpr. Within this scope, the value of 'this'
1724	/// is overridden to be the object under construction.
1725	class CXXDefaultInitExprScope {
1726	public:
1727	CXXDefaultInitExprScope(CodeGenFunction &CGF, const CXXDefaultInitExpr *E)
1728	: CGF(CGF), OldCXXThisValue(CGF.CXXThisValue),
1729	OldCXXThisAlignment(CGF.CXXThisAlignment),
1730	SourceLocScope(E, CGF.CurSourceLocExprScope) {
1731	CGF.CXXThisValue = CGF.CXXDefaultInitExprThis.getBasePointer();
1732	CGF.CXXThisAlignment = CGF.CXXDefaultInitExprThis.getAlignment();
1733	}
1734	~CXXDefaultInitExprScope() {
1735	CGF.CXXThisValue = OldCXXThisValue;
1736	CGF.CXXThisAlignment = OldCXXThisAlignment;
1737	}
1738
1739	public:
1740	CodeGenFunction &CGF;
1741	llvm::Value *OldCXXThisValue;
1742	CharUnits OldCXXThisAlignment;
1743	SourceLocExprScopeGuard SourceLocScope;
1744	};
1745
1746	struct CXXDefaultArgExprScope : SourceLocExprScopeGuard {
1747	CXXDefaultArgExprScope(CodeGenFunction &CGF, const CXXDefaultArgExpr *E)
1748	: SourceLocExprScopeGuard(E, CGF.CurSourceLocExprScope) {}
1749	};
1750
1751	/// The scope of an ArrayInitLoopExpr. Within this scope, the value of the
1752	/// current loop index is overridden.
1753	class ArrayInitLoopExprScope {
1754	public:
1755	ArrayInitLoopExprScope(CodeGenFunction &CGF, llvm::Value *Index)
1756	: CGF(CGF), OldArrayInitIndex(CGF.ArrayInitIndex) {
1757	CGF.ArrayInitIndex = Index;
1758	}
1759	~ArrayInitLoopExprScope() {
1760	CGF.ArrayInitIndex = OldArrayInitIndex;
1761	}
1762
1763	private:
1764	CodeGenFunction &CGF;
1765	llvm::Value *OldArrayInitIndex;
1766	};
1767
1768	class InlinedInheritingConstructorScope {
1769	public:
1770	InlinedInheritingConstructorScope(CodeGenFunction &CGF, GlobalDecl GD)
1771	: CGF(CGF), OldCurGD(CGF.CurGD), OldCurFuncDecl(CGF.CurFuncDecl),
1772	OldCurCodeDecl(CGF.CurCodeDecl),
1773	OldCXXABIThisDecl(CGF.CXXABIThisDecl),
1774	OldCXXABIThisValue(CGF.CXXABIThisValue),
1775	OldCXXThisValue(CGF.CXXThisValue),
1776	OldCXXABIThisAlignment(CGF.CXXABIThisAlignment),
1777	OldCXXThisAlignment(CGF.CXXThisAlignment),
1778	OldReturnValue(CGF.ReturnValue), OldFnRetTy(CGF.FnRetTy),
1779	OldCXXInheritedCtorInitExprArgs(
1780	std::move(CGF.CXXInheritedCtorInitExprArgs)) {
1781	CGF.CurGD = GD;
1782	CGF.CurFuncDecl = CGF.CurCodeDecl =
1783	cast<CXXConstructorDecl>(Val: GD.getDecl());
1784	CGF.CXXABIThisDecl = nullptr;
1785	CGF.CXXABIThisValue = nullptr;
1786	CGF.CXXThisValue = nullptr;
1787	CGF.CXXABIThisAlignment = CharUnits();
1788	CGF.CXXThisAlignment = CharUnits();
1789	CGF.ReturnValue = Address::invalid();
1790	CGF.FnRetTy = QualType();
1791	CGF.CXXInheritedCtorInitExprArgs.clear();
1792	}
1793	~InlinedInheritingConstructorScope() {
1794	CGF.CurGD = OldCurGD;
1795	CGF.CurFuncDecl = OldCurFuncDecl;
1796	CGF.CurCodeDecl = OldCurCodeDecl;
1797	CGF.CXXABIThisDecl = OldCXXABIThisDecl;
1798	CGF.CXXABIThisValue = OldCXXABIThisValue;
1799	CGF.CXXThisValue = OldCXXThisValue;
1800	CGF.CXXABIThisAlignment = OldCXXABIThisAlignment;
1801	CGF.CXXThisAlignment = OldCXXThisAlignment;
1802	CGF.ReturnValue = OldReturnValue;
1803	CGF.FnRetTy = OldFnRetTy;
1804	CGF.CXXInheritedCtorInitExprArgs =
1805	std::move(OldCXXInheritedCtorInitExprArgs);
1806	}
1807
1808	private:
1809	CodeGenFunction &CGF;
1810	GlobalDecl OldCurGD;
1811	const Decl *OldCurFuncDecl;
1812	const Decl *OldCurCodeDecl;
1813	ImplicitParamDecl *OldCXXABIThisDecl;
1814	llvm::Value *OldCXXABIThisValue;
1815	llvm::Value *OldCXXThisValue;
1816	CharUnits OldCXXABIThisAlignment;
1817	CharUnits OldCXXThisAlignment;
1818	Address OldReturnValue;
1819	QualType OldFnRetTy;
1820	CallArgList OldCXXInheritedCtorInitExprArgs;
1821	};
1822
1823	// Helper class for the OpenMP IR Builder. Allows reusability of code used for
1824	// region body, and finalization codegen callbacks. This will class will also
1825	// contain privatization functions used by the privatization call backs
1826	//
1827	// TODO: this is temporary class for things that are being moved out of
1828	// CGOpenMPRuntime, new versions of current CodeGenFunction methods, or
1829	// utility function for use with the OMPBuilder. Once that move to use the
1830	// OMPBuilder is done, everything here will either become part of CodeGenFunc.
1831	// directly, or a new helper class that will contain functions used by both
1832	// this and the OMPBuilder
1833
1834	struct OMPBuilderCBHelpers {
1835
1836	OMPBuilderCBHelpers() = delete;
1837	OMPBuilderCBHelpers(const OMPBuilderCBHelpers &) = delete;
1838	OMPBuilderCBHelpers &operator=(const OMPBuilderCBHelpers &) = delete;
1839
1840	using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
1841
1842	/// Cleanup action for allocate support.
1843	class OMPAllocateCleanupTy final : public EHScopeStack::Cleanup {
1844
1845	private:
1846	llvm::CallInst *RTLFnCI;
1847
1848	public:
1849	OMPAllocateCleanupTy(llvm::CallInst *RLFnCI) : RTLFnCI(RLFnCI) {
1850	RLFnCI->removeFromParent();
1851	}
1852
1853	void Emit(CodeGenFunction &CGF, Flags /*flags*/) override {
1854	if (!CGF.HaveInsertPoint())
1855	return;
1856	CGF.Builder.Insert(I: RTLFnCI);
1857	}
1858	};
1859
1860	/// Returns address of the threadprivate variable for the current
1861	/// thread. This Also create any necessary OMP runtime calls.
1862	///
1863	/// \param VD VarDecl for Threadprivate variable.
1864	/// \param VDAddr Address of the Vardecl
1865	/// \param Loc The location where the barrier directive was encountered
1866	static Address getAddrOfThreadPrivate(CodeGenFunction &CGF,
1867	const VarDecl *VD, Address VDAddr,
1868	SourceLocation Loc);
1869
1870	/// Gets the OpenMP-specific address of the local variable /p VD.
1871	static Address getAddressOfLocalVariable(CodeGenFunction &CGF,
1872	const VarDecl *VD);
1873	/// Get the platform-specific name separator.
1874	/// \param Parts different parts of the final name that needs separation
1875	/// \param FirstSeparator First separator used between the initial two
1876	/// parts of the name.
1877	/// \param Separator separator used between all of the rest consecutinve
1878	/// parts of the name
1879	static std::string getNameWithSeparators(ArrayRef<StringRef> Parts,
1880	StringRef FirstSeparator = ".",
1881	StringRef Separator = ".");
1882	/// Emit the Finalization for an OMP region
1883	/// \param CGF The Codegen function this belongs to
1884	/// \param IP Insertion point for generating the finalization code.
1885	static void FinalizeOMPRegion(CodeGenFunction &CGF, InsertPointTy IP) {
1886	CGBuilderTy::InsertPointGuard IPG(CGF.Builder);
1887	assert(IP.getBlock()->end() != IP.getPoint() &&
1888	"OpenMP IR Builder should cause terminated block!");
1889
1890	llvm::BasicBlock *IPBB = IP.getBlock();
1891	llvm::BasicBlock *DestBB = IPBB->getUniqueSuccessor();
1892	assert(DestBB && "Finalization block should have one successor!");
1893
1894	// erase and replace with cleanup branch.
1895	IPBB->getTerminator()->eraseFromParent();
1896	CGF.Builder.SetInsertPoint(IPBB);
1897	CodeGenFunction::JumpDest Dest = CGF.getJumpDestInCurrentScope(Target: DestBB);
1898	CGF.EmitBranchThroughCleanup(Dest);
1899	}
1900
1901	/// Emit the body of an OMP region
1902	/// \param CGF The Codegen function this belongs to
1903	/// \param RegionBodyStmt The body statement for the OpenMP region being
1904	/// generated
1905	/// \param AllocaIP Where to insert alloca instructions
1906	/// \param CodeGenIP Where to insert the region code
1907	/// \param RegionName Name to be used for new blocks
1908	static void EmitOMPInlinedRegionBody(CodeGenFunction &CGF,
1909	const Stmt *RegionBodyStmt,
1910	InsertPointTy AllocaIP,
1911	InsertPointTy CodeGenIP,
1912	Twine RegionName);
1913
1914	static void EmitCaptureStmt(CodeGenFunction &CGF, InsertPointTy CodeGenIP,
1915	llvm::BasicBlock &FiniBB, llvm::Function *Fn,
1916	ArrayRef<llvm::Value *> Args) {
1917	llvm::BasicBlock *CodeGenIPBB = CodeGenIP.getBlock();
1918	if (llvm::Instruction *CodeGenIPBBTI = CodeGenIPBB->getTerminator())
1919	CodeGenIPBBTI->eraseFromParent();
1920
1921	CGF.Builder.SetInsertPoint(CodeGenIPBB);
1922
1923	if (Fn->doesNotThrow())
1924	CGF.EmitNounwindRuntimeCall(callee: Fn, args: Args);
1925	else
1926	CGF.EmitRuntimeCall(callee: Fn, args: Args);
1927
1928	if (CGF.Builder.saveIP().isSet())
1929	CGF.Builder.CreateBr(Dest: &FiniBB);
1930	}
1931
1932	/// Emit the body of an OMP region that will be outlined in
1933	/// OpenMPIRBuilder::finalize().
1934	/// \param CGF The Codegen function this belongs to
1935	/// \param RegionBodyStmt The body statement for the OpenMP region being
1936	/// generated
1937	/// \param AllocaIP Where to insert alloca instructions
1938	/// \param CodeGenIP Where to insert the region code
1939	/// \param RegionName Name to be used for new blocks
1940	static void EmitOMPOutlinedRegionBody(CodeGenFunction &CGF,
1941	const Stmt *RegionBodyStmt,
1942	InsertPointTy AllocaIP,
1943	InsertPointTy CodeGenIP,
1944	Twine RegionName);
1945
1946	/// RAII for preserving necessary info during Outlined region body codegen.
1947	class OutlinedRegionBodyRAII {
1948
1949	llvm::AssertingVH<llvm::Instruction> OldAllocaIP;
1950	CodeGenFunction::JumpDest OldReturnBlock;
1951	CodeGenFunction &CGF;
1952
1953	public:
1954	OutlinedRegionBodyRAII(CodeGenFunction &cgf, InsertPointTy &AllocaIP,
1955	llvm::BasicBlock &RetBB)
1956	: CGF(cgf) {
1957	assert(AllocaIP.isSet() &&
1958	"Must specify Insertion point for allocas of outlined function");
1959	OldAllocaIP = CGF.AllocaInsertPt;
1960	CGF.AllocaInsertPt = &*AllocaIP.getPoint();
1961
1962	OldReturnBlock = CGF.ReturnBlock;
1963	CGF.ReturnBlock = CGF.getJumpDestInCurrentScope(Target: &RetBB);
1964	}
1965
1966	~OutlinedRegionBodyRAII() {
1967	CGF.AllocaInsertPt = OldAllocaIP;
1968	CGF.ReturnBlock = OldReturnBlock;
1969	}
1970	};
1971
1972	/// RAII for preserving necessary info during inlined region body codegen.
1973	class InlinedRegionBodyRAII {
1974
1975	llvm::AssertingVH<llvm::Instruction> OldAllocaIP;
1976	CodeGenFunction &CGF;
1977
1978	public:
1979	InlinedRegionBodyRAII(CodeGenFunction &cgf, InsertPointTy &AllocaIP,
1980	llvm::BasicBlock &FiniBB)
1981	: CGF(cgf) {
1982	// Alloca insertion block should be in the entry block of the containing
1983	// function so it expects an empty AllocaIP in which case will reuse the
1984	// old alloca insertion point, or a new AllocaIP in the same block as
1985	// the old one
1986	assert((!AllocaIP.isSet() ||
1987	CGF.AllocaInsertPt->getParent() == AllocaIP.getBlock()) &&
1988	"Insertion point should be in the entry block of containing "
1989	"function!");
1990	OldAllocaIP = CGF.AllocaInsertPt;
1991	if (AllocaIP.isSet())
1992	CGF.AllocaInsertPt = &*AllocaIP.getPoint();
1993
1994	// TODO: Remove the call, after making sure the counter is not used by
1995	// the EHStack.
1996	// Since this is an inlined region, it should not modify the
1997	// ReturnBlock, and should reuse the one for the enclosing outlined
1998	// region. So, the JumpDest being return by the function is discarded
1999	(void)CGF.getJumpDestInCurrentScope(Target: &FiniBB);
2000	}
2001
2002	~InlinedRegionBodyRAII() { CGF.AllocaInsertPt = OldAllocaIP; }
2003	};
2004	};
2005
2006	private:
2007	/// CXXThisDecl - When generating code for a C++ member function,
2008	/// this will hold the implicit 'this' declaration.
2009	ImplicitParamDecl *CXXABIThisDecl = nullptr;
2010	llvm::Value *CXXABIThisValue = nullptr;
2011	llvm::Value *CXXThisValue = nullptr;
2012	CharUnits CXXABIThisAlignment;
2013	CharUnits CXXThisAlignment;
2014
2015	/// The value of 'this' to use when evaluating CXXDefaultInitExprs within
2016	/// this expression.
2017	Address CXXDefaultInitExprThis = Address::invalid();
2018
2019	/// The current array initialization index when evaluating an
2020	/// ArrayInitIndexExpr within an ArrayInitLoopExpr.
2021	llvm::Value *ArrayInitIndex = nullptr;
2022
2023	/// The values of function arguments to use when evaluating
2024	/// CXXInheritedCtorInitExprs within this context.
2025	CallArgList CXXInheritedCtorInitExprArgs;
2026
2027	/// CXXStructorImplicitParamDecl - When generating code for a constructor or
2028	/// destructor, this will hold the implicit argument (e.g. VTT).
2029	ImplicitParamDecl *CXXStructorImplicitParamDecl = nullptr;
2030	llvm::Value *CXXStructorImplicitParamValue = nullptr;
2031
2032	/// OutermostConditional - Points to the outermost active
2033	/// conditional control. This is used so that we know if a
2034	/// temporary should be destroyed conditionally.
2035	ConditionalEvaluation *OutermostConditional = nullptr;
2036
2037	/// The current lexical scope.
2038	LexicalScope *CurLexicalScope = nullptr;
2039
2040	/// The current source location that should be used for exception
2041	/// handling code.
2042	SourceLocation CurEHLocation;
2043
2044	/// BlockByrefInfos - For each __block variable, contains
2045	/// information about the layout of the variable.
2046	llvm::DenseMap<const ValueDecl *, BlockByrefInfo> BlockByrefInfos;
2047
2048	/// Used by -fsanitize=nullability-return to determine whether the return
2049	/// value can be checked.
2050	llvm::Value *RetValNullabilityPrecondition = nullptr;
2051
2052	/// Check if -fsanitize=nullability-return instrumentation is required for
2053	/// this function.
2054	bool requiresReturnValueNullabilityCheck() const {
2055	return RetValNullabilityPrecondition;
2056	}
2057
2058	/// Used to store precise source locations for return statements by the
2059	/// runtime return value checks.
2060	Address ReturnLocation = Address::invalid();
2061
2062	/// Check if the return value of this function requires sanitization.
2063	bool requiresReturnValueCheck() const;
2064
2065	bool isInAllocaArgument(CGCXXABI &ABI, QualType Ty);
2066	bool hasInAllocaArg(const CXXMethodDecl *MD);
2067
2068	llvm::BasicBlock *TerminateLandingPad = nullptr;
2069	llvm::BasicBlock *TerminateHandler = nullptr;
2070	llvm::SmallVector<llvm::BasicBlock *, 2> TrapBBs;
2071
2072	/// Terminate funclets keyed by parent funclet pad.
2073	llvm::MapVector<llvm::Value *, llvm::BasicBlock *> TerminateFunclets;
2074
2075	/// Largest vector width used in ths function. Will be used to create a
2076	/// function attribute.
2077	unsigned LargestVectorWidth = 0;
2078
2079	/// True if we need emit the life-time markers. This is initially set in
2080	/// the constructor, but could be overwritten to true if this is a coroutine.
2081	bool ShouldEmitLifetimeMarkers;
2082
2083	/// Add OpenCL kernel arg metadata and the kernel attribute metadata to
2084	/// the function metadata.
2085	void EmitKernelMetadata(const FunctionDecl *FD, llvm::Function *Fn);
2086
2087	public:
2088	CodeGenFunction(CodeGenModule &cgm, bool suppressNewContext=false);
2089	~CodeGenFunction();
2090
2091	CodeGenTypes &getTypes() const { return CGM.getTypes(); }
2092	ASTContext &getContext() const { return CGM.getContext(); }
2093	CGDebugInfo *getDebugInfo() {
2094	if (DisableDebugInfo)
2095	return nullptr;
2096	return DebugInfo;
2097	}
2098	void disableDebugInfo() { DisableDebugInfo = true; }
2099	void enableDebugInfo() { DisableDebugInfo = false; }
2100
2101	bool shouldUseFusedARCCalls() {
2102	return CGM.getCodeGenOpts().OptimizationLevel == 0;
2103	}
2104
2105	const LangOptions &getLangOpts() const { return CGM.getLangOpts(); }
2106
2107	/// Returns a pointer to the function's exception object and selector slot,
2108	/// which is assigned in every landing pad.
2109	Address getExceptionSlot();
2110	Address getEHSelectorSlot();
2111
2112	/// Returns the contents of the function's exception object and selector
2113	/// slots.
2114	llvm::Value *getExceptionFromSlot();
2115	llvm::Value *getSelectorFromSlot();
2116
2117	RawAddress getNormalCleanupDestSlot();
2118
2119	llvm::BasicBlock *getUnreachableBlock() {
2120	if (!UnreachableBlock) {
2121	UnreachableBlock = createBasicBlock(name: "unreachable");
2122	new llvm::UnreachableInst(getLLVMContext(), UnreachableBlock);
2123	}
2124	return UnreachableBlock;
2125	}
2126
2127	llvm::BasicBlock *getInvokeDest() {
2128	if (!EHStack.requiresLandingPad()) return nullptr;
2129	return getInvokeDestImpl();
2130	}
2131
2132	bool currentFunctionUsesSEHTry() const { return !!CurSEHParent; }
2133
2134	const TargetInfo &getTarget() const { return Target; }
2135	llvm::LLVMContext &getLLVMContext() { return CGM.getLLVMContext(); }
2136	const TargetCodeGenInfo &getTargetHooks() const {
2137	return CGM.getTargetCodeGenInfo();
2138	}
2139
2140	//===--------------------------------------------------------------------===//
2141	// Cleanups
2142	//===--------------------------------------------------------------------===//
2143
2144	typedef void Destroyer(CodeGenFunction &CGF, Address addr, QualType ty);
2145
2146	void pushIrregularPartialArrayCleanup(llvm::Value *arrayBegin,
2147	Address arrayEndPointer,
2148	QualType elementType,
2149	CharUnits elementAlignment,
2150	Destroyer *destroyer);
2151	void pushRegularPartialArrayCleanup(llvm::Value *arrayBegin,
2152	llvm::Value *arrayEnd,
2153	QualType elementType,
2154	CharUnits elementAlignment,
2155	Destroyer *destroyer);
2156
2157	void pushDestroy(QualType::DestructionKind dtorKind,
2158	Address addr, QualType type);
2159	void pushEHDestroy(QualType::DestructionKind dtorKind,
2160	Address addr, QualType type);
2161	void pushDestroy(CleanupKind kind, Address addr, QualType type,
2162	Destroyer *destroyer, bool useEHCleanupForArray);
2163	void pushLifetimeExtendedDestroy(CleanupKind kind, Address addr,
2164	QualType type, Destroyer *destroyer,
2165	bool useEHCleanupForArray);
2166	void pushCallObjectDeleteCleanup(const FunctionDecl *OperatorDelete,
2167	llvm::Value *CompletePtr,
2168	QualType ElementType);
2169	void pushStackRestore(CleanupKind kind, Address SPMem);
2170	void pushKmpcAllocFree(CleanupKind Kind,
2171	std::pair<llvm::Value *, llvm::Value *> AddrSizePair);
2172	void emitDestroy(Address addr, QualType type, Destroyer *destroyer,
2173	bool useEHCleanupForArray);
2174	llvm::Function *generateDestroyHelper(Address addr, QualType type,
2175	Destroyer *destroyer,
2176	bool useEHCleanupForArray,
2177	const VarDecl *VD);
2178	void emitArrayDestroy(llvm::Value *begin, llvm::Value *end,
2179	QualType elementType, CharUnits elementAlign,
2180	Destroyer *destroyer,
2181	bool checkZeroLength, bool useEHCleanup);
2182
2183	Destroyer *getDestroyer(QualType::DestructionKind destructionKind);
2184
2185	/// Determines whether an EH cleanup is required to destroy a type
2186	/// with the given destruction kind.
2187	bool needsEHCleanup(QualType::DestructionKind kind) {
2188	switch (kind) {
2189	case QualType::DK_none:
2190	return false;
2191	case QualType::DK_cxx_destructor:
2192	case QualType::DK_objc_weak_lifetime:
2193	case QualType::DK_nontrivial_c_struct:
2194	return getLangOpts().Exceptions;
2195	case QualType::DK_objc_strong_lifetime:
2196	return getLangOpts().Exceptions &&
2197	CGM.getCodeGenOpts().ObjCAutoRefCountExceptions;
2198	}
2199	llvm_unreachable("bad destruction kind");
2200	}
2201
2202	CleanupKind getCleanupKind(QualType::DestructionKind kind) {
2203	return (needsEHCleanup(kind) ? NormalAndEHCleanup : NormalCleanup);
2204	}
2205
2206	//===--------------------------------------------------------------------===//
2207	// Objective-C
2208	//===--------------------------------------------------------------------===//
2209
2210	void GenerateObjCMethod(const ObjCMethodDecl *OMD);
2211
2212	void StartObjCMethod(const ObjCMethodDecl *MD, const ObjCContainerDecl *CD);
2213
2214	/// GenerateObjCGetter - Synthesize an Objective-C property getter function.
2215	void GenerateObjCGetter(ObjCImplementationDecl *IMP,
2216	const ObjCPropertyImplDecl *PID);
2217	void generateObjCGetterBody(const ObjCImplementationDecl *classImpl,
2218	const ObjCPropertyImplDecl *propImpl,
2219	const ObjCMethodDecl *GetterMothodDecl,
2220	llvm::Constant *AtomicHelperFn);
2221
2222	void GenerateObjCCtorDtorMethod(ObjCImplementationDecl *IMP,
2223	ObjCMethodDecl *MD, bool ctor);
2224
2225	/// GenerateObjCSetter - Synthesize an Objective-C property setter function
2226	/// for the given property.
2227	void GenerateObjCSetter(ObjCImplementationDecl *IMP,
2228	const ObjCPropertyImplDecl *PID);
2229	void generateObjCSetterBody(const ObjCImplementationDecl *classImpl,
2230	const ObjCPropertyImplDecl *propImpl,
2231	llvm::Constant *AtomicHelperFn);
2232
2233	//===--------------------------------------------------------------------===//
2234	// Block Bits
2235	//===--------------------------------------------------------------------===//
2236
2237	/// Emit block literal.
2238	/// \return an LLVM value which is a pointer to a struct which contains
2239	/// information about the block, including the block invoke function, the
2240	/// captured variables, etc.
2241	llvm::Value *EmitBlockLiteral(const BlockExpr *);
2242
2243	llvm::Function *GenerateBlockFunction(GlobalDecl GD,
2244	const CGBlockInfo &Info,
2245	const DeclMapTy &ldm,
2246	bool IsLambdaConversionToBlock,
2247	bool BuildGlobalBlock);
2248
2249	/// Check if \p T is a C++ class that has a destructor that can throw.
2250	static bool cxxDestructorCanThrow(QualType T);
2251
2252	llvm::Constant *GenerateCopyHelperFunction(const CGBlockInfo &blockInfo);
2253	llvm::Constant *GenerateDestroyHelperFunction(const CGBlockInfo &blockInfo);
2254	llvm::Constant *GenerateObjCAtomicSetterCopyHelperFunction(
2255	const ObjCPropertyImplDecl *PID);
2256	llvm::Constant *GenerateObjCAtomicGetterCopyHelperFunction(
2257	const ObjCPropertyImplDecl *PID);
2258	llvm::Value *EmitBlockCopyAndAutorelease(llvm::Value *Block, QualType Ty);
2259
2260	void BuildBlockRelease(llvm::Value *DeclPtr, BlockFieldFlags flags,
2261	bool CanThrow);
2262
2263	class AutoVarEmission;
2264
2265	void emitByrefStructureInit(const AutoVarEmission &emission);
2266
2267	/// Enter a cleanup to destroy a __block variable. Note that this
2268	/// cleanup should be a no-op if the variable hasn't left the stack
2269	/// yet; if a cleanup is required for the variable itself, that needs
2270	/// to be done externally.
2271	///
2272	/// \param Kind Cleanup kind.
2273	///
2274	/// \param Addr When \p LoadBlockVarAddr is false, the address of the __block
2275	/// structure that will be passed to _Block_object_dispose. When
2276	/// \p LoadBlockVarAddr is true, the address of the field of the block
2277	/// structure that holds the address of the __block structure.
2278	///
2279	/// \param Flags The flag that will be passed to _Block_object_dispose.
2280	///
2281	/// \param LoadBlockVarAddr Indicates whether we need to emit a load from
2282	/// \p Addr to get the address of the __block structure.
2283	void enterByrefCleanup(CleanupKind Kind, Address Addr, BlockFieldFlags Flags,
2284	bool LoadBlockVarAddr, bool CanThrow);
2285
2286	void setBlockContextParameter(const ImplicitParamDecl *D, unsigned argNum,
2287	llvm::Value *ptr);
2288
2289	Address LoadBlockStruct();
2290	Address GetAddrOfBlockDecl(const VarDecl *var);
2291
2292	/// BuildBlockByrefAddress - Computes the location of the
2293	/// data in a variable which is declared as __block.
2294	Address emitBlockByrefAddress(Address baseAddr, const VarDecl *V,
2295	bool followForward = true);
2296	Address emitBlockByrefAddress(Address baseAddr,
2297	const BlockByrefInfo &info,
2298	bool followForward,
2299	const llvm::Twine &name);
2300
2301	const BlockByrefInfo &getBlockByrefInfo(const VarDecl *var);
2302
2303	QualType BuildFunctionArgList(GlobalDecl GD, FunctionArgList &Args);
2304
2305	void GenerateCode(GlobalDecl GD, llvm::Function *Fn,
2306	const CGFunctionInfo &FnInfo);
2307
2308	/// Annotate the function with an attribute that disables TSan checking at
2309	/// runtime.
2310	void markAsIgnoreThreadCheckingAtRuntime(llvm::Function *Fn);
2311
2312	/// Emit code for the start of a function.
2313	/// \param Loc The location to be associated with the function.
2314	/// \param StartLoc The location of the function body.
2315	void StartFunction(GlobalDecl GD,
2316	QualType RetTy,
2317	llvm::Function *Fn,
2318	const CGFunctionInfo &FnInfo,
2319	const FunctionArgList &Args,
2320	SourceLocation Loc = SourceLocation(),
2321	SourceLocation StartLoc = SourceLocation());
2322
2323	static bool IsConstructorDelegationValid(const CXXConstructorDecl *Ctor);
2324
2325	void EmitConstructorBody(FunctionArgList &Args);
2326	void EmitDestructorBody(FunctionArgList &Args);
2327	void emitImplicitAssignmentOperatorBody(FunctionArgList &Args);
2328	void EmitFunctionBody(const Stmt *Body);
2329	void EmitBlockWithFallThrough(llvm::BasicBlock *BB, const Stmt *S);
2330
2331	void EmitForwardingCallToLambda(const CXXMethodDecl *LambdaCallOperator,
2332	CallArgList &CallArgs,
2333	const CGFunctionInfo *CallOpFnInfo = nullptr,
2334	llvm::Constant *CallOpFn = nullptr);
2335	void EmitLambdaBlockInvokeBody();
2336	void EmitLambdaStaticInvokeBody(const CXXMethodDecl *MD);
2337	void EmitLambdaDelegatingInvokeBody(const CXXMethodDecl *MD,
2338	CallArgList &CallArgs);
2339	void EmitLambdaInAllocaImplFn(const CXXMethodDecl *CallOp,
2340	const CGFunctionInfo **ImplFnInfo,
2341	llvm::Function **ImplFn);
2342	void EmitLambdaInAllocaCallOpBody(const CXXMethodDecl *MD);
2343	void EmitLambdaVLACapture(const VariableArrayType *VAT, LValue LV) {
2344	EmitStoreThroughLValue(Src: RValue::get(V: VLASizeMap[VAT->getSizeExpr()]), Dst: LV);
2345	}
2346	void EmitAsanPrologueOrEpilogue(bool Prologue);
2347
2348	/// Emit the unified return block, trying to avoid its emission when
2349	/// possible.
2350	/// \return The debug location of the user written return statement if the
2351	/// return block is avoided.
2352	llvm::DebugLoc EmitReturnBlock();
2353
2354	/// FinishFunction - Complete IR generation of the current function. It is
2355	/// legal to call this function even if there is no current insertion point.
2356	void FinishFunction(SourceLocation EndLoc=SourceLocation());
2357
2358	void StartThunk(llvm::Function *Fn, GlobalDecl GD,
2359	const CGFunctionInfo &FnInfo, bool IsUnprototyped);
2360
2361	void EmitCallAndReturnForThunk(llvm::FunctionCallee Callee,
2362	const ThunkInfo *Thunk, bool IsUnprototyped);
2363
2364	void FinishThunk();
2365
2366	/// Emit a musttail call for a thunk with a potentially adjusted this pointer.
2367	void EmitMustTailThunk(GlobalDecl GD, llvm::Value *AdjustedThisPtr,
2368	llvm::FunctionCallee Callee);
2369
2370	/// Generate a thunk for the given method.
2371	void generateThunk(llvm::Function *Fn, const CGFunctionInfo &FnInfo,
2372	GlobalDecl GD, const ThunkInfo &Thunk,
2373	bool IsUnprototyped);
2374
2375	llvm::Function *GenerateVarArgsThunk(llvm::Function *Fn,
2376	const CGFunctionInfo &FnInfo,
2377	GlobalDecl GD, const ThunkInfo &Thunk);
2378
2379	void EmitCtorPrologue(const CXXConstructorDecl *CD, CXXCtorType Type,
2380	FunctionArgList &Args);
2381
2382	void EmitInitializerForField(FieldDecl *Field, LValue LHS, Expr *Init);
2383
2384	/// Struct with all information about dynamic [sub]class needed to set vptr.
2385	struct VPtr {
2386	BaseSubobject Base;
2387	const CXXRecordDecl *NearestVBase;
2388	CharUnits OffsetFromNearestVBase;
2389	const CXXRecordDecl *VTableClass;
2390	};
2391
2392	/// Initialize the vtable pointer of the given subobject.
2393	void InitializeVTablePointer(const VPtr &vptr);
2394
2395	typedef llvm::SmallVector<VPtr, 4> VPtrsVector;
2396
2397	typedef llvm::SmallPtrSet<const CXXRecordDecl *, 4> VisitedVirtualBasesSetTy;
2398	VPtrsVector getVTablePointers(const CXXRecordDecl *VTableClass);
2399
2400	void getVTablePointers(BaseSubobject Base, const CXXRecordDecl *NearestVBase,
2401	CharUnits OffsetFromNearestVBase,
2402	bool BaseIsNonVirtualPrimaryBase,
2403	const CXXRecordDecl *VTableClass,
2404	VisitedVirtualBasesSetTy &VBases, VPtrsVector &vptrs);
2405
2406	void InitializeVTablePointers(const CXXRecordDecl *ClassDecl);
2407
2408	/// GetVTablePtr - Return the Value of the vtable pointer member pointed
2409	/// to by This.
2410	llvm::Value *GetVTablePtr(Address This, llvm::Type *VTableTy,
2411	const CXXRecordDecl *VTableClass);
2412
2413	enum CFITypeCheckKind {
2414	CFITCK_VCall,
2415	CFITCK_NVCall,
2416	CFITCK_DerivedCast,
2417	CFITCK_UnrelatedCast,
2418	CFITCK_ICall,
2419	CFITCK_NVMFCall,
2420	CFITCK_VMFCall,
2421	};
2422
2423	/// Derived is the presumed address of an object of type T after a
2424	/// cast. If T is a polymorphic class type, emit a check that the virtual
2425	/// table for Derived belongs to a class derived from T.
2426	void EmitVTablePtrCheckForCast(QualType T, Address Derived, bool MayBeNull,
2427	CFITypeCheckKind TCK, SourceLocation Loc);
2428
2429	/// EmitVTablePtrCheckForCall - Virtual method MD is being called via VTable.
2430	/// If vptr CFI is enabled, emit a check that VTable is valid.
2431	void EmitVTablePtrCheckForCall(const CXXRecordDecl *RD, llvm::Value *VTable,
2432	CFITypeCheckKind TCK, SourceLocation Loc);
2433
2434	/// EmitVTablePtrCheck - Emit a check that VTable is a valid virtual table for
2435	/// RD using llvm.type.test.
2436	void EmitVTablePtrCheck(const CXXRecordDecl *RD, llvm::Value *VTable,
2437	CFITypeCheckKind TCK, SourceLocation Loc);
2438
2439	/// If whole-program virtual table optimization is enabled, emit an assumption
2440	/// that VTable is a member of RD's type identifier. Or, if vptr CFI is
2441	/// enabled, emit a check that VTable is a member of RD's type identifier.
2442	void EmitTypeMetadataCodeForVCall(const CXXRecordDecl *RD,
2443	llvm::Value *VTable, SourceLocation Loc);
2444
2445	/// Returns whether we should perform a type checked load when loading a
2446	/// virtual function for virtual calls to members of RD. This is generally
2447	/// true when both vcall CFI and whole-program-vtables are enabled.
2448	bool ShouldEmitVTableTypeCheckedLoad(const CXXRecordDecl *RD);
2449
2450	/// Emit a type checked load from the given vtable.
2451	llvm::Value *EmitVTableTypeCheckedLoad(const CXXRecordDecl *RD,
2452	llvm::Value *VTable,
2453	llvm::Type *VTableTy,
2454	uint64_t VTableByteOffset);
2455
2456	/// EnterDtorCleanups - Enter the cleanups necessary to complete the
2457	/// given phase of destruction for a destructor. The end result
2458	/// should call destructors on members and base classes in reverse
2459	/// order of their construction.
2460	void EnterDtorCleanups(const CXXDestructorDecl *Dtor, CXXDtorType Type);
2461
2462	/// ShouldInstrumentFunction - Return true if the current function should be
2463	/// instrumented with __cyg_profile_func_* calls
2464	bool ShouldInstrumentFunction();
2465
2466	/// ShouldSkipSanitizerInstrumentation - Return true if the current function
2467	/// should not be instrumented with sanitizers.
2468	bool ShouldSkipSanitizerInstrumentation();
2469
2470	/// ShouldXRayInstrument - Return true if the current function should be
2471	/// instrumented with XRay nop sleds.
2472	bool ShouldXRayInstrumentFunction() const;
2473
2474	/// AlwaysEmitXRayCustomEvents - Return true if we must unconditionally emit
2475	/// XRay custom event handling calls.
2476	bool AlwaysEmitXRayCustomEvents() const;
2477
2478	/// AlwaysEmitXRayTypedEvents - Return true if clang must unconditionally emit
2479	/// XRay typed event handling calls.
2480	bool AlwaysEmitXRayTypedEvents() const;
2481
2482	/// Return a type hash constant for a function instrumented by
2483	/// -fsanitize=function.
2484	llvm::ConstantInt *getUBSanFunctionTypeHash(QualType T) const;
2485
2486	/// EmitFunctionProlog - Emit the target specific LLVM code to load the
2487	/// arguments for the given function. This is also responsible for naming the
2488	/// LLVM function arguments.
2489	void EmitFunctionProlog(const CGFunctionInfo &FI,
2490	llvm::Function *Fn,
2491	const FunctionArgList &Args);
2492
2493	/// EmitFunctionEpilog - Emit the target specific LLVM code to return the
2494	/// given temporary.
2495	void EmitFunctionEpilog(const CGFunctionInfo &FI, bool EmitRetDbgLoc,
2496	SourceLocation EndLoc);
2497
2498	/// Emit a test that checks if the return value \p RV is nonnull.
2499	void EmitReturnValueCheck(llvm::Value *RV);
2500
2501	/// EmitStartEHSpec - Emit the start of the exception spec.
2502	void EmitStartEHSpec(const Decl *D);
2503
2504	/// EmitEndEHSpec - Emit the end of the exception spec.
2505	void EmitEndEHSpec(const Decl *D);
2506
2507	/// getTerminateLandingPad - Return a landing pad that just calls terminate.
2508	llvm::BasicBlock *getTerminateLandingPad();
2509
2510	/// getTerminateLandingPad - Return a cleanup funclet that just calls
2511	/// terminate.
2512	llvm::BasicBlock *getTerminateFunclet();
2513
2514	/// getTerminateHandler - Return a handler (not a landing pad, just
2515	/// a catch handler) that just calls terminate. This is used when
2516	/// a terminate scope encloses a try.
2517	llvm::BasicBlock *getTerminateHandler();
2518
2519	llvm::Type *ConvertTypeForMem(QualType T);
2520	llvm::Type *ConvertType(QualType T);
2521	llvm::Type *ConvertType(const TypeDecl *T) {
2522	return ConvertType(T: getContext().getTypeDeclType(Decl: T));
2523	}
2524
2525	/// LoadObjCSelf - Load the value of self. This function is only valid while
2526	/// generating code for an Objective-C method.
2527	llvm::Value *LoadObjCSelf();
2528
2529	/// TypeOfSelfObject - Return type of object that this self represents.
2530	QualType TypeOfSelfObject();
2531
2532	/// getEvaluationKind - Return the TypeEvaluationKind of QualType \c T.
2533	static TypeEvaluationKind getEvaluationKind(QualType T);
2534
2535	static bool hasScalarEvaluationKind(QualType T) {
2536	return getEvaluationKind(T) == TEK_Scalar;
2537	}
2538
2539	static bool hasAggregateEvaluationKind(QualType T) {
2540	return getEvaluationKind(T) == TEK_Aggregate;
2541	}
2542
2543	/// createBasicBlock - Create an LLVM basic block.
2544	llvm::BasicBlock *createBasicBlock(const Twine &name = "",
2545	llvm::Function *parent = nullptr,
2546	llvm::BasicBlock *before = nullptr) {
2547	return llvm::BasicBlock::Create(Context&: getLLVMContext(), Name: name, Parent: parent, InsertBefore: before);
2548	}
2549
2550	/// getBasicBlockForLabel - Return the LLVM basicblock that the specified
2551	/// label maps to.
2552	JumpDest getJumpDestForLabel(const LabelDecl *S);
2553
2554	/// SimplifyForwardingBlocks - If the given basic block is only a branch to
2555	/// another basic block, simplify it. This assumes that no other code could
2556	/// potentially reference the basic block.
2557	void SimplifyForwardingBlocks(llvm::BasicBlock *BB);
2558
2559	/// EmitBlock - Emit the given block \arg BB and set it as the insert point,
2560	/// adding a fall-through branch from the current insert block if
2561	/// necessary. It is legal to call this function even if there is no current
2562	/// insertion point.
2563	///
2564	/// IsFinished - If true, indicates that the caller has finished emitting
2565	/// branches to the given block and does not expect to emit code into it. This
2566	/// means the block can be ignored if it is unreachable.
2567	void EmitBlock(llvm::BasicBlock *BB, bool IsFinished=false);
2568
2569	/// EmitBlockAfterUses - Emit the given block somewhere hopefully
2570	/// near its uses, and leave the insertion point in it.
2571	void EmitBlockAfterUses(llvm::BasicBlock *BB);
2572
2573	/// EmitBranch - Emit a branch to the specified basic block from the current
2574	/// insert block, taking care to avoid creation of branches from dummy
2575	/// blocks. It is legal to call this function even if there is no current
2576	/// insertion point.
2577	///
2578	/// This function clears the current insertion point. The caller should follow
2579	/// calls to this function with calls to Emit*Block prior to generation new
2580	/// code.
2581	void EmitBranch(llvm::BasicBlock *Block);
2582
2583	/// HaveInsertPoint - True if an insertion point is defined. If not, this
2584	/// indicates that the current code being emitted is unreachable.
2585	bool HaveInsertPoint() const {
2586	return Builder.GetInsertBlock() != nullptr;
2587	}
2588
2589	/// EnsureInsertPoint - Ensure that an insertion point is defined so that
2590	/// emitted IR has a place to go. Note that by definition, if this function
2591	/// creates a block then that block is unreachable; callers may do better to
2592	/// detect when no insertion point is defined and simply skip IR generation.
2593	void EnsureInsertPoint() {
2594	if (!HaveInsertPoint())
2595	EmitBlock(BB: createBasicBlock());
2596	}
2597
2598	/// ErrorUnsupported - Print out an error that codegen doesn't support the
2599	/// specified stmt yet.
2600	void ErrorUnsupported(const Stmt *S, const char *Type);
2601
2602	//===--------------------------------------------------------------------===//
2603	// Helpers
2604	//===--------------------------------------------------------------------===//
2605
2606	Address mergeAddressesInConditionalExpr(Address LHS, Address RHS,
2607	llvm::BasicBlock *LHSBlock,
2608	llvm::BasicBlock *RHSBlock,
2609	llvm::BasicBlock *MergeBlock,
2610	QualType MergedType) {
2611	Builder.SetInsertPoint(MergeBlock);
2612	llvm::PHINode *PtrPhi = Builder.CreatePHI(Ty: LHS.getType(), NumReservedValues: 2, Name: "cond");
2613	PtrPhi->addIncoming(V: LHS.getBasePointer(), BB: LHSBlock);
2614	PtrPhi->addIncoming(V: RHS.getBasePointer(), BB: RHSBlock);
2615	LHS.replaceBasePointer(P: PtrPhi);
2616	LHS.setAlignment(std::min(a: LHS.getAlignment(), b: RHS.getAlignment()));
2617	return LHS;
2618	}
2619
2620	/// Construct an address with the natural alignment of T. If a pointer to T
2621	/// is expected to be signed, the pointer passed to this function must have
2622	/// been signed, and the returned Address will have the pointer authentication
2623	/// information needed to authenticate the signed pointer.
2624	Address makeNaturalAddressForPointer(
2625	llvm::Value *Ptr, QualType T, CharUnits Alignment = CharUnits::Zero(),
2626	bool ForPointeeType = false, LValueBaseInfo *BaseInfo = nullptr,
2627	TBAAAccessInfo *TBAAInfo = nullptr,
2628	KnownNonNull_t IsKnownNonNull = NotKnownNonNull) {
2629	if (Alignment.isZero())
2630	Alignment =
2631	CGM.getNaturalTypeAlignment(T, BaseInfo, TBAAInfo, forPointeeType: ForPointeeType);
2632	return Address(Ptr, ConvertTypeForMem(T), Alignment, nullptr,
2633	IsKnownNonNull);
2634	}
2635
2636	LValue MakeAddrLValue(Address Addr, QualType T,
2637	AlignmentSource Source = AlignmentSource::Type) {
2638	return MakeAddrLValue(Addr, T, BaseInfo: LValueBaseInfo(Source),
2639	TBAAInfo: CGM.getTBAAAccessInfo(AccessType: T));
2640	}
2641
2642	LValue MakeAddrLValue(Address Addr, QualType T, LValueBaseInfo BaseInfo,
2643	TBAAAccessInfo TBAAInfo) {
2644	return LValue::MakeAddr(Addr, type: T, Context&: getContext(), BaseInfo, TBAAInfo);
2645	}
2646
2647	LValue MakeAddrLValue(llvm::Value *V, QualType T, CharUnits Alignment,
2648	AlignmentSource Source = AlignmentSource::Type) {
2649	return MakeAddrLValue(Addr: makeNaturalAddressForPointer(Ptr: V, T, Alignment), T,
2650	BaseInfo: LValueBaseInfo(Source), TBAAInfo: CGM.getTBAAAccessInfo(AccessType: T));
2651	}
2652
2653	/// Same as MakeAddrLValue above except that the pointer is known to be
2654	/// unsigned.
2655	LValue MakeRawAddrLValue(llvm::Value *V, QualType T, CharUnits Alignment,
2656	AlignmentSource Source = AlignmentSource::Type) {
2657	Address Addr(V, ConvertTypeForMem(T), Alignment);
2658	return LValue::MakeAddr(Addr, type: T, Context&: getContext(), BaseInfo: LValueBaseInfo(Source),
2659	TBAAInfo: CGM.getTBAAAccessInfo(AccessType: T));
2660	}
2661
2662	LValue
2663	MakeAddrLValueWithoutTBAA(Address Addr, QualType T,
2664	AlignmentSource Source = AlignmentSource::Type) {
2665	return LValue::MakeAddr(Addr, type: T, Context&: getContext(), BaseInfo: LValueBaseInfo(Source),
2666	TBAAInfo: TBAAAccessInfo());
2667	}
2668
2669	/// Given a value of type T* that may not be to a complete object, construct
2670	/// an l-value with the natural pointee alignment of T.
2671	LValue MakeNaturalAlignPointeeAddrLValue(llvm::Value *V, QualType T);
2672
2673	LValue MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T);
2674
2675	/// Same as MakeNaturalAlignPointeeAddrLValue except that the pointer is known
2676	/// to be unsigned.
2677	LValue MakeNaturalAlignPointeeRawAddrLValue(llvm::Value *V, QualType T);
2678
2679	LValue MakeNaturalAlignRawAddrLValue(llvm::Value *V, QualType T);
2680
2681	Address EmitLoadOfReference(LValue RefLVal,
2682	LValueBaseInfo *PointeeBaseInfo = nullptr,
2683	TBAAAccessInfo *PointeeTBAAInfo = nullptr);
2684	LValue EmitLoadOfReferenceLValue(LValue RefLVal);
2685	LValue EmitLoadOfReferenceLValue(Address RefAddr, QualType RefTy,
2686	AlignmentSource Source =
2687	AlignmentSource::Type) {
2688	LValue RefLVal = MakeAddrLValue(Addr: RefAddr, T: RefTy, BaseInfo: LValueBaseInfo(Source),
2689	TBAAInfo: CGM.getTBAAAccessInfo(AccessType: RefTy));
2690	return EmitLoadOfReferenceLValue(RefLVal);
2691	}
2692
2693	/// Load a pointer with type \p PtrTy stored at address \p Ptr.
2694	/// Note that \p PtrTy is the type of the loaded pointer, not the addresses
2695	/// it is loaded from.
2696	Address EmitLoadOfPointer(Address Ptr, const PointerType *PtrTy,
2697	LValueBaseInfo *BaseInfo = nullptr,
2698	TBAAAccessInfo *TBAAInfo = nullptr);
2699	LValue EmitLoadOfPointerLValue(Address Ptr, const PointerType *PtrTy);
2700
2701	/// CreateTempAlloca - This creates an alloca and inserts it into the entry
2702	/// block if \p ArraySize is nullptr, otherwise inserts it at the current
2703	/// insertion point of the builder. The caller is responsible for setting an
2704	/// appropriate alignment on
2705	/// the alloca.
2706	///
2707	/// \p ArraySize is the number of array elements to be allocated if it
2708	/// is not nullptr.
2709	///
2710	/// LangAS::Default is the address space of pointers to local variables and
2711	/// temporaries, as exposed in the source language. In certain
2712	/// configurations, this is not the same as the alloca address space, and a
2713	/// cast is needed to lift the pointer from the alloca AS into
2714	/// LangAS::Default. This can happen when the target uses a restricted
2715	/// address space for the stack but the source language requires
2716	/// LangAS::Default to be a generic address space. The latter condition is
2717	/// common for most programming languages; OpenCL is an exception in that
2718	/// LangAS::Default is the private address space, which naturally maps
2719	/// to the stack.
2720	///
2721	/// Because the address of a temporary is often exposed to the program in
2722	/// various ways, this function will perform the cast. The original alloca
2723	/// instruction is returned through \p Alloca if it is not nullptr.
2724	///
2725	/// The cast is not performaed in CreateTempAllocaWithoutCast. This is
2726	/// more efficient if the caller knows that the address will not be exposed.
2727	llvm::AllocaInst *CreateTempAlloca(llvm::Type *Ty, const Twine &Name = "tmp",
2728	llvm::Value *ArraySize = nullptr);
2729	RawAddress CreateTempAlloca(llvm::Type *Ty, CharUnits align,
2730	const Twine &Name = "tmp",
2731	llvm::Value *ArraySize = nullptr,
2732	RawAddress *Alloca = nullptr);
2733	RawAddress CreateTempAllocaWithoutCast(llvm::Type *Ty, CharUnits align,
2734	const Twine &Name = "tmp",
2735	llvm::Value *ArraySize = nullptr);
2736
2737	/// CreateDefaultAlignedTempAlloca - This creates an alloca with the
2738	/// default ABI alignment of the given LLVM type.
2739	///
2740	/// IMPORTANT NOTE: This is *not* generally the right alignment for
2741	/// any given AST type that happens to have been lowered to the
2742	/// given IR type. This should only ever be used for function-local,
2743	/// IR-driven manipulations like saving and restoring a value. Do
2744	/// not hand this address off to arbitrary IRGen routines, and especially
2745	/// do not pass it as an argument to a function that might expect a
2746	/// properly ABI-aligned value.
2747	RawAddress CreateDefaultAlignTempAlloca(llvm::Type *Ty,
2748	const Twine &Name = "tmp");
2749
2750	/// CreateIRTemp - Create a temporary IR object of the given type, with
2751	/// appropriate alignment. This routine should only be used when an temporary
2752	/// value needs to be stored into an alloca (for example, to avoid explicit
2753	/// PHI construction), but the type is the IR type, not the type appropriate
2754	/// for storing in memory.
2755	///
2756	/// That is, this is exactly equivalent to CreateMemTemp, but calling
2757	/// ConvertType instead of ConvertTypeForMem.
2758	RawAddress CreateIRTemp(QualType T, const Twine &Name = "tmp");
2759
2760	/// CreateMemTemp - Create a temporary memory object of the given type, with
2761	/// appropriate alignmen and cast it to the default address space. Returns
2762	/// the original alloca instruction by \p Alloca if it is not nullptr.
2763	RawAddress CreateMemTemp(QualType T, const Twine &Name = "tmp",
2764	RawAddress *Alloca = nullptr);
2765	RawAddress CreateMemTemp(QualType T, CharUnits Align,
2766	const Twine &Name = "tmp",
2767	RawAddress *Alloca = nullptr);
2768
2769	/// CreateMemTemp - Create a temporary memory object of the given type, with
2770	/// appropriate alignmen without casting it to the default address space.
2771	RawAddress CreateMemTempWithoutCast(QualType T, const Twine &Name = "tmp");
2772	RawAddress CreateMemTempWithoutCast(QualType T, CharUnits Align,
2773	const Twine &Name = "tmp");
2774
2775	/// CreateAggTemp - Create a temporary memory object for the given
2776	/// aggregate type.
2777	AggValueSlot CreateAggTemp(QualType T, const Twine &Name = "tmp",
2778	RawAddress *Alloca = nullptr) {
2779	return AggValueSlot::forAddr(
2780	addr: CreateMemTemp(T, Name, Alloca), quals: T.getQualifiers(),
2781	isDestructed: AggValueSlot::IsNotDestructed, needsGC: AggValueSlot::DoesNotNeedGCBarriers,
2782	isAliased: AggValueSlot::IsNotAliased, mayOverlap: AggValueSlot::DoesNotOverlap);
2783	}
2784
2785	/// EvaluateExprAsBool - Perform the usual unary conversions on the specified
2786	/// expression and compare the result against zero, returning an Int1Ty value.
2787	llvm::Value *EvaluateExprAsBool(const Expr *E);
2788
2789	/// Retrieve the implicit cast expression of the rhs in a binary operator
2790	/// expression by passing pointers to Value and QualType
2791	/// This is used for implicit bitfield conversion checks, which
2792	/// must compare with the value before potential truncation.
2793	llvm::Value *EmitWithOriginalRHSBitfieldAssignment(const BinaryOperator *E,
2794	llvm::Value **Previous,
2795	QualType *SrcType);
2796
2797	/// Emit a check that an [implicit] conversion of a bitfield. It is not UB,
2798	/// so we use the value after conversion.
2799	void EmitBitfieldConversionCheck(llvm::Value *Src, QualType SrcType,
2800	llvm::Value *Dst, QualType DstType,
2801	const CGBitFieldInfo &Info,
2802	SourceLocation Loc);
2803
2804	/// EmitIgnoredExpr - Emit an expression in a context which ignores the result.
2805	void EmitIgnoredExpr(const Expr *E);
2806
2807	/// EmitAnyExpr - Emit code to compute the specified expression which can have
2808	/// any type. The result is returned as an RValue struct. If this is an
2809	/// aggregate expression, the aggloc/agglocvolatile arguments indicate where
2810	/// the result should be returned.
2811	///
2812	/// \param ignoreResult True if the resulting value isn't used.
2813	RValue EmitAnyExpr(const Expr *E,
2814	AggValueSlot aggSlot = AggValueSlot::ignored(),
2815	bool ignoreResult = false);
2816
2817	// EmitVAListRef - Emit a "reference" to a va_list; this is either the address
2818	// or the value of the expression, depending on how va_list is defined.
2819	Address EmitVAListRef(const Expr *E);
2820
2821	/// Emit a "reference" to a __builtin_ms_va_list; this is
2822	/// always the value of the expression, because a __builtin_ms_va_list is a
2823	/// pointer to a char.
2824	Address EmitMSVAListRef(const Expr *E);
2825
2826	/// EmitAnyExprToTemp - Similarly to EmitAnyExpr(), however, the result will
2827	/// always be accessible even if no aggregate location is provided.
2828	RValue EmitAnyExprToTemp(const Expr *E);
2829
2830	/// EmitAnyExprToMem - Emits the code necessary to evaluate an
2831	/// arbitrary expression into the given memory location.
2832	void EmitAnyExprToMem(const Expr *E, Address Location,
2833	Qualifiers Quals, bool IsInitializer);
2834
2835	void EmitAnyExprToExn(const Expr *E, Address Addr);
2836
2837	/// EmitExprAsInit - Emits the code necessary to initialize a
2838	/// location in memory with the given initializer.
2839	void EmitExprAsInit(const Expr *init, const ValueDecl *D, LValue lvalue,
2840	bool capturedByInit);
2841
2842	/// hasVolatileMember - returns true if aggregate type has a volatile
2843	/// member.
2844	bool hasVolatileMember(QualType T) {
2845	if (const RecordType *RT = T->getAs<RecordType>()) {
2846	const RecordDecl *RD = cast<RecordDecl>(Val: RT->getDecl());
2847	return RD->hasVolatileMember();
2848	}
2849	return false;
2850	}
2851
2852	/// Determine whether a return value slot may overlap some other object.
2853	AggValueSlot::Overlap_t getOverlapForReturnValue() {
2854	// FIXME: Assuming no overlap here breaks guaranteed copy elision for base
2855	// class subobjects. These cases may need to be revisited depending on the
2856	// resolution of the relevant core issue.
2857	return AggValueSlot::DoesNotOverlap;
2858	}
2859
2860	/// Determine whether a field initialization may overlap some other object.
2861	AggValueSlot::Overlap_t getOverlapForFieldInit(const FieldDecl *FD);
2862
2863	/// Determine whether a base class initialization may overlap some other
2864	/// object.
2865	AggValueSlot::Overlap_t getOverlapForBaseInit(const CXXRecordDecl *RD,
2866	const CXXRecordDecl *BaseRD,
2867	bool IsVirtual);
2868
2869	/// Emit an aggregate assignment.
2870	void EmitAggregateAssign(LValue Dest, LValue Src, QualType EltTy) {
2871	bool IsVolatile = hasVolatileMember(T: EltTy);
2872	EmitAggregateCopy(Dest, Src, EltTy, MayOverlap: AggValueSlot::MayOverlap, isVolatile: IsVolatile);
2873	}
2874
2875	void EmitAggregateCopyCtor(LValue Dest, LValue Src,
2876	AggValueSlot::Overlap_t MayOverlap) {
2877	EmitAggregateCopy(Dest, Src, EltTy: Src.getType(), MayOverlap);
2878	}
2879
2880	/// EmitAggregateCopy - Emit an aggregate copy.
2881	///
2882	/// \param isVolatile \c true iff either the source or the destination is
2883	/// volatile.
2884	/// \param MayOverlap Whether the tail padding of the destination might be
2885	/// occupied by some other object. More efficient code can often be
2886	/// generated if not.
2887	void EmitAggregateCopy(LValue Dest, LValue Src, QualType EltTy,
2888	AggValueSlot::Overlap_t MayOverlap,
2889	bool isVolatile = false);
2890
2891	/// GetAddrOfLocalVar - Return the address of a local variable.
2892	Address GetAddrOfLocalVar(const VarDecl *VD) {
2893	auto it = LocalDeclMap.find(VD);
2894	assert(it != LocalDeclMap.end() &&
2895	"Invalid argument to GetAddrOfLocalVar(), no decl!");
2896	return it->second;
2897	}
2898
2899	/// Given an opaque value expression, return its LValue mapping if it exists,
2900	/// otherwise create one.
2901	LValue getOrCreateOpaqueLValueMapping(const OpaqueValueExpr *e);
2902
2903	/// Given an opaque value expression, return its RValue mapping if it exists,
2904	/// otherwise create one.
2905	RValue getOrCreateOpaqueRValueMapping(const OpaqueValueExpr *e);
2906
2907	/// Get the index of the current ArrayInitLoopExpr, if any.
2908	llvm::Value *getArrayInitIndex() { return ArrayInitIndex; }
2909
2910	/// getAccessedFieldNo - Given an encoded value and a result number, return
2911	/// the input field number being accessed.
2912	static unsigned getAccessedFieldNo(unsigned Idx, const llvm::Constant *Elts);
2913
2914	llvm::BlockAddress *GetAddrOfLabel(const LabelDecl *L);
2915	llvm::BasicBlock *GetIndirectGotoBlock();
2916
2917	/// Check if \p E is a C++ "this" pointer wrapped in value-preserving casts.
2918	static bool IsWrappedCXXThis(const Expr *E);
2919
2920	/// EmitNullInitialization - Generate code to set a value of the given type to
2921	/// null, If the type contains data member pointers, they will be initialized
2922	/// to -1 in accordance with the Itanium C++ ABI.
2923	void EmitNullInitialization(Address DestPtr, QualType Ty);
2924
2925	/// Emits a call to an LLVM variable-argument intrinsic, either
2926	/// \c llvm.va_start or \c llvm.va_end.
2927	/// \param ArgValue A reference to the \c va_list as emitted by either
2928	/// \c EmitVAListRef or \c EmitMSVAListRef.
2929	/// \param IsStart If \c true, emits a call to \c llvm.va_start; otherwise,
2930	/// calls \c llvm.va_end.
2931	llvm::Value *EmitVAStartEnd(llvm::Value *ArgValue, bool IsStart);
2932
2933	/// Generate code to get an argument from the passed in pointer
2934	/// and update it accordingly.
2935	/// \param VE The \c VAArgExpr for which to generate code.
2936	/// \param VAListAddr Receives a reference to the \c va_list as emitted by
2937	/// either \c EmitVAListRef or \c EmitMSVAListRef.
2938	/// \returns A pointer to the argument.
2939	// FIXME: We should be able to get rid of this method and use the va_arg
2940	// instruction in LLVM instead once it works well enough.
2941	Address EmitVAArg(VAArgExpr *VE, Address &VAListAddr);
2942
2943	/// emitArrayLength - Compute the length of an array, even if it's a
2944	/// VLA, and drill down to the base element type.
2945	llvm::Value *emitArrayLength(const ArrayType *arrayType,
2946	QualType &baseType,
2947	Address &addr);
2948
2949	/// EmitVLASize - Capture all the sizes for the VLA expressions in
2950	/// the given variably-modified type and store them in the VLASizeMap.
2951	///
2952	/// This function can be called with a null (unreachable) insert point.
2953	void EmitVariablyModifiedType(QualType Ty);
2954
2955	struct VlaSizePair {
2956	llvm::Value *NumElts;
2957	QualType Type;
2958
2959	VlaSizePair(llvm::Value *NE, QualType T) : NumElts(NE), Type(T) {}
2960	};
2961
2962	/// Return the number of elements for a single dimension
2963	/// for the given array type.
2964	VlaSizePair getVLAElements1D(const VariableArrayType *vla);
2965	VlaSizePair getVLAElements1D(QualType vla);
2966
2967	/// Returns an LLVM value that corresponds to the size,
2968	/// in non-variably-sized elements, of a variable length array type,
2969	/// plus that largest non-variably-sized element type. Assumes that
2970	/// the type has already been emitted with EmitVariablyModifiedType.
2971	VlaSizePair getVLASize(const VariableArrayType *vla);
2972	VlaSizePair getVLASize(QualType vla);
2973
2974	/// LoadCXXThis - Load the value of 'this'. This function is only valid while
2975	/// generating code for an C++ member function.
2976	llvm::Value *LoadCXXThis() {
2977	assert(CXXThisValue && "no 'this' value for this function");
2978	return CXXThisValue;
2979	}
2980	Address LoadCXXThisAddress();
2981
2982	/// LoadCXXVTT - Load the VTT parameter to base constructors/destructors have
2983	/// virtual bases.
2984	// FIXME: Every place that calls LoadCXXVTT is something
2985	// that needs to be abstracted properly.
2986	llvm::Value *LoadCXXVTT() {
2987	assert(CXXStructorImplicitParamValue && "no VTT value for this function");
2988	return CXXStructorImplicitParamValue;
2989	}
2990
2991	/// GetAddressOfBaseOfCompleteClass - Convert the given pointer to a
2992	/// complete class to the given direct base.
2993	Address
2994	GetAddressOfDirectBaseInCompleteClass(Address Value,
2995	const CXXRecordDecl *Derived,
2996	const CXXRecordDecl *Base,
2997	bool BaseIsVirtual);
2998
2999	static bool ShouldNullCheckClassCastValue(const CastExpr *Cast);
3000
3001	/// GetAddressOfBaseClass - This function will add the necessary delta to the
3002	/// load of 'this' and returns address of the base class.
3003	Address GetAddressOfBaseClass(Address Value,
3004	const CXXRecordDecl *Derived,
3005	CastExpr::path_const_iterator PathBegin,
3006	CastExpr::path_const_iterator PathEnd,
3007	bool NullCheckValue, SourceLocation Loc);
3008
3009	Address GetAddressOfDerivedClass(Address Value,
3010	const CXXRecordDecl *Derived,
3011	CastExpr::path_const_iterator PathBegin,
3012	CastExpr::path_const_iterator PathEnd,
3013	bool NullCheckValue);
3014
3015	/// GetVTTParameter - Return the VTT parameter that should be passed to a
3016	/// base constructor/destructor with virtual bases.
3017	/// FIXME: VTTs are Itanium ABI-specific, so the definition should move
3018	/// to ItaniumCXXABI.cpp together with all the references to VTT.
3019	llvm::Value *GetVTTParameter(GlobalDecl GD, bool ForVirtualBase,
3020	bool Delegating);
3021
3022	void EmitDelegateCXXConstructorCall(const CXXConstructorDecl *Ctor,
3023	CXXCtorType CtorType,
3024	const FunctionArgList &Args,
3025	SourceLocation Loc);
3026	// It's important not to confuse this and the previous function. Delegating
3027	// constructors are the C++0x feature. The constructor delegate optimization
3028	// is used to reduce duplication in the base and complete consturctors where
3029	// they are substantially the same.
3030	void EmitDelegatingCXXConstructorCall(const CXXConstructorDecl *Ctor,
3031	const FunctionArgList &Args);
3032
3033	/// Emit a call to an inheriting constructor (that is, one that invokes a
3034	/// constructor inherited from a base class) by inlining its definition. This
3035	/// is necessary if the ABI does not support forwarding the arguments to the
3036	/// base class constructor (because they're variadic or similar).
3037	void EmitInlinedInheritingCXXConstructorCall(const CXXConstructorDecl *Ctor,
3038	CXXCtorType CtorType,
3039	bool ForVirtualBase,
3040	bool Delegating,
3041	CallArgList &Args);
3042
3043	/// Emit a call to a constructor inherited from a base class, passing the
3044	/// current constructor's arguments along unmodified (without even making
3045	/// a copy).
3046	void EmitInheritedCXXConstructorCall(const CXXConstructorDecl *D,
3047	bool ForVirtualBase, Address This,
3048	bool InheritedFromVBase,
3049	const CXXInheritedCtorInitExpr *E);
3050
3051	void EmitCXXConstructorCall(const CXXConstructorDecl *D, CXXCtorType Type,
3052	bool ForVirtualBase, bool Delegating,
3053	AggValueSlot ThisAVS, const CXXConstructExpr *E);
3054
3055	void EmitCXXConstructorCall(const CXXConstructorDecl *D, CXXCtorType Type,
3056	bool ForVirtualBase, bool Delegating,
3057	Address This, CallArgList &Args,
3058	AggValueSlot::Overlap_t Overlap,
3059	SourceLocation Loc, bool NewPointerIsChecked);
3060
3061	/// Emit assumption load for all bases. Requires to be called only on
3062	/// most-derived class and not under construction of the object.
3063	void EmitVTableAssumptionLoads(const CXXRecordDecl *ClassDecl, Address This);
3064
3065	/// Emit assumption that vptr load == global vtable.
3066	void EmitVTableAssumptionLoad(const VPtr &vptr, Address This);
3067
3068	void EmitSynthesizedCXXCopyCtorCall(const CXXConstructorDecl *D,
3069	Address This, Address Src,
3070	const CXXConstructExpr *E);
3071
3072	void EmitCXXAggrConstructorCall(const CXXConstructorDecl *D,
3073	const ArrayType *ArrayTy,
3074	Address ArrayPtr,
3075	const CXXConstructExpr *E,
3076	bool NewPointerIsChecked,
3077	bool ZeroInitialization = false);
3078
3079	void EmitCXXAggrConstructorCall(const CXXConstructorDecl *D,
3080	llvm::Value *NumElements,
3081	Address ArrayPtr,
3082	const CXXConstructExpr *E,
3083	bool NewPointerIsChecked,
3084	bool ZeroInitialization = false);
3085
3086	static Destroyer destroyCXXObject;
3087
3088	void EmitCXXDestructorCall(const CXXDestructorDecl *D, CXXDtorType Type,
3089	bool ForVirtualBase, bool Delegating, Address This,
3090	QualType ThisTy);
3091
3092	void EmitNewArrayInitializer(const CXXNewExpr *E, QualType elementType,
3093	llvm::Type *ElementTy, Address NewPtr,
3094	llvm::Value *NumElements,
3095	llvm::Value *AllocSizeWithoutCookie);
3096
3097	void EmitCXXTemporary(const CXXTemporary *Temporary, QualType TempType,
3098	Address Ptr);
3099
3100	void EmitSehCppScopeBegin();
3101	void EmitSehCppScopeEnd();
3102	void EmitSehTryScopeBegin();
3103	void EmitSehTryScopeEnd();
3104
3105	llvm::Value *EmitLifetimeStart(llvm::TypeSize Size, llvm::Value *Addr);
3106	void EmitLifetimeEnd(llvm::Value *Size, llvm::Value *Addr);
3107
3108	llvm::Value *EmitCXXNewExpr(const CXXNewExpr *E);
3109	void EmitCXXDeleteExpr(const CXXDeleteExpr *E);
3110
3111	void EmitDeleteCall(const FunctionDecl *DeleteFD, llvm::Value *Ptr,
3112	QualType DeleteTy, llvm::Value *NumElements = nullptr,
3113	CharUnits CookieSize = CharUnits());
3114
3115	RValue EmitBuiltinNewDeleteCall(const FunctionProtoType *Type,
3116	const CallExpr *TheCallExpr, bool IsDelete);
3117
3118	llvm::Value *EmitCXXTypeidExpr(const CXXTypeidExpr *E);
3119	llvm::Value *EmitDynamicCast(Address V, const CXXDynamicCastExpr *DCE);
3120	Address EmitCXXUuidofExpr(const CXXUuidofExpr *E);
3121
3122	/// Situations in which we might emit a check for the suitability of a
3123	/// pointer or glvalue. Needs to be kept in sync with ubsan_handlers.cpp in
3124	/// compiler-rt.
3125	enum TypeCheckKind {
3126	/// Checking the operand of a load. Must be suitably sized and aligned.
3127	TCK_Load,
3128	/// Checking the destination of a store. Must be suitably sized and aligned.
3129	TCK_Store,
3130	/// Checking the bound value in a reference binding. Must be suitably sized
3131	/// and aligned, but is not required to refer to an object (until the
3132	/// reference is used), per core issue 453.
3133	TCK_ReferenceBinding,
3134	/// Checking the object expression in a non-static data member access. Must
3135	/// be an object within its lifetime.
3136	TCK_MemberAccess,
3137	/// Checking the 'this' pointer for a call to a non-static member function.
3138	/// Must be an object within its lifetime.
3139	TCK_MemberCall,
3140	/// Checking the 'this' pointer for a constructor call.
3141	TCK_ConstructorCall,
3142	/// Checking the operand of a static_cast to a derived pointer type. Must be
3143	/// null or an object within its lifetime.
3144	TCK_DowncastPointer,
3145	/// Checking the operand of a static_cast to a derived reference type. Must
3146	/// be an object within its lifetime.
3147	TCK_DowncastReference,
3148	/// Checking the operand of a cast to a base object. Must be suitably sized
3149	/// and aligned.
3150	TCK_Upcast,
3151	/// Checking the operand of a cast to a virtual base object. Must be an
3152	/// object within its lifetime.
3153	TCK_UpcastToVirtualBase,
3154	/// Checking the value assigned to a _Nonnull pointer. Must not be null.
3155	TCK_NonnullAssign,
3156	/// Checking the operand of a dynamic_cast or a typeid expression. Must be
3157	/// null or an object within its lifetime.
3158	TCK_DynamicOperation
3159	};
3160
3161	/// Determine whether the pointer type check \p TCK permits null pointers.
3162	static bool isNullPointerAllowed(TypeCheckKind TCK);
3163
3164	/// Determine whether the pointer type check \p TCK requires a vptr check.
3165	static bool isVptrCheckRequired(TypeCheckKind TCK, QualType Ty);
3166
3167	/// Whether any type-checking sanitizers are enabled. If \c false,
3168	/// calls to EmitTypeCheck can be skipped.
3169	bool sanitizePerformTypeCheck() const;
3170
3171	void EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc, LValue LV,
3172	QualType Type, SanitizerSet SkippedChecks = SanitizerSet(),
3173	llvm::Value *ArraySize = nullptr) {
3174	if (!sanitizePerformTypeCheck())
3175	return;
3176	EmitTypeCheck(TCK, Loc, V: LV.emitRawPointer(CGF&: *this), Type, Alignment: LV.getAlignment(),
3177	SkippedChecks, ArraySize);
3178	}
3179
3180	void EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc, Address Addr,
3181	QualType Type, CharUnits Alignment = CharUnits::Zero(),
3182	SanitizerSet SkippedChecks = SanitizerSet(),
3183	llvm::Value *ArraySize = nullptr) {
3184	if (!sanitizePerformTypeCheck())
3185	return;
3186	EmitTypeCheck(TCK, Loc, V: Addr.emitRawPointer(CGF&: *this), Type, Alignment,
3187	SkippedChecks, ArraySize);
3188	}
3189
3190	/// Emit a check that \p V is the address of storage of the
3191	/// appropriate size and alignment for an object of type \p Type
3192	/// (or if ArraySize is provided, for an array of that bound).
3193	void EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc, llvm::Value *V,
3194	QualType Type, CharUnits Alignment = CharUnits::Zero(),
3195	SanitizerSet SkippedChecks = SanitizerSet(),
3196	llvm::Value *ArraySize = nullptr);
3197
3198	/// Emit a check that \p Base points into an array object, which
3199	/// we can access at index \p Index. \p Accessed should be \c false if we
3200	/// this expression is used as an lvalue, for instance in "&Arr[Idx]".
3201	void EmitBoundsCheck(const Expr *E, const Expr *Base, llvm::Value *Index,
3202	QualType IndexType, bool Accessed);
3203	void EmitBoundsCheckImpl(const Expr *E, llvm::Value *Bound,
3204	llvm::Value *Index, QualType IndexType,
3205	QualType IndexedType, bool Accessed);
3206
3207	// Find a struct's flexible array member. It may be embedded inside multiple
3208	// sub-structs, but must still be the last field.
3209	const FieldDecl *FindFlexibleArrayMemberField(ASTContext &Ctx,
3210	const RecordDecl *RD,
3211	StringRef Name,
3212	uint64_t &Offset);
3213
3214	/// Find the FieldDecl specified in a FAM's "counted_by" attribute. Returns
3215	/// \p nullptr if either the attribute or the field doesn't exist.
3216	const FieldDecl *FindCountedByField(const FieldDecl *FD);
3217
3218	/// Build an expression accessing the "counted_by" field.
3219	llvm::Value *EmitCountedByFieldExpr(const Expr *Base,
3220	const FieldDecl *FAMDecl,
3221	const FieldDecl *CountDecl);
3222
3223	llvm::Value *EmitScalarPrePostIncDec(const UnaryOperator *E, LValue LV,
3224	bool isInc, bool isPre);
3225	ComplexPairTy EmitComplexPrePostIncDec(const UnaryOperator *E, LValue LV,
3226	bool isInc, bool isPre);
3227
3228	/// Converts Location to a DebugLoc, if debug information is enabled.
3229	llvm::DebugLoc SourceLocToDebugLoc(SourceLocation Location);
3230
3231	/// Get the record field index as represented in debug info.
3232	unsigned getDebugInfoFIndex(const RecordDecl *Rec, unsigned FieldIndex);
3233
3234
3235	//===--------------------------------------------------------------------===//
3236	// Declaration Emission
3237	//===--------------------------------------------------------------------===//
3238
3239	/// EmitDecl - Emit a declaration.
3240	///
3241	/// This function can be called with a null (unreachable) insert point.
3242	void EmitDecl(const Decl &D);
3243
3244	/// EmitVarDecl - Emit a local variable declaration.
3245	///
3246	/// This function can be called with a null (unreachable) insert point.
3247	void EmitVarDecl(const VarDecl &D);
3248
3249	void EmitScalarInit(const Expr *init, const ValueDecl *D, LValue lvalue,
3250	bool capturedByInit);
3251
3252	typedef void SpecialInitFn(CodeGenFunction &Init, const VarDecl &D,
3253	llvm::Value *Address);
3254
3255	/// Determine whether the given initializer is trivial in the sense
3256	/// that it requires no code to be generated.
3257	bool isTrivialInitializer(const Expr *Init);
3258
3259	/// EmitAutoVarDecl - Emit an auto variable declaration.
3260	///
3261	/// This function can be called with a null (unreachable) insert point.
3262	void EmitAutoVarDecl(const VarDecl &D);
3263
3264	class AutoVarEmission {
3265	friend class CodeGenFunction;
3266
3267	const VarDecl *Variable;
3268
3269	/// The address of the alloca for languages with explicit address space
3270	/// (e.g. OpenCL) or alloca casted to generic pointer for address space
3271	/// agnostic languages (e.g. C++). Invalid if the variable was emitted
3272	/// as a global constant.
3273	Address Addr;
3274
3275	llvm::Value *NRVOFlag;
3276
3277	/// True if the variable is a __block variable that is captured by an
3278	/// escaping block.
3279	bool IsEscapingByRef;
3280
3281	/// True if the variable is of aggregate type and has a constant
3282	/// initializer.
3283	bool IsConstantAggregate;
3284
3285	/// Non-null if we should use lifetime annotations.
3286	llvm::Value *SizeForLifetimeMarkers;
3287
3288	/// Address with original alloca instruction. Invalid if the variable was
3289	/// emitted as a global constant.
3290	RawAddress AllocaAddr;
3291
3292	struct Invalid {};
3293	AutoVarEmission(Invalid)
3294	: Variable(nullptr), Addr(Address::invalid()),
3295	AllocaAddr(RawAddress::invalid()) {}
3296
3297	AutoVarEmission(const VarDecl &variable)
3298	: Variable(&variable), Addr(Address::invalid()), NRVOFlag(nullptr),
3299	IsEscapingByRef(false), IsConstantAggregate(false),
3300	SizeForLifetimeMarkers(nullptr), AllocaAddr(RawAddress::invalid()) {}
3301
3302	bool wasEmittedAsGlobal() const { return !Addr.isValid(); }
3303
3304	public:
3305	static AutoVarEmission invalid() { return AutoVarEmission(Invalid()); }
3306
3307	bool useLifetimeMarkers() const {
3308	return SizeForLifetimeMarkers != nullptr;
3309	}
3310	llvm::Value *getSizeForLifetimeMarkers() const {
3311	assert(useLifetimeMarkers());
3312	return SizeForLifetimeMarkers;
3313	}
3314
3315	/// Returns the raw, allocated address, which is not necessarily
3316	/// the address of the object itself. It is casted to default
3317	/// address space for address space agnostic languages.
3318	Address getAllocatedAddress() const {
3319	return Addr;
3320	}
3321
3322	/// Returns the address for the original alloca instruction.
3323	RawAddress getOriginalAllocatedAddress() const { return AllocaAddr; }
3324
3325	/// Returns the address of the object within this declaration.
3326	/// Note that this does not chase the forwarding pointer for
3327	/// __block decls.
3328	Address getObjectAddress(CodeGenFunction &CGF) const {
3329	if (!IsEscapingByRef) return Addr;
3330
3331	return CGF.emitBlockByrefAddress(baseAddr: Addr, V: Variable, /*forward*/ followForward: false);
3332	}
3333	};
3334	AutoVarEmission EmitAutoVarAlloca(const VarDecl &var);
3335	void EmitAutoVarInit(const AutoVarEmission &emission);
3336	void EmitAutoVarCleanups(const AutoVarEmission &emission);
3337	void emitAutoVarTypeCleanup(const AutoVarEmission &emission,
3338	QualType::DestructionKind dtorKind);
3339
3340	/// Emits the alloca and debug information for the size expressions for each
3341	/// dimension of an array. It registers the association of its (1-dimensional)
3342	/// QualTypes and size expression's debug node, so that CGDebugInfo can
3343	/// reference this node when creating the DISubrange object to describe the
3344	/// array types.
3345	void EmitAndRegisterVariableArrayDimensions(CGDebugInfo *DI,
3346	const VarDecl &D,
3347	bool EmitDebugInfo);
3348
3349	void EmitStaticVarDecl(const VarDecl &D,
3350	llvm::GlobalValue::LinkageTypes Linkage);
3351
3352	class ParamValue {
3353	union {
3354	Address Addr;
3355	llvm::Value *Value;
3356	};
3357
3358	bool IsIndirect;
3359
3360	ParamValue(llvm::Value *V) : Value(V), IsIndirect(false) {}
3361	ParamValue(Address A) : Addr(A), IsIndirect(true) {}
3362
3363	public:
3364	static ParamValue forDirect(llvm::Value *value) {
3365	return ParamValue(value);
3366	}
3367	static ParamValue forIndirect(Address addr) {
3368	assert(!addr.getAlignment().isZero());
3369	return ParamValue(addr);
3370	}
3371
3372	bool isIndirect() const { return IsIndirect; }
3373	llvm::Value *getAnyValue() const {
3374	if (!isIndirect())
3375	return Value;
3376	assert(!Addr.hasOffset() && "unexpected offset");
3377	return Addr.getBasePointer();
3378	}
3379
3380	llvm::Value *getDirectValue() const {
3381	assert(!isIndirect());
3382	return Value;
3383	}
3384
3385	Address getIndirectAddress() const {
3386	assert(isIndirect());
3387	return Addr;
3388	}
3389	};
3390
3391	/// EmitParmDecl - Emit a ParmVarDecl or an ImplicitParamDecl.
3392	void EmitParmDecl(const VarDecl &D, ParamValue Arg, unsigned ArgNo);
3393
3394	/// protectFromPeepholes - Protect a value that we're intending to
3395	/// store to the side, but which will probably be used later, from
3396	/// aggressive peepholing optimizations that might delete it.
3397	///
3398	/// Pass the result to unprotectFromPeepholes to declare that
3399	/// protection is no longer required.
3400	///
3401	/// There's no particular reason why this shouldn't apply to
3402	/// l-values, it's just that no existing peepholes work on pointers.
3403	PeepholeProtection protectFromPeepholes(RValue rvalue);
3404	void unprotectFromPeepholes(PeepholeProtection protection);
3405
3406	void emitAlignmentAssumptionCheck(llvm::Value *Ptr, QualType Ty,
3407	SourceLocation Loc,
3408	SourceLocation AssumptionLoc,
3409	llvm::Value *Alignment,
3410	llvm::Value *OffsetValue,
3411	llvm::Value *TheCheck,
3412	llvm::Instruction *Assumption);
3413
3414	void emitAlignmentAssumption(llvm::Value *PtrValue, QualType Ty,
3415	SourceLocation Loc, SourceLocation AssumptionLoc,
3416	llvm::Value *Alignment,
3417	llvm::Value *OffsetValue = nullptr);
3418
3419	void emitAlignmentAssumption(llvm::Value *PtrValue, const Expr *E,
3420	SourceLocation AssumptionLoc,
3421	llvm::Value *Alignment,
3422	llvm::Value *OffsetValue = nullptr);
3423
3424	//===--------------------------------------------------------------------===//
3425	// Statement Emission
3426	//===--------------------------------------------------------------------===//
3427
3428	/// EmitStopPoint - Emit a debug stoppoint if we are emitting debug info.
3429	void EmitStopPoint(const Stmt *S);
3430
3431	/// EmitStmt - Emit the code for the statement \arg S. It is legal to call
3432	/// this function even if there is no current insertion point.
3433	///
3434	/// This function may clear the current insertion point; callers should use
3435	/// EnsureInsertPoint if they wish to subsequently generate code without first
3436	/// calling EmitBlock, EmitBranch, or EmitStmt.
3437	void EmitStmt(const Stmt *S, ArrayRef<const Attr *> Attrs = std::nullopt);
3438
3439	/// EmitSimpleStmt - Try to emit a "simple" statement which does not
3440	/// necessarily require an insertion point or debug information; typically
3441	/// because the statement amounts to a jump or a container of other
3442	/// statements.
3443	///
3444	/// \return True if the statement was handled.
3445	bool EmitSimpleStmt(const Stmt *S, ArrayRef<const Attr *> Attrs);
3446
3447	Address EmitCompoundStmt(const CompoundStmt &S, bool GetLast = false,
3448	AggValueSlot AVS = AggValueSlot::ignored());
3449	Address EmitCompoundStmtWithoutScope(const CompoundStmt &S,
3450	bool GetLast = false,
3451	AggValueSlot AVS =
3452	AggValueSlot::ignored());
3453
3454	/// EmitLabel - Emit the block for the given label. It is legal to call this
3455	/// function even if there is no current insertion point.
3456	void EmitLabel(const LabelDecl *D); // helper for EmitLabelStmt.
3457
3458	void EmitLabelStmt(const LabelStmt &S);
3459	void EmitAttributedStmt(const AttributedStmt &S);
3460	void EmitGotoStmt(const GotoStmt &S);
3461	void EmitIndirectGotoStmt(const IndirectGotoStmt &S);
3462	void EmitIfStmt(const IfStmt &S);
3463
3464	void EmitWhileStmt(const WhileStmt &S,
3465	ArrayRef<const Attr *> Attrs = std::nullopt);
3466	void EmitDoStmt(const DoStmt &S, ArrayRef<const Attr *> Attrs = std::nullopt);
3467	void EmitForStmt(const ForStmt &S,
3468	ArrayRef<const Attr *> Attrs = std::nullopt);
3469	void EmitReturnStmt(const ReturnStmt &S);
3470	void EmitDeclStmt(const DeclStmt &S);
3471	void EmitBreakStmt(const BreakStmt &S);
3472	void EmitContinueStmt(const ContinueStmt &S);
3473	void EmitSwitchStmt(const SwitchStmt &S);
3474	void EmitDefaultStmt(const DefaultStmt &S, ArrayRef<const Attr *> Attrs);
3475	void EmitCaseStmt(const CaseStmt &S, ArrayRef<const Attr *> Attrs);
3476	void EmitCaseStmtRange(const CaseStmt &S, ArrayRef<const Attr *> Attrs);
3477	void EmitAsmStmt(const AsmStmt &S);
3478
3479	void EmitObjCForCollectionStmt(const ObjCForCollectionStmt &S);
3480	void EmitObjCAtTryStmt(const ObjCAtTryStmt &S);
3481	void EmitObjCAtThrowStmt(const ObjCAtThrowStmt &S);
3482	void EmitObjCAtSynchronizedStmt(const ObjCAtSynchronizedStmt &S);
3483	void EmitObjCAutoreleasePoolStmt(const ObjCAutoreleasePoolStmt &S);
3484
3485	void EmitCoroutineBody(const CoroutineBodyStmt &S);
3486	void EmitCoreturnStmt(const CoreturnStmt &S);
3487	RValue EmitCoawaitExpr(const CoawaitExpr &E,
3488	AggValueSlot aggSlot = AggValueSlot::ignored(),
3489	bool ignoreResult = false);
3490	LValue EmitCoawaitLValue(const CoawaitExpr *E);
3491	RValue EmitCoyieldExpr(const CoyieldExpr &E,
3492	AggValueSlot aggSlot = AggValueSlot::ignored(),
3493	bool ignoreResult = false);
3494	LValue EmitCoyieldLValue(const CoyieldExpr *E);
3495	RValue EmitCoroutineIntrinsic(const CallExpr *E, unsigned int IID);
3496
3497	void EnterCXXTryStmt(const CXXTryStmt &S, bool IsFnTryBlock = false);
3498	void ExitCXXTryStmt(const CXXTryStmt &S, bool IsFnTryBlock = false);
3499
3500	void EmitCXXTryStmt(const CXXTryStmt &S);
3501	void EmitSEHTryStmt(const SEHTryStmt &S);
3502	void EmitSEHLeaveStmt(const SEHLeaveStmt &S);
3503	void EnterSEHTryStmt(const SEHTryStmt &S);
3504	void ExitSEHTryStmt(const SEHTryStmt &S);
3505	void VolatilizeTryBlocks(llvm::BasicBlock *BB,
3506	llvm::SmallPtrSet<llvm::BasicBlock *, 10> &V);
3507
3508	void pushSEHCleanup(CleanupKind kind,
3509	llvm::Function *FinallyFunc);
3510	void startOutlinedSEHHelper(CodeGenFunction &ParentCGF, bool IsFilter,
3511	const Stmt *OutlinedStmt);
3512
3513	llvm::Function *GenerateSEHFilterFunction(CodeGenFunction &ParentCGF,
3514	const SEHExceptStmt &Except);
3515
3516	llvm::Function *GenerateSEHFinallyFunction(CodeGenFunction &ParentCGF,
3517	const SEHFinallyStmt &Finally);
3518
3519	void EmitSEHExceptionCodeSave(CodeGenFunction &ParentCGF,
3520	llvm::Value *ParentFP,
3521	llvm::Value *EntryEBP);
3522	llvm::Value *EmitSEHExceptionCode();
3523	llvm::Value *EmitSEHExceptionInfo();
3524	llvm::Value *EmitSEHAbnormalTermination();
3525
3526	/// Emit simple code for OpenMP directives in Simd-only mode.
3527	void EmitSimpleOMPExecutableDirective(const OMPExecutableDirective &D);
3528
3529	/// Scan the outlined statement for captures from the parent function. For
3530	/// each capture, mark the capture as escaped and emit a call to
3531	/// llvm.localrecover. Insert the localrecover result into the LocalDeclMap.
3532	void EmitCapturedLocals(CodeGenFunction &ParentCGF, const Stmt *OutlinedStmt,
3533	bool IsFilter);
3534
3535	/// Recovers the address of a local in a parent function. ParentVar is the
3536	/// address of the variable used in the immediate parent function. It can
3537	/// either be an alloca or a call to llvm.localrecover if there are nested
3538	/// outlined functions. ParentFP is the frame pointer of the outermost parent
3539	/// frame.
3540	Address recoverAddrOfEscapedLocal(CodeGenFunction &ParentCGF,
3541	Address ParentVar,
3542	llvm::Value *ParentFP);
3543
3544	void EmitCXXForRangeStmt(const CXXForRangeStmt &S,
3545	ArrayRef<const Attr *> Attrs = std::nullopt);
3546
3547	/// Controls insertion of cancellation exit blocks in worksharing constructs.
3548	class OMPCancelStackRAII {
3549	CodeGenFunction &CGF;
3550
3551	public:
3552	OMPCancelStackRAII(CodeGenFunction &CGF, OpenMPDirectiveKind Kind,
3553	bool HasCancel)
3554	: CGF(CGF) {
3555	CGF.OMPCancelStack.enter(CGF, Kind, HasCancel);
3556	}
3557	~OMPCancelStackRAII() { CGF.OMPCancelStack.exit(CGF); }
3558	};
3559
3560	/// Returns calculated size of the specified type.
3561	llvm::Value *getTypeSize(QualType Ty);
3562	LValue InitCapturedStruct(const CapturedStmt &S);
3563	llvm::Function *EmitCapturedStmt(const CapturedStmt &S, CapturedRegionKind K);
3564	llvm::Function *GenerateCapturedStmtFunction(const CapturedStmt &S);
3565	Address GenerateCapturedStmtArgument(const CapturedStmt &S);
3566	llvm::Function *GenerateOpenMPCapturedStmtFunction(const CapturedStmt &S,
3567	SourceLocation Loc);
3568	void GenerateOpenMPCapturedVars(const CapturedStmt &S,
3569	SmallVectorImpl<llvm::Value *> &CapturedVars);
3570	void emitOMPSimpleStore(LValue LVal, RValue RVal, QualType RValTy,
3571	SourceLocation Loc);
3572	/// Perform element by element copying of arrays with type \a
3573	/// OriginalType from \a SrcAddr to \a DestAddr using copying procedure
3574	/// generated by \a CopyGen.
3575	///
3576	/// \param DestAddr Address of the destination array.
3577	/// \param SrcAddr Address of the source array.
3578	/// \param OriginalType Type of destination and source arrays.
3579	/// \param CopyGen Copying procedure that copies value of single array element
3580	/// to another single array element.
3581	void EmitOMPAggregateAssign(
3582	Address DestAddr, Address SrcAddr, QualType OriginalType,
3583	const llvm::function_ref<void(Address, Address)> CopyGen);
3584	/// Emit proper copying of data from one variable to another.
3585	///
3586	/// \param OriginalType Original type of the copied variables.
3587	/// \param DestAddr Destination address.
3588	/// \param SrcAddr Source address.
3589	/// \param DestVD Destination variable used in \a CopyExpr (for arrays, has
3590	/// type of the base array element).
3591	/// \param SrcVD Source variable used in \a CopyExpr (for arrays, has type of
3592	/// the base array element).
3593	/// \param Copy Actual copygin expression for copying data from \a SrcVD to \a
3594	/// DestVD.
3595	void EmitOMPCopy(QualType OriginalType,
3596	Address DestAddr, Address SrcAddr,
3597	const VarDecl *DestVD, const VarDecl *SrcVD,
3598	const Expr *Copy);
3599	/// Emit atomic update code for constructs: \a X = \a X \a BO \a E or
3600	/// \a X = \a E \a BO \a E.
3601	///
3602	/// \param X Value to be updated.
3603	/// \param E Update value.
3604	/// \param BO Binary operation for update operation.
3605	/// \param IsXLHSInRHSPart true if \a X is LHS in RHS part of the update
3606	/// expression, false otherwise.
3607	/// \param AO Atomic ordering of the generated atomic instructions.
3608	/// \param CommonGen Code generator for complex expressions that cannot be
3609	/// expressed through atomicrmw instruction.
3610	/// \returns <true, OldAtomicValue> if simple 'atomicrmw' instruction was
3611	/// generated, <false, RValue::get(nullptr)> otherwise.
3612	std::pair<bool, RValue> EmitOMPAtomicSimpleUpdateExpr(
3613	LValue X, RValue E, BinaryOperatorKind BO, bool IsXLHSInRHSPart,
3614	llvm::AtomicOrdering AO, SourceLocation Loc,
3615	const llvm::function_ref<RValue(RValue)> CommonGen);
3616	bool EmitOMPFirstprivateClause(const OMPExecutableDirective &D,
3617	OMPPrivateScope &PrivateScope);
3618	void EmitOMPPrivateClause(const OMPExecutableDirective &D,
3619	OMPPrivateScope &PrivateScope);
3620	void EmitOMPUseDevicePtrClause(
3621	const OMPUseDevicePtrClause &C, OMPPrivateScope &PrivateScope,
3622	const llvm::DenseMap<const ValueDecl *, llvm::Value *>
3623	CaptureDeviceAddrMap);
3624	void EmitOMPUseDeviceAddrClause(
3625	const OMPUseDeviceAddrClause &C, OMPPrivateScope &PrivateScope,
3626	const llvm::DenseMap<const ValueDecl *, llvm::Value *>
3627	CaptureDeviceAddrMap);
3628	/// Emit code for copyin clause in \a D directive. The next code is
3629	/// generated at the start of outlined functions for directives:
3630	/// \code
3631	/// threadprivate_var1 = master_threadprivate_var1;
3632	/// operator=(threadprivate_var2, master_threadprivate_var2);
3633	/// ...
3634	/// __kmpc_barrier(&loc, global_tid);
3635	/// \endcode
3636	///
3637	/// \param D OpenMP directive possibly with 'copyin' clause(s).
3638	/// \returns true if at least one copyin variable is found, false otherwise.
3639	bool EmitOMPCopyinClause(const OMPExecutableDirective &D);
3640	/// Emit initial code for lastprivate variables. If some variable is
3641	/// not also firstprivate, then the default initialization is used. Otherwise
3642	/// initialization of this variable is performed by EmitOMPFirstprivateClause
3643	/// method.
3644	///
3645	/// \param D Directive that may have 'lastprivate' directives.
3646	/// \param PrivateScope Private scope for capturing lastprivate variables for
3647	/// proper codegen in internal captured statement.
3648	///
3649	/// \returns true if there is at least one lastprivate variable, false
3650	/// otherwise.
3651	bool EmitOMPLastprivateClauseInit(const OMPExecutableDirective &D,
3652	OMPPrivateScope &PrivateScope);
3653	/// Emit final copying of lastprivate values to original variables at
3654	/// the end of the worksharing or simd directive.
3655	///
3656	/// \param D Directive that has at least one 'lastprivate' directives.
3657	/// \param IsLastIterCond Boolean condition that must be set to 'i1 true' if
3658	/// it is the last iteration of the loop code in associated directive, or to
3659	/// 'i1 false' otherwise. If this item is nullptr, no final check is required.
3660	void EmitOMPLastprivateClauseFinal(const OMPExecutableDirective &D,
3661	bool NoFinals,
3662	llvm::Value *IsLastIterCond = nullptr);
3663	/// Emit initial code for linear clauses.
3664	void EmitOMPLinearClause(const OMPLoopDirective &D,
3665	CodeGenFunction::OMPPrivateScope &PrivateScope);
3666	/// Emit final code for linear clauses.
3667	/// \param CondGen Optional conditional code for final part of codegen for
3668	/// linear clause.
3669	void EmitOMPLinearClauseFinal(
3670	const OMPLoopDirective &D,
3671	const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen);
3672	/// Emit initial code for reduction variables. Creates reduction copies
3673	/// and initializes them with the values according to OpenMP standard.
3674	///
3675	/// \param D Directive (possibly) with the 'reduction' clause.
3676	/// \param PrivateScope Private scope for capturing reduction variables for
3677	/// proper codegen in internal captured statement.
3678	///
3679	void EmitOMPReductionClauseInit(const OMPExecutableDirective &D,
3680	OMPPrivateScope &PrivateScope,
3681	bool ForInscan = false);
3682	/// Emit final update of reduction values to original variables at
3683	/// the end of the directive.
3684	///
3685	/// \param D Directive that has at least one 'reduction' directives.
3686	/// \param ReductionKind The kind of reduction to perform.
3687	void EmitOMPReductionClauseFinal(const OMPExecutableDirective &D,
3688	const OpenMPDirectiveKind ReductionKind);
3689	/// Emit initial code for linear variables. Creates private copies
3690	/// and initializes them with the values according to OpenMP standard.
3691	///
3692	/// \param D Directive (possibly) with the 'linear' clause.
3693	/// \return true if at least one linear variable is found that should be
3694	/// initialized with the value of the original variable, false otherwise.
3695	bool EmitOMPLinearClauseInit(const OMPLoopDirective &D);
3696
3697	typedef const llvm::function_ref<void(CodeGenFunction & /*CGF*/,
3698	llvm::Function * /*OutlinedFn*/,
3699	const OMPTaskDataTy & /*Data*/)>
3700	TaskGenTy;
3701	void EmitOMPTaskBasedDirective(const OMPExecutableDirective &S,
3702	const OpenMPDirectiveKind CapturedRegion,
3703	const RegionCodeGenTy &BodyGen,
3704	const TaskGenTy &TaskGen, OMPTaskDataTy &Data);
3705	struct OMPTargetDataInfo {
3706	Address BasePointersArray = Address::invalid();
3707	Address PointersArray = Address::invalid();
3708	Address SizesArray = Address::invalid();
3709	Address MappersArray = Address::invalid();
3710	unsigned NumberOfTargetItems = 0;
3711	explicit OMPTargetDataInfo() = default;
3712	OMPTargetDataInfo(Address BasePointersArray, Address PointersArray,
3713	Address SizesArray, Address MappersArray,
3714	unsigned NumberOfTargetItems)
3715	: BasePointersArray(BasePointersArray), PointersArray(PointersArray),
3716	SizesArray(SizesArray), MappersArray(MappersArray),
3717	NumberOfTargetItems(NumberOfTargetItems) {}
3718	};
3719	void EmitOMPTargetTaskBasedDirective(const OMPExecutableDirective &S,
3720	const RegionCodeGenTy &BodyGen,
3721	OMPTargetDataInfo &InputInfo);
3722	void processInReduction(const OMPExecutableDirective &S,
3723	OMPTaskDataTy &Data,
3724	CodeGenFunction &CGF,
3725	const CapturedStmt *CS,
3726	OMPPrivateScope &Scope);
3727	void EmitOMPMetaDirective(const OMPMetaDirective &S);
3728	void EmitOMPParallelDirective(const OMPParallelDirective &S);
3729	void EmitOMPSimdDirective(const OMPSimdDirective &S);
3730	void EmitOMPTileDirective(const OMPTileDirective &S);
3731	void EmitOMPUnrollDirective(const OMPUnrollDirective &S);
3732	void EmitOMPForDirective(const OMPForDirective &S);
3733	void EmitOMPForSimdDirective(const OMPForSimdDirective &S);
3734	void EmitOMPSectionsDirective(const OMPSectionsDirective &S);
3735	void EmitOMPSectionDirective(const OMPSectionDirective &S);
3736	void EmitOMPSingleDirective(const OMPSingleDirective &S);
3737	void EmitOMPMasterDirective(const OMPMasterDirective &S);
3738	void EmitOMPMaskedDirective(const OMPMaskedDirective &S);
3739	void EmitOMPCriticalDirective(const OMPCriticalDirective &S);
3740	void EmitOMPParallelForDirective(const OMPParallelForDirective &S);
3741	void EmitOMPParallelForSimdDirective(const OMPParallelForSimdDirective &S);
3742	void EmitOMPParallelSectionsDirective(const OMPParallelSectionsDirective &S);
3743	void EmitOMPParallelMasterDirective(const OMPParallelMasterDirective &S);
3744	void EmitOMPTaskDirective(const OMPTaskDirective &S);
3745	void EmitOMPTaskyieldDirective(const OMPTaskyieldDirective &S);
3746	void EmitOMPErrorDirective(const OMPErrorDirective &S);
3747	void EmitOMPBarrierDirective(const OMPBarrierDirective &S);
3748	void EmitOMPTaskwaitDirective(const OMPTaskwaitDirective &S);
3749	void EmitOMPTaskgroupDirective(const OMPTaskgroupDirective &S);
3750	void EmitOMPFlushDirective(const OMPFlushDirective &S);
3751	void EmitOMPDepobjDirective(const OMPDepobjDirective &S);
3752	void EmitOMPScanDirective(const OMPScanDirective &S);
3753	void EmitOMPOrderedDirective(const OMPOrderedDirective &S);
3754	void EmitOMPAtomicDirective(const OMPAtomicDirective &S);
3755	void EmitOMPTargetDirective(const OMPTargetDirective &S);
3756	void EmitOMPTargetDataDirective(const OMPTargetDataDirective &S);
3757	void EmitOMPTargetEnterDataDirective(const OMPTargetEnterDataDirective &S);
3758	void EmitOMPTargetExitDataDirective(const OMPTargetExitDataDirective &S);
3759	void EmitOMPTargetUpdateDirective(const OMPTargetUpdateDirective &S);
3760	void EmitOMPTargetParallelDirective(const OMPTargetParallelDirective &S);
3761	void
3762	EmitOMPTargetParallelForDirective(const OMPTargetParallelForDirective &S);
3763	void EmitOMPTeamsDirective(const OMPTeamsDirective &S);
3764	void
3765	EmitOMPCancellationPointDirective(const OMPCancellationPointDirective &S);
3766	void EmitOMPCancelDirective(const OMPCancelDirective &S);
3767	void EmitOMPTaskLoopBasedDirective(const OMPLoopDirective &S);
3768	void EmitOMPTaskLoopDirective(const OMPTaskLoopDirective &S);
3769	void EmitOMPTaskLoopSimdDirective(const OMPTaskLoopSimdDirective &S);
3770	void EmitOMPMasterTaskLoopDirective(const OMPMasterTaskLoopDirective &S);
3771	void
3772	EmitOMPMasterTaskLoopSimdDirective(const OMPMasterTaskLoopSimdDirective &S);
3773	void EmitOMPParallelMasterTaskLoopDirective(
3774	const OMPParallelMasterTaskLoopDirective &S);
3775	void EmitOMPParallelMasterTaskLoopSimdDirective(
3776	const OMPParallelMasterTaskLoopSimdDirective &S);
3777	void EmitOMPDistributeDirective(const OMPDistributeDirective &S);
3778	void EmitOMPDistributeParallelForDirective(
3779	const OMPDistributeParallelForDirective &S);
3780	void EmitOMPDistributeParallelForSimdDirective(
3781	const OMPDistributeParallelForSimdDirective &S);
3782	void EmitOMPDistributeSimdDirective(const OMPDistributeSimdDirective &S);
3783	void EmitOMPTargetParallelForSimdDirective(
3784	const OMPTargetParallelForSimdDirective &S);
3785	void EmitOMPTargetSimdDirective(const OMPTargetSimdDirective &S);
3786	void EmitOMPTeamsDistributeDirective(const OMPTeamsDistributeDirective &S);
3787	void
3788	EmitOMPTeamsDistributeSimdDirective(const OMPTeamsDistributeSimdDirective &S);
3789	void EmitOMPTeamsDistributeParallelForSimdDirective(
3790	const OMPTeamsDistributeParallelForSimdDirective &S);
3791	void EmitOMPTeamsDistributeParallelForDirective(
3792	const OMPTeamsDistributeParallelForDirective &S);
3793	void EmitOMPTargetTeamsDirective(const OMPTargetTeamsDirective &S);
3794	void EmitOMPTargetTeamsDistributeDirective(
3795	const OMPTargetTeamsDistributeDirective &S);
3796	void EmitOMPTargetTeamsDistributeParallelForDirective(
3797	const OMPTargetTeamsDistributeParallelForDirective &S);
3798	void EmitOMPTargetTeamsDistributeParallelForSimdDirective(
3799	const OMPTargetTeamsDistributeParallelForSimdDirective &S);
3800	void EmitOMPTargetTeamsDistributeSimdDirective(
3801	const OMPTargetTeamsDistributeSimdDirective &S);
3802	void EmitOMPGenericLoopDirective(const OMPGenericLoopDirective &S);
3803	void EmitOMPParallelGenericLoopDirective(const OMPLoopDirective &S);
3804	void EmitOMPTargetParallelGenericLoopDirective(
3805	const OMPTargetParallelGenericLoopDirective &S);
3806	void EmitOMPTargetTeamsGenericLoopDirective(
3807	const OMPTargetTeamsGenericLoopDirective &S);
3808	void EmitOMPTeamsGenericLoopDirective(const OMPTeamsGenericLoopDirective &S);
3809	void EmitOMPInteropDirective(const OMPInteropDirective &S);
3810	void EmitOMPParallelMaskedDirective(const OMPParallelMaskedDirective &S);
3811
3812	/// Emit device code for the target directive.
3813	static void EmitOMPTargetDeviceFunction(CodeGenModule &CGM,
3814	StringRef ParentName,
3815	const OMPTargetDirective &S);
3816	static void
3817	EmitOMPTargetParallelDeviceFunction(CodeGenModule &CGM, StringRef ParentName,
3818	const OMPTargetParallelDirective &S);
3819	/// Emit device code for the target parallel for directive.
3820	static void EmitOMPTargetParallelForDeviceFunction(
3821	CodeGenModule &CGM, StringRef ParentName,
3822	const OMPTargetParallelForDirective &S);
3823	/// Emit device code for the target parallel for simd directive.
3824	static void EmitOMPTargetParallelForSimdDeviceFunction(
3825	CodeGenModule &CGM, StringRef ParentName,
3826	const OMPTargetParallelForSimdDirective &S);
3827	/// Emit device code for the target teams directive.
3828	static void
3829	EmitOMPTargetTeamsDeviceFunction(CodeGenModule &CGM, StringRef ParentName,
3830	const OMPTargetTeamsDirective &S);
3831	/// Emit device code for the target teams distribute directive.
3832	static void EmitOMPTargetTeamsDistributeDeviceFunction(
3833	CodeGenModule &CGM, StringRef ParentName,
3834	const OMPTargetTeamsDistributeDirective &S);
3835	/// Emit device code for the target teams distribute simd directive.
3836	static void EmitOMPTargetTeamsDistributeSimdDeviceFunction(
3837	CodeGenModule &CGM, StringRef ParentName,
3838	const OMPTargetTeamsDistributeSimdDirective &S);
3839	/// Emit device code for the target simd directive.
3840	static void EmitOMPTargetSimdDeviceFunction(CodeGenModule &CGM,
3841	StringRef ParentName,
3842	const OMPTargetSimdDirective &S);
3843	/// Emit device code for the target teams distribute parallel for simd
3844	/// directive.
3845	static void EmitOMPTargetTeamsDistributeParallelForSimdDeviceFunction(
3846	CodeGenModule &CGM, StringRef ParentName,
3847	const OMPTargetTeamsDistributeParallelForSimdDirective &S);
3848
3849	/// Emit device code for the target teams loop directive.
3850	static void EmitOMPTargetTeamsGenericLoopDeviceFunction(
3851	CodeGenModule &CGM, StringRef ParentName,
3852	const OMPTargetTeamsGenericLoopDirective &S);
3853
3854	/// Emit device code for the target parallel loop directive.
3855	static void EmitOMPTargetParallelGenericLoopDeviceFunction(
3856	CodeGenModule &CGM, StringRef ParentName,
3857	const OMPTargetParallelGenericLoopDirective &S);
3858
3859	static void EmitOMPTargetTeamsDistributeParallelForDeviceFunction(
3860	CodeGenModule &CGM, StringRef ParentName,
3861	const OMPTargetTeamsDistributeParallelForDirective &S);
3862
3863	/// Emit the Stmt \p S and return its topmost canonical loop, if any.
3864	/// TODO: The \p Depth paramter is not yet implemented and must be 1. In the
3865	/// future it is meant to be the number of loops expected in the loop nests
3866	/// (usually specified by the "collapse" clause) that are collapsed to a
3867	/// single loop by this function.
3868	llvm::CanonicalLoopInfo *EmitOMPCollapsedCanonicalLoopNest(const Stmt *S,
3869	int Depth);
3870
3871	/// Emit an OMPCanonicalLoop using the OpenMPIRBuilder.
3872	void EmitOMPCanonicalLoop(const OMPCanonicalLoop *S);
3873
3874	/// Emit inner loop of the worksharing/simd construct.
3875	///
3876	/// \param S Directive, for which the inner loop must be emitted.
3877	/// \param RequiresCleanup true, if directive has some associated private
3878	/// variables.
3879	/// \param LoopCond Bollean condition for loop continuation.
3880	/// \param IncExpr Increment expression for loop control variable.
3881	/// \param BodyGen Generator for the inner body of the inner loop.
3882	/// \param PostIncGen Genrator for post-increment code (required for ordered
3883	/// loop directvies).
3884	void EmitOMPInnerLoop(
3885	const OMPExecutableDirective &S, bool RequiresCleanup,
3886	const Expr *LoopCond, const Expr *IncExpr,
3887	const llvm::function_ref<void(CodeGenFunction &)> BodyGen,
3888	const llvm::function_ref<void(CodeGenFunction &)> PostIncGen);
3889
3890	JumpDest getOMPCancelDestination(OpenMPDirectiveKind Kind);
3891	/// Emit initial code for loop counters of loop-based directives.
3892	void EmitOMPPrivateLoopCounters(const OMPLoopDirective &S,
3893	OMPPrivateScope &LoopScope);
3894
3895	/// Helper for the OpenMP loop directives.
3896	void EmitOMPLoopBody(const OMPLoopDirective &D, JumpDest LoopExit);
3897
3898	/// Emit code for the worksharing loop-based directive.
3899	/// \return true, if this construct has any lastprivate clause, false -
3900	/// otherwise.
3901	bool EmitOMPWorksharingLoop(const OMPLoopDirective &S, Expr *EUB,
3902	const CodeGenLoopBoundsTy &CodeGenLoopBounds,
3903	const CodeGenDispatchBoundsTy &CGDispatchBounds);
3904
3905	/// Emit code for the distribute loop-based directive.
3906	void EmitOMPDistributeLoop(const OMPLoopDirective &S,
3907	const CodeGenLoopTy &CodeGenLoop, Expr *IncExpr);
3908
3909	/// Helpers for the OpenMP loop directives.
3910	void EmitOMPSimdInit(const OMPLoopDirective &D);
3911	void EmitOMPSimdFinal(
3912	const OMPLoopDirective &D,
3913	const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen);
3914
3915	/// Emits the lvalue for the expression with possibly captured variable.
3916	LValue EmitOMPSharedLValue(const Expr *E);
3917
3918	private:
3919	/// Helpers for blocks.
3920	llvm::Value *EmitBlockLiteral(const CGBlockInfo &Info);
3921
3922	/// struct with the values to be passed to the OpenMP loop-related functions
3923	struct OMPLoopArguments {
3924	/// loop lower bound
3925	Address LB = Address::invalid();
3926	/// loop upper bound
3927	Address UB = Address::invalid();
3928	/// loop stride
3929	Address ST = Address::invalid();
3930	/// isLastIteration argument for runtime functions
3931	Address IL = Address::invalid();
3932	/// Chunk value generated by sema
3933	llvm::Value *Chunk = nullptr;
3934	/// EnsureUpperBound
3935	Expr *EUB = nullptr;
3936	/// IncrementExpression
3937	Expr *IncExpr = nullptr;
3938	/// Loop initialization
3939	Expr *Init = nullptr;
3940	/// Loop exit condition
3941	Expr *Cond = nullptr;
3942	/// Update of LB after a whole chunk has been executed
3943	Expr *NextLB = nullptr;
3944	/// Update of UB after a whole chunk has been executed
3945	Expr *NextUB = nullptr;
3946	/// Distinguish between the for distribute and sections
3947	OpenMPDirectiveKind DKind = llvm::omp::OMPD_unknown;
3948	OMPLoopArguments() = default;
3949	OMPLoopArguments(Address LB, Address UB, Address ST, Address IL,
3950	llvm::Value *Chunk = nullptr, Expr *EUB = nullptr,
3951	Expr *IncExpr = nullptr, Expr *Init = nullptr,
3952	Expr *Cond = nullptr, Expr *NextLB = nullptr,
3953	Expr *NextUB = nullptr)
3954	: LB(LB), UB(UB), ST(ST), IL(IL), Chunk(Chunk), EUB(EUB),
3955	IncExpr(IncExpr), Init(Init), Cond(Cond), NextLB(NextLB),
3956	NextUB(NextUB) {}
3957	};
3958	void EmitOMPOuterLoop(bool DynamicOrOrdered, bool IsMonotonic,
3959	const OMPLoopDirective &S, OMPPrivateScope &LoopScope,
3960	const OMPLoopArguments &LoopArgs,
3961	const CodeGenLoopTy &CodeGenLoop,
3962	const CodeGenOrderedTy &CodeGenOrdered);
3963	void EmitOMPForOuterLoop(const OpenMPScheduleTy &ScheduleKind,
3964	bool IsMonotonic, const OMPLoopDirective &S,
3965	OMPPrivateScope &LoopScope, bool Ordered,
3966	const OMPLoopArguments &LoopArgs,
3967	const CodeGenDispatchBoundsTy &CGDispatchBounds);
3968	void EmitOMPDistributeOuterLoop(OpenMPDistScheduleClauseKind ScheduleKind,
3969	const OMPLoopDirective &S,
3970	OMPPrivateScope &LoopScope,
3971	const OMPLoopArguments &LoopArgs,
3972	const CodeGenLoopTy &CodeGenLoopContent);
3973	/// Emit code for sections directive.
3974	void EmitSections(const OMPExecutableDirective &S);
3975
3976	public:
3977	//===--------------------------------------------------------------------===//
3978	// OpenACC Emission
3979	//===--------------------------------------------------------------------===//
3980	void EmitOpenACCComputeConstruct(const OpenACCComputeConstruct &S) {
3981	// TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
3982	// simply emitting its structured block, but in the future we will implement
3983	// some sort of IR.
3984	EmitStmt(S: S.getStructuredBlock());
3985	}
3986
3987	//===--------------------------------------------------------------------===//
3988	// LValue Expression Emission
3989	//===--------------------------------------------------------------------===//
3990
3991	/// Create a check that a scalar RValue is non-null.
3992	llvm::Value *EmitNonNullRValueCheck(RValue RV, QualType T);
3993
3994	/// GetUndefRValue - Get an appropriate 'undef' rvalue for the given type.
3995	RValue GetUndefRValue(QualType Ty);
3996
3997	/// EmitUnsupportedRValue - Emit a dummy r-value using the type of E
3998	/// and issue an ErrorUnsupported style diagnostic (using the
3999	/// provided Name).
4000	RValue EmitUnsupportedRValue(const Expr *E,
4001	const char *Name);
4002
4003	/// EmitUnsupportedLValue - Emit a dummy l-value using the type of E and issue
4004	/// an ErrorUnsupported style diagnostic (using the provided Name).
4005	LValue EmitUnsupportedLValue(const Expr *E,
4006	const char *Name);
4007
4008	/// EmitLValue - Emit code to compute a designator that specifies the location
4009	/// of the expression.
4010	///
4011	/// This can return one of two things: a simple address or a bitfield
4012	/// reference. In either case, the LLVM Value* in the LValue structure is
4013	/// guaranteed to be an LLVM pointer type.
4014	///
4015	/// If this returns a bitfield reference, nothing about the pointee type of
4016	/// the LLVM value is known: For example, it may not be a pointer to an
4017	/// integer.
4018	///
4019	/// If this returns a normal address, and if the lvalue's C type is fixed
4020	/// size, this method guarantees that the returned pointer type will point to
4021	/// an LLVM type of the same size of the lvalue's type. If the lvalue has a
4022	/// variable length type, this is not possible.
4023	///
4024	LValue EmitLValue(const Expr *E,
4025	KnownNonNull_t IsKnownNonNull = NotKnownNonNull);
4026
4027	private:
4028	LValue EmitLValueHelper(const Expr *E, KnownNonNull_t IsKnownNonNull);
4029
4030	public:
4031	/// Same as EmitLValue but additionally we generate checking code to
4032	/// guard against undefined behavior. This is only suitable when we know
4033	/// that the address will be used to access the object.
4034	LValue EmitCheckedLValue(const Expr *E, TypeCheckKind TCK);
4035
4036	RValue convertTempToRValue(Address addr, QualType type,
4037	SourceLocation Loc);
4038
4039	void EmitAtomicInit(Expr *E, LValue lvalue);
4040
4041	bool LValueIsSuitableForInlineAtomic(LValue Src);
4042
4043	RValue EmitAtomicLoad(LValue LV, SourceLocation SL,
4044	AggValueSlot Slot = AggValueSlot::ignored());
4045
4046	RValue EmitAtomicLoad(LValue lvalue, SourceLocation loc,
4047	llvm::AtomicOrdering AO, bool IsVolatile = false,
4048	AggValueSlot slot = AggValueSlot::ignored());
4049
4050	void EmitAtomicStore(RValue rvalue, LValue lvalue, bool isInit);
4051
4052	void EmitAtomicStore(RValue rvalue, LValue lvalue, llvm::AtomicOrdering AO,
4053	bool IsVolatile, bool isInit);
4054
4055	std::pair<RValue, llvm::Value *> EmitAtomicCompareExchange(
4056	LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc,
4057	llvm::AtomicOrdering Success =
4058	llvm::AtomicOrdering::SequentiallyConsistent,
4059	llvm::AtomicOrdering Failure =
4060	llvm::AtomicOrdering::SequentiallyConsistent,
4061	bool IsWeak = false, AggValueSlot Slot = AggValueSlot::ignored());
4062
4063	void EmitAtomicUpdate(LValue LVal, llvm::AtomicOrdering AO,
4064	const llvm::function_ref<RValue(RValue)> &UpdateOp,
4065	bool IsVolatile);
4066
4067	/// EmitToMemory - Change a scalar value from its value
4068	/// representation to its in-memory representation.
4069	llvm::Value *EmitToMemory(llvm::Value *Value, QualType Ty);
4070
4071	/// EmitFromMemory - Change a scalar value from its memory
4072	/// representation to its value representation.
4073	llvm::Value *EmitFromMemory(llvm::Value *Value, QualType Ty);
4074
4075	/// Check if the scalar \p Value is within the valid range for the given
4076	/// type \p Ty.
4077	///
4078	/// Returns true if a check is needed (even if the range is unknown).
4079	bool EmitScalarRangeCheck(llvm::Value *Value, QualType Ty,
4080	SourceLocation Loc);
4081
4082	/// EmitLoadOfScalar - Load a scalar value from an address, taking
4083	/// care to appropriately convert from the memory representation to
4084	/// the LLVM value representation.
4085	llvm::Value *EmitLoadOfScalar(Address Addr, bool Volatile, QualType Ty,
4086	SourceLocation Loc,
4087	AlignmentSource Source = AlignmentSource::Type,
4088	bool isNontemporal = false) {
4089	return EmitLoadOfScalar(Addr, Volatile, Ty, Loc, BaseInfo: LValueBaseInfo(Source),
4090	TBAAInfo: CGM.getTBAAAccessInfo(AccessType: Ty), isNontemporal);
4091	}
4092
4093	llvm::Value *EmitLoadOfScalar(Address Addr, bool Volatile, QualType Ty,
4094	SourceLocation Loc, LValueBaseInfo BaseInfo,
4095	TBAAAccessInfo TBAAInfo,
4096	bool isNontemporal = false);
4097
4098	/// EmitLoadOfScalar - Load a scalar value from an address, taking
4099	/// care to appropriately convert from the memory representation to
4100	/// the LLVM value representation. The l-value must be a simple
4101	/// l-value.
4102	llvm::Value *EmitLoadOfScalar(LValue lvalue, SourceLocation Loc);
4103
4104	/// EmitStoreOfScalar - Store a scalar value to an address, taking
4105	/// care to appropriately convert from the memory representation to
4106	/// the LLVM value representation.
4107	void EmitStoreOfScalar(llvm::Value *Value, Address Addr,
4108	bool Volatile, QualType Ty,
4109	AlignmentSource Source = AlignmentSource::Type,
4110	bool isInit = false, bool isNontemporal = false) {
4111	EmitStoreOfScalar(Value, Addr, Volatile, Ty, BaseInfo: LValueBaseInfo(Source),
4112	TBAAInfo: CGM.getTBAAAccessInfo(AccessType: Ty), isInit, isNontemporal);
4113	}
4114
4115	void EmitStoreOfScalar(llvm::Value *Value, Address Addr,
4116	bool Volatile, QualType Ty,
4117	LValueBaseInfo BaseInfo, TBAAAccessInfo TBAAInfo,
4118	bool isInit = false, bool isNontemporal = false);
4119
4120	/// EmitStoreOfScalar - Store a scalar value to an address, taking
4121	/// care to appropriately convert from the memory representation to
4122	/// the LLVM value representation. The l-value must be a simple
4123	/// l-value. The isInit flag indicates whether this is an initialization.
4124	/// If so, atomic qualifiers are ignored and the store is always non-atomic.
4125	void EmitStoreOfScalar(llvm::Value *value, LValue lvalue, bool isInit=false);
4126
4127	/// EmitLoadOfLValue - Given an expression that represents a value lvalue,
4128	/// this method emits the address of the lvalue, then loads the result as an
4129	/// rvalue, returning the rvalue.
4130	RValue EmitLoadOfLValue(LValue V, SourceLocation Loc);
4131	RValue EmitLoadOfExtVectorElementLValue(LValue V);
4132	RValue EmitLoadOfBitfieldLValue(LValue LV, SourceLocation Loc);
4133	RValue EmitLoadOfGlobalRegLValue(LValue LV);
4134
4135	/// EmitStoreThroughLValue - Store the specified rvalue into the specified
4136	/// lvalue, where both are guaranteed to the have the same type, and that type
4137	/// is 'Ty'.
4138	void EmitStoreThroughLValue(RValue Src, LValue Dst, bool isInit = false);
4139	void EmitStoreThroughExtVectorComponentLValue(RValue Src, LValue Dst);
4140	void EmitStoreThroughGlobalRegLValue(RValue Src, LValue Dst);
4141
4142	/// EmitStoreThroughBitfieldLValue - Store Src into Dst with same constraints
4143	/// as EmitStoreThroughLValue.
4144	///
4145	/// \param Result [out] - If non-null, this will be set to a Value* for the
4146	/// bit-field contents after the store, appropriate for use as the result of
4147	/// an assignment to the bit-field.
4148	void EmitStoreThroughBitfieldLValue(RValue Src, LValue Dst,
4149	llvm::Value **Result=nullptr);
4150
4151	/// Emit an l-value for an assignment (simple or compound) of complex type.
4152	LValue EmitComplexAssignmentLValue(const BinaryOperator *E);
4153	LValue EmitComplexCompoundAssignmentLValue(const CompoundAssignOperator *E);
4154	LValue EmitScalarCompoundAssignWithComplex(const CompoundAssignOperator *E,
4155	llvm::Value *&Result);
4156
4157	// Note: only available for agg return types
4158	LValue EmitBinaryOperatorLValue(const BinaryOperator *E);
4159	LValue EmitCompoundAssignmentLValue(const CompoundAssignOperator *E);
4160	// Note: only available for agg return types
4161	LValue EmitCallExprLValue(const CallExpr *E);
4162	// Note: only available for agg return types
4163	LValue EmitVAArgExprLValue(const VAArgExpr *E);
4164	LValue EmitDeclRefLValue(const DeclRefExpr *E);
4165	LValue EmitStringLiteralLValue(const StringLiteral *E);
4166	LValue EmitObjCEncodeExprLValue(const ObjCEncodeExpr *E);
4167	LValue EmitPredefinedLValue(const PredefinedExpr *E);
4168	LValue EmitUnaryOpLValue(const UnaryOperator *E);
4169	LValue EmitArraySubscriptExpr(const ArraySubscriptExpr *E,
4170	bool Accessed = false);
4171	LValue EmitMatrixSubscriptExpr(const MatrixSubscriptExpr *E);
4172	LValue EmitOMPArraySectionExpr(const OMPArraySectionExpr *E,
4173	bool IsLowerBound = true);
4174	LValue EmitExtVectorElementExpr(const ExtVectorElementExpr *E);
4175	LValue EmitMemberExpr(const MemberExpr *E);
4176	LValue EmitObjCIsaExpr(const ObjCIsaExpr *E);
4177	LValue EmitCompoundLiteralLValue(const CompoundLiteralExpr *E);
4178	LValue EmitInitListLValue(const InitListExpr *E);
4179	void EmitIgnoredConditionalOperator(const AbstractConditionalOperator *E);
4180	LValue EmitConditionalOperatorLValue(const AbstractConditionalOperator *E);
4181	LValue EmitCastLValue(const CastExpr *E);
4182	LValue EmitMaterializeTemporaryExpr(const MaterializeTemporaryExpr *E);
4183	LValue EmitOpaqueValueLValue(const OpaqueValueExpr *e);
4184
4185	Address EmitExtVectorElementLValue(LValue V);
4186
4187	RValue EmitRValueForField(LValue LV, const FieldDecl *FD, SourceLocation Loc);
4188
4189	Address EmitArrayToPointerDecay(const Expr *Array,
4190	LValueBaseInfo *BaseInfo = nullptr,
4191	TBAAAccessInfo *TBAAInfo = nullptr);
4192
4193	class ConstantEmission {
4194	llvm::PointerIntPair<llvm::Constant*, 1, bool> ValueAndIsReference;
4195	ConstantEmission(llvm::Constant *C, bool isReference)
4196	: ValueAndIsReference(C, isReference) {}
4197	public:
4198	ConstantEmission() {}
4199	static ConstantEmission forReference(llvm::Constant *C) {
4200	return ConstantEmission(C, true);
4201	}
4202	static ConstantEmission forValue(llvm::Constant *C) {
4203	return ConstantEmission(C, false);
4204	}
4205
4206	explicit operator bool() const {
4207	return ValueAndIsReference.getOpaqueValue() != nullptr;
4208	}
4209
4210	bool isReference() const { return ValueAndIsReference.getInt(); }
4211	LValue getReferenceLValue(CodeGenFunction &CGF, Expr *refExpr) const {
4212	assert(isReference());
4213	return CGF.MakeNaturalAlignAddrLValue(V: ValueAndIsReference.getPointer(),
4214	T: refExpr->getType());
4215	}
4216
4217	llvm::Constant *getValue() const {
4218	assert(!isReference());
4219	return ValueAndIsReference.getPointer();
4220	}
4221	};
4222
4223	ConstantEmission tryEmitAsConstant(DeclRefExpr *refExpr);
4224	ConstantEmission tryEmitAsConstant(const MemberExpr *ME);
4225	llvm::Value *emitScalarConstant(const ConstantEmission &Constant, Expr *E);
4226
4227	RValue EmitPseudoObjectRValue(const PseudoObjectExpr *e,
4228	AggValueSlot slot = AggValueSlot::ignored());
4229	LValue EmitPseudoObjectLValue(const PseudoObjectExpr *e);
4230
4231	llvm::Value *EmitIvarOffset(const ObjCInterfaceDecl *Interface,
4232	const ObjCIvarDecl *Ivar);
4233	llvm::Value *EmitIvarOffsetAsPointerDiff(const ObjCInterfaceDecl *Interface,
4234	const ObjCIvarDecl *Ivar);
4235	LValue EmitLValueForField(LValue Base, const FieldDecl* Field);
4236	LValue EmitLValueForLambdaField(const FieldDecl *Field);
4237	LValue EmitLValueForLambdaField(const FieldDecl *Field,
4238	llvm::Value *ThisValue);
4239
4240	/// EmitLValueForFieldInitialization - Like EmitLValueForField, except that
4241	/// if the Field is a reference, this will return the address of the reference
4242	/// and not the address of the value stored in the reference.
4243	LValue EmitLValueForFieldInitialization(LValue Base,
4244	const FieldDecl* Field);
4245
4246	LValue EmitLValueForIvar(QualType ObjectTy,
4247	llvm::Value* Base, const ObjCIvarDecl *Ivar,
4248	unsigned CVRQualifiers);
4249
4250	LValue EmitCXXConstructLValue(const CXXConstructExpr *E);
4251	LValue EmitCXXBindTemporaryLValue(const CXXBindTemporaryExpr *E);
4252	LValue EmitCXXTypeidLValue(const CXXTypeidExpr *E);
4253	LValue EmitCXXUuidofLValue(const CXXUuidofExpr *E);
4254
4255	LValue EmitObjCMessageExprLValue(const ObjCMessageExpr *E);
4256	LValue EmitObjCIvarRefLValue(const ObjCIvarRefExpr *E);
4257	LValue EmitStmtExprLValue(const StmtExpr *E);
4258	LValue EmitPointerToDataMemberBinaryExpr(const BinaryOperator *E);
4259	LValue EmitObjCSelectorLValue(const ObjCSelectorExpr *E);
4260	void EmitDeclRefExprDbgValue(const DeclRefExpr *E, const APValue &Init);
4261
4262	//===--------------------------------------------------------------------===//
4263	// Scalar Expression Emission
4264	//===--------------------------------------------------------------------===//
4265
4266	/// EmitCall - Generate a call of the given function, expecting the given
4267	/// result type, and using the given argument list which specifies both the
4268	/// LLVM arguments and the types they were derived from.
4269	RValue EmitCall(const CGFunctionInfo &CallInfo, const CGCallee &Callee,
4270	ReturnValueSlot ReturnValue, const CallArgList &Args,
4271	llvm::CallBase **callOrInvoke, bool IsMustTail,
4272	SourceLocation Loc);
4273	RValue EmitCall(const CGFunctionInfo &CallInfo, const CGCallee &Callee,
4274	ReturnValueSlot ReturnValue, const CallArgList &Args,
4275	llvm::CallBase **callOrInvoke = nullptr,
4276	bool IsMustTail = false) {
4277	return EmitCall(CallInfo, Callee, ReturnValue, Args, callOrInvoke,
4278	IsMustTail, Loc: SourceLocation());
4279	}
4280	RValue EmitCall(QualType FnType, const CGCallee &Callee, const CallExpr *E,
4281	ReturnValueSlot ReturnValue, llvm::Value *Chain = nullptr);
4282	RValue EmitCallExpr(const CallExpr *E,
4283	ReturnValueSlot ReturnValue = ReturnValueSlot());
4284	RValue EmitSimpleCallExpr(const CallExpr *E, ReturnValueSlot ReturnValue);
4285	CGCallee EmitCallee(const Expr *E);
4286
4287	void checkTargetFeatures(const CallExpr *E, const FunctionDecl *TargetDecl);
4288	void checkTargetFeatures(SourceLocation Loc, const FunctionDecl *TargetDecl);
4289
4290	llvm::CallInst *EmitRuntimeCall(llvm::FunctionCallee callee,
4291	const Twine &name = "");
4292	llvm::CallInst *EmitRuntimeCall(llvm::FunctionCallee callee,
4293	ArrayRef<llvm::Value *> args,
4294	const Twine &name = "");
4295	llvm::CallInst *EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
4296	const Twine &name = "");
4297	llvm::CallInst *EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
4298	ArrayRef<Address> args,
4299	const Twine &name = "");
4300	llvm::CallInst *EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
4301	ArrayRef<llvm::Value *> args,
4302	const Twine &name = "");
4303
4304	SmallVector<llvm::OperandBundleDef, 1>
4305	getBundlesForFunclet(llvm::Value *Callee);
4306
4307	llvm::CallBase *EmitCallOrInvoke(llvm::FunctionCallee Callee,
4308	ArrayRef<llvm::Value *> Args,
4309	const Twine &Name = "");
4310	llvm::CallBase *EmitRuntimeCallOrInvoke(llvm::FunctionCallee callee,
4311	ArrayRef<llvm::Value *> args,
4312	const Twine &name = "");
4313	llvm::CallBase *EmitRuntimeCallOrInvoke(llvm::FunctionCallee callee,
4314	const Twine &name = "");
4315	void EmitNoreturnRuntimeCallOrInvoke(llvm::FunctionCallee callee,
4316	ArrayRef<llvm::Value *> args);
4317
4318	CGCallee BuildAppleKextVirtualCall(const CXXMethodDecl *MD,
4319	NestedNameSpecifier *Qual,
4320	llvm::Type *Ty);
4321
4322	CGCallee BuildAppleKextVirtualDestructorCall(const CXXDestructorDecl *DD,
4323	CXXDtorType Type,
4324	const CXXRecordDecl *RD);
4325
4326	llvm::Value *getAsNaturalPointerTo(Address Addr, QualType PointeeType) {
4327	return Addr.getBasePointer();
4328	}
4329
4330	bool isPointerKnownNonNull(const Expr *E);
4331
4332	// Return the copy constructor name with the prefix "__copy_constructor_"
4333	// removed.
4334	static std::string getNonTrivialCopyConstructorStr(QualType QT,
4335	CharUnits Alignment,
4336	bool IsVolatile,
4337	ASTContext &Ctx);
4338
4339	// Return the destructor name with the prefix "__destructor_" removed.
4340	static std::string getNonTrivialDestructorStr(QualType QT,
4341	CharUnits Alignment,
4342	bool IsVolatile,
4343	ASTContext &Ctx);
4344
4345	// These functions emit calls to the special functions of non-trivial C
4346	// structs.
4347	void defaultInitNonTrivialCStructVar(LValue Dst);
4348	void callCStructDefaultConstructor(LValue Dst);
4349	void callCStructDestructor(LValue Dst);
4350	void callCStructCopyConstructor(LValue Dst, LValue Src);
4351	void callCStructMoveConstructor(LValue Dst, LValue Src);
4352	void callCStructCopyAssignmentOperator(LValue Dst, LValue Src);
4353	void callCStructMoveAssignmentOperator(LValue Dst, LValue Src);
4354
4355	RValue
4356	EmitCXXMemberOrOperatorCall(const CXXMethodDecl *Method,
4357	const CGCallee &Callee,
4358	ReturnValueSlot ReturnValue, llvm::Value *This,
4359	llvm::Value *ImplicitParam,
4360	QualType ImplicitParamTy, const CallExpr *E,
4361	CallArgList *RtlArgs);
4362	RValue EmitCXXDestructorCall(GlobalDecl Dtor, const CGCallee &Callee,
4363	llvm::Value *This, QualType ThisTy,
4364	llvm::Value *ImplicitParam,
4365	QualType ImplicitParamTy, const CallExpr *E);
4366	RValue EmitCXXMemberCallExpr(const CXXMemberCallExpr *E,
4367	ReturnValueSlot ReturnValue);
4368	RValue EmitCXXMemberOrOperatorMemberCallExpr(const CallExpr *CE,
4369	const CXXMethodDecl *MD,
4370	ReturnValueSlot ReturnValue,
4371	bool HasQualifier,
4372	NestedNameSpecifier *Qualifier,
4373	bool IsArrow, const Expr *Base);
4374	// Compute the object pointer.
4375	Address EmitCXXMemberDataPointerAddress(const Expr *E, Address base,
4376	llvm::Value *memberPtr,
4377	const MemberPointerType *memberPtrType,
4378	LValueBaseInfo *BaseInfo = nullptr,
4379	TBAAAccessInfo *TBAAInfo = nullptr);
4380	RValue EmitCXXMemberPointerCallExpr(const CXXMemberCallExpr *E,
4381	ReturnValueSlot ReturnValue);
4382
4383	RValue EmitCXXOperatorMemberCallExpr(const CXXOperatorCallExpr *E,
4384	const CXXMethodDecl *MD,
4385	ReturnValueSlot ReturnValue);
4386	RValue EmitCXXPseudoDestructorExpr(const CXXPseudoDestructorExpr *E);
4387
4388	RValue EmitCUDAKernelCallExpr(const CUDAKernelCallExpr *E,
4389	ReturnValueSlot ReturnValue);
4390
4391	RValue EmitNVPTXDevicePrintfCallExpr(const CallExpr *E);
4392	RValue EmitAMDGPUDevicePrintfCallExpr(const CallExpr *E);
4393	RValue EmitOpenMPDevicePrintfCallExpr(const CallExpr *E);
4394
4395	RValue EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
4396	const CallExpr *E, ReturnValueSlot ReturnValue);
4397
4398	RValue emitRotate(const CallExpr *E, bool IsRotateRight);
4399
4400	/// Emit IR for __builtin_os_log_format.
4401	RValue emitBuiltinOSLogFormat(const CallExpr &E);
4402
4403	/// Emit IR for __builtin_is_aligned.
4404	RValue EmitBuiltinIsAligned(const CallExpr *E);
4405	/// Emit IR for __builtin_align_up/__builtin_align_down.
4406	RValue EmitBuiltinAlignTo(const CallExpr *E, bool AlignUp);
4407
4408	llvm::Function *generateBuiltinOSLogHelperFunction(
4409	const analyze_os_log::OSLogBufferLayout &Layout,
4410	CharUnits BufferAlignment);
4411
4412	RValue EmitBlockCallExpr(const CallExpr *E, ReturnValueSlot ReturnValue);
4413
4414	/// EmitTargetBuiltinExpr - Emit the given builtin call. Returns 0 if the call
4415	/// is unhandled by the current target.
4416	llvm::Value *EmitTargetBuiltinExpr(unsigned BuiltinID, const CallExpr *E,
4417	ReturnValueSlot ReturnValue);
4418
4419	llvm::Value *EmitAArch64CompareBuiltinExpr(llvm::Value *Op, llvm::Type *Ty,
4420	const llvm::CmpInst::Predicate Fp,
4421	const llvm::CmpInst::Predicate Ip,
4422	const llvm::Twine &Name = "");
4423	llvm::Value *EmitARMBuiltinExpr(unsigned BuiltinID, const CallExpr *E,
4424	ReturnValueSlot ReturnValue,
4425	llvm::Triple::ArchType Arch);
4426	llvm::Value *EmitARMMVEBuiltinExpr(unsigned BuiltinID, const CallExpr *E,
4427	ReturnValueSlot ReturnValue,
4428	llvm::Triple::ArchType Arch);
4429	llvm::Value *EmitARMCDEBuiltinExpr(unsigned BuiltinID, const CallExpr *E,
4430	ReturnValueSlot ReturnValue,
4431	llvm::Triple::ArchType Arch);
4432	llvm::Value *EmitCMSEClearRecord(llvm::Value *V, llvm::IntegerType *ITy,
4433	QualType RTy);
4434	llvm::Value *EmitCMSEClearRecord(llvm::Value *V, llvm::ArrayType *ATy,
4435	QualType RTy);
4436
4437	llvm::Value *EmitCommonNeonBuiltinExpr(unsigned BuiltinID,
4438	unsigned LLVMIntrinsic,
4439	unsigned AltLLVMIntrinsic,
4440	const char *NameHint,
4441	unsigned Modifier,
4442	const CallExpr *E,
4443	SmallVectorImpl<llvm::Value *> &Ops,
4444	Address PtrOp0, Address PtrOp1,
4445	llvm::Triple::ArchType Arch);
4446
4447	llvm::Function *LookupNeonLLVMIntrinsic(unsigned IntrinsicID,
4448	unsigned Modifier, llvm::Type *ArgTy,
4449	const CallExpr *E);
4450	llvm::Value *EmitNeonCall(llvm::Function *F,
4451	SmallVectorImpl<llvm::Value*> &O,
4452	const char *name,
4453	unsigned shift = 0, bool rightshift = false);
4454	llvm::Value *EmitNeonSplat(llvm::Value *V, llvm::Constant *Idx,
4455	const llvm::ElementCount &Count);
4456	llvm::Value *EmitNeonSplat(llvm::Value *V, llvm::Constant *Idx);
4457	llvm::Value *EmitNeonShiftVector(llvm::Value *V, llvm::Type *Ty,
4458	bool negateForRightShift);
4459	llvm::Value *EmitNeonRShiftImm(llvm::Value *Vec, llvm::Value *Amt,
4460	llvm::Type *Ty, bool usgn, const char *name);
4461	llvm::Value *vectorWrapScalar16(llvm::Value *Op);
4462	/// SVEBuiltinMemEltTy - Returns the memory element type for this memory
4463	/// access builtin. Only required if it can't be inferred from the base
4464	/// pointer operand.
4465	llvm::Type *SVEBuiltinMemEltTy(const SVETypeFlags &TypeFlags);
4466
4467	SmallVector<llvm::Type *, 2>
4468	getSVEOverloadTypes(const SVETypeFlags &TypeFlags, llvm::Type *ReturnType,
4469	ArrayRef<llvm::Value *> Ops);
4470	llvm::Type *getEltType(const SVETypeFlags &TypeFlags);
4471	llvm::ScalableVectorType *getSVEType(const SVETypeFlags &TypeFlags);
4472	llvm::ScalableVectorType *getSVEPredType(const SVETypeFlags &TypeFlags);
4473	llvm::Value *EmitSVETupleSetOrGet(const SVETypeFlags &TypeFlags,
4474	llvm::Type *ReturnType,
4475	ArrayRef<llvm::Value *> Ops);
4476	llvm::Value *EmitSVETupleCreate(const SVETypeFlags &TypeFlags,
4477	llvm::Type *ReturnType,
4478	ArrayRef<llvm::Value *> Ops);
4479	llvm::Value *EmitSVEAllTruePred(const SVETypeFlags &TypeFlags);
4480	llvm::Value *EmitSVEDupX(llvm::Value *Scalar);
4481	llvm::Value *EmitSVEDupX(llvm::Value *Scalar, llvm::Type *Ty);
4482	llvm::Value *EmitSVEReinterpret(llvm::Value *Val, llvm::Type *Ty);
4483	llvm::Value *EmitSVEPMull(const SVETypeFlags &TypeFlags,
4484	llvm::SmallVectorImpl<llvm::Value *> &Ops,
4485	unsigned BuiltinID);
4486	llvm::Value *EmitSVEMovl(const SVETypeFlags &TypeFlags,
4487	llvm::ArrayRef<llvm::Value *> Ops,
4488	unsigned BuiltinID);
4489	llvm::Value *EmitSVEPredicateCast(llvm::Value *Pred,
4490	llvm::ScalableVectorType *VTy);
4491	llvm::Value *EmitSVEGatherLoad(const SVETypeFlags &TypeFlags,
4492	llvm::SmallVectorImpl<llvm::Value *> &Ops,
4493	unsigned IntID);
4494	llvm::Value *EmitSVEScatterStore(const SVETypeFlags &TypeFlags,
4495	llvm::SmallVectorImpl<llvm::Value *> &Ops,
4496	unsigned IntID);
4497	llvm::Value *EmitSVEMaskedLoad(const CallExpr *, llvm::Type *ReturnTy,
4498	SmallVectorImpl<llvm::Value *> &Ops,
4499	unsigned BuiltinID, bool IsZExtReturn);
4500	llvm::Value *EmitSVEMaskedStore(const CallExpr *,
4501	SmallVectorImpl<llvm::Value *> &Ops,
4502	unsigned BuiltinID);
4503	llvm::Value *EmitSVEPrefetchLoad(const SVETypeFlags &TypeFlags,
4504	SmallVectorImpl<llvm::Value *> &Ops,
4505	unsigned BuiltinID);
4506	llvm::Value *EmitSVEGatherPrefetch(const SVETypeFlags &TypeFlags,
4507	SmallVectorImpl<llvm::Value *> &Ops,
4508	unsigned IntID);
4509	llvm::Value *EmitSVEStructLoad(const SVETypeFlags &TypeFlags,
4510	SmallVectorImpl<llvm::Value *> &Ops,
4511	unsigned IntID);
4512	llvm::Value *EmitSVEStructStore(const SVETypeFlags &TypeFlags,
4513	SmallVectorImpl<llvm::Value *> &Ops,
4514	unsigned IntID);
4515	/// FormSVEBuiltinResult - Returns the struct of scalable vectors as a wider
4516	/// vector. It extracts the scalable vector from the struct and inserts into
4517	/// the wider vector. This avoids the error when allocating space in llvm
4518	/// for struct of scalable vectors if a function returns struct.
4519	llvm::Value *FormSVEBuiltinResult(llvm::Value *Call);
4520
4521	llvm::Value *EmitAArch64SVEBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
4522
4523	llvm::Value *EmitSMELd1St1(const SVETypeFlags &TypeFlags,
4524	llvm::SmallVectorImpl<llvm::Value *> &Ops,
4525	unsigned IntID);
4526	llvm::Value *EmitSMEReadWrite(const SVETypeFlags &TypeFlags,
4527	llvm::SmallVectorImpl<llvm::Value *> &Ops,
4528	unsigned IntID);
4529	llvm::Value *EmitSMEZero(const SVETypeFlags &TypeFlags,
4530	llvm::SmallVectorImpl<llvm::Value *> &Ops,
4531	unsigned IntID);
4532	llvm::Value *EmitSMELdrStr(const SVETypeFlags &TypeFlags,
4533	llvm::SmallVectorImpl<llvm::Value *> &Ops,
4534	unsigned IntID);
4535
4536	void GetAArch64SVEProcessedOperands(unsigned BuiltinID, const CallExpr *E,
4537	SmallVectorImpl<llvm::Value *> &Ops,
4538	SVETypeFlags TypeFlags);
4539
4540	llvm::Value *EmitAArch64SMEBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
4541
4542	llvm::Value *EmitAArch64BuiltinExpr(unsigned BuiltinID, const CallExpr *E,
4543	llvm::Triple::ArchType Arch);
4544	llvm::Value *EmitBPFBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
4545
4546	llvm::Value *BuildVector(ArrayRef<llvm::Value*> Ops);
4547	llvm::Value *EmitX86BuiltinExpr(unsigned BuiltinID, const CallExpr *E);
4548	llvm::Value *EmitPPCBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
4549	llvm::Value *EmitAMDGPUBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
4550	llvm::Value *EmitHLSLBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
4551	llvm::Value *EmitScalarOrConstFoldImmArg(unsigned ICEArguments, unsigned Idx,
4552	const CallExpr *E);
4553	llvm::Value *EmitSystemZBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
4554	llvm::Value *EmitNVPTXBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
4555	llvm::Value *EmitWebAssemblyBuiltinExpr(unsigned BuiltinID,
4556	const CallExpr *E);
4557	llvm::Value *EmitHexagonBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
4558	llvm::Value *EmitRISCVBuiltinExpr(unsigned BuiltinID, const CallExpr *E,
4559	ReturnValueSlot ReturnValue);
4560	void ProcessOrderScopeAMDGCN(llvm::Value *Order, llvm::Value *Scope,
4561	llvm::AtomicOrdering &AO,
4562	llvm::SyncScope::ID &SSID);
4563
4564	enum class MSVCIntrin;
4565	llvm::Value *EmitMSVCBuiltinExpr(MSVCIntrin BuiltinID, const CallExpr *E);
4566
4567	llvm::Value *EmitBuiltinAvailable(const VersionTuple &Version);
4568
4569	llvm::Value *EmitObjCProtocolExpr(const ObjCProtocolExpr *E);
4570	llvm::Value *EmitObjCStringLiteral(const ObjCStringLiteral *E);
4571	llvm::Value *EmitObjCBoxedExpr(const ObjCBoxedExpr *E);
4572	llvm::Value *EmitObjCArrayLiteral(const ObjCArrayLiteral *E);
4573	llvm::Value *EmitObjCDictionaryLiteral(const ObjCDictionaryLiteral *E);
4574	llvm::Value *EmitObjCCollectionLiteral(const Expr *E,
4575	const ObjCMethodDecl *MethodWithObjects);
4576	llvm::Value *EmitObjCSelectorExpr(const ObjCSelectorExpr *E);
4577	RValue EmitObjCMessageExpr(const ObjCMessageExpr *E,
4578	ReturnValueSlot Return = ReturnValueSlot());
4579
4580	/// Retrieves the default cleanup kind for an ARC cleanup.
4581	/// Except under -fobjc-arc-eh, ARC cleanups are normal-only.
4582	CleanupKind getARCCleanupKind() {
4583	return CGM.getCodeGenOpts().ObjCAutoRefCountExceptions
4584	? NormalAndEHCleanup : NormalCleanup;
4585	}
4586
4587	// ARC primitives.
4588	void EmitARCInitWeak(Address addr, llvm::Value *value);
4589	void EmitARCDestroyWeak(Address addr);
4590	llvm::Value *EmitARCLoadWeak(Address addr);
4591	llvm::Value *EmitARCLoadWeakRetained(Address addr);
4592	llvm::Value *EmitARCStoreWeak(Address addr, llvm::Value *value, bool ignored);
4593	void emitARCCopyAssignWeak(QualType Ty, Address DstAddr, Address SrcAddr);
4594	void emitARCMoveAssignWeak(QualType Ty, Address DstAddr, Address SrcAddr);
4595	void EmitARCCopyWeak(Address dst, Address src);
4596	void EmitARCMoveWeak(Address dst, Address src);
4597	llvm::Value *EmitARCRetainAutorelease(QualType type, llvm::Value *value);
4598	llvm::Value *EmitARCRetainAutoreleaseNonBlock(llvm::Value *value);
4599	llvm::Value *EmitARCStoreStrong(LValue lvalue, llvm::Value *value,
4600	bool resultIgnored);
4601	llvm::Value *EmitARCStoreStrongCall(Address addr, llvm::Value *value,
4602	bool resultIgnored);
4603	llvm::Value *EmitARCRetain(QualType type, llvm::Value *value);
4604	llvm::Value *EmitARCRetainNonBlock(llvm::Value *value);
4605	llvm::Value *EmitARCRetainBlock(llvm::Value *value, bool mandatory);
4606	void EmitARCDestroyStrong(Address addr, ARCPreciseLifetime_t precise);
4607	void EmitARCRelease(llvm::Value *value, ARCPreciseLifetime_t precise);
4608	llvm::Value *EmitARCAutorelease(llvm::Value *value);
4609	llvm::Value *EmitARCAutoreleaseReturnValue(llvm::Value *value);
4610	llvm::Value *EmitARCRetainAutoreleaseReturnValue(llvm::Value *value);
4611	llvm::Value *EmitARCRetainAutoreleasedReturnValue(llvm::Value *value);
4612	llvm::Value *EmitARCUnsafeClaimAutoreleasedReturnValue(llvm::Value *value);
4613
4614	llvm::Value *EmitObjCAutorelease(llvm::Value *value, llvm::Type *returnType);
4615	llvm::Value *EmitObjCRetainNonBlock(llvm::Value *value,
4616	llvm::Type *returnType);
4617	void EmitObjCRelease(llvm::Value *value, ARCPreciseLifetime_t precise);
4618
4619	std::pair<LValue,llvm::Value*>
4620	EmitARCStoreAutoreleasing(const BinaryOperator *e);
4621	std::pair<LValue,llvm::Value*>
4622	EmitARCStoreStrong(const BinaryOperator *e, bool ignored);
4623	std::pair<LValue,llvm::Value*>
4624	EmitARCStoreUnsafeUnretained(const BinaryOperator *e, bool ignored);
4625
4626	llvm::Value *EmitObjCAlloc(llvm::Value *value,
4627	llvm::Type *returnType);
4628	llvm::Value *EmitObjCAllocWithZone(llvm::Value *value,
4629	llvm::Type *returnType);
4630	llvm::Value *EmitObjCAllocInit(llvm::Value *value, llvm::Type *resultType);
4631
4632	llvm::Value *EmitObjCThrowOperand(const Expr *expr);
4633	llvm::Value *EmitObjCConsumeObject(QualType T, llvm::Value *Ptr);
4634	llvm::Value *EmitObjCExtendObjectLifetime(QualType T, llvm::Value *Ptr);
4635
4636	llvm::Value *EmitARCExtendBlockObject(const Expr *expr);
4637	llvm::Value *EmitARCReclaimReturnedObject(const Expr *e,
4638	bool allowUnsafeClaim);
4639	llvm::Value *EmitARCRetainScalarExpr(const Expr *expr);
4640	llvm::Value *EmitARCRetainAutoreleaseScalarExpr(const Expr *expr);
4641	llvm::Value *EmitARCUnsafeUnretainedScalarExpr(const Expr *expr);
4642
4643	void EmitARCIntrinsicUse(ArrayRef<llvm::Value*> values);
4644
4645	void EmitARCNoopIntrinsicUse(ArrayRef<llvm::Value *> values);
4646
4647	static Destroyer destroyARCStrongImprecise;
4648	static Destroyer destroyARCStrongPrecise;
4649	static Destroyer destroyARCWeak;
4650	static Destroyer emitARCIntrinsicUse;
4651	static Destroyer destroyNonTrivialCStruct;
4652
4653	void EmitObjCAutoreleasePoolPop(llvm::Value *Ptr);
4654	llvm::Value *EmitObjCAutoreleasePoolPush();
4655	llvm::Value *EmitObjCMRRAutoreleasePoolPush();
4656	void EmitObjCAutoreleasePoolCleanup(llvm::Value *Ptr);
4657	void EmitObjCMRRAutoreleasePoolPop(llvm::Value *Ptr);
4658
4659	/// Emits a reference binding to the passed in expression.
4660	RValue EmitReferenceBindingToExpr(const Expr *E);
4661
4662	//===--------------------------------------------------------------------===//
4663	// Expression Emission
4664	//===--------------------------------------------------------------------===//
4665
4666	// Expressions are broken into three classes: scalar, complex, aggregate.
4667
4668	/// EmitScalarExpr - Emit the computation of the specified expression of LLVM
4669	/// scalar type, returning the result.
4670	llvm::Value *EmitScalarExpr(const Expr *E , bool IgnoreResultAssign = false);
4671
4672	/// Emit a conversion from the specified type to the specified destination
4673	/// type, both of which are LLVM scalar types.
4674	llvm::Value *EmitScalarConversion(llvm::Value *Src, QualType SrcTy,
4675	QualType DstTy, SourceLocation Loc);
4676
4677	/// Emit a conversion from the specified complex type to the specified
4678	/// destination type, where the destination type is an LLVM scalar type.
4679	llvm::Value *EmitComplexToScalarConversion(ComplexPairTy Src, QualType SrcTy,
4680	QualType DstTy,
4681	SourceLocation Loc);
4682
4683	/// EmitAggExpr - Emit the computation of the specified expression
4684	/// of aggregate type. The result is computed into the given slot,
4685	/// which may be null to indicate that the value is not needed.
4686	void EmitAggExpr(const Expr *E, AggValueSlot AS);
4687
4688	/// EmitAggExprToLValue - Emit the computation of the specified expression of
4689	/// aggregate type into a temporary LValue.
4690	LValue EmitAggExprToLValue(const Expr *E);
4691
4692	/// Build all the stores needed to initialize an aggregate at Dest with the
4693	/// value Val.
4694	void EmitAggregateStore(llvm::Value *Val, Address Dest, bool DestIsVolatile);
4695
4696	/// EmitExtendGCLifetime - Given a pointer to an Objective-C object,
4697	/// make sure it survives garbage collection until this point.
4698	void EmitExtendGCLifetime(llvm::Value *object);
4699
4700	/// EmitComplexExpr - Emit the computation of the specified expression of
4701	/// complex type, returning the result.
4702	ComplexPairTy EmitComplexExpr(const Expr *E,
4703	bool IgnoreReal = false,
4704	bool IgnoreImag = false);
4705
4706	/// EmitComplexExprIntoLValue - Emit the given expression of complex
4707	/// type and place its result into the specified l-value.
4708	void EmitComplexExprIntoLValue(const Expr *E, LValue dest, bool isInit);
4709
4710	/// EmitStoreOfComplex - Store a complex number into the specified l-value.
4711	void EmitStoreOfComplex(ComplexPairTy V, LValue dest, bool isInit);
4712
4713	/// EmitLoadOfComplex - Load a complex number from the specified l-value.
4714	ComplexPairTy EmitLoadOfComplex(LValue src, SourceLocation loc);
4715
4716	ComplexPairTy EmitPromotedComplexExpr(const Expr *E, QualType PromotionType);
4717	llvm::Value *EmitPromotedScalarExpr(const Expr *E, QualType PromotionType);
4718	ComplexPairTy EmitPromotedValue(ComplexPairTy result, QualType PromotionType);
4719	ComplexPairTy EmitUnPromotedValue(ComplexPairTy result, QualType PromotionType);
4720
4721	Address emitAddrOfRealComponent(Address complex, QualType complexType);
4722	Address emitAddrOfImagComponent(Address complex, QualType complexType);
4723
4724	/// AddInitializerToStaticVarDecl - Add the initializer for 'D' to the
4725	/// global variable that has already been created for it. If the initializer
4726	/// has a different type than GV does, this may free GV and return a different
4727	/// one. Otherwise it just returns GV.
4728	llvm::GlobalVariable *
4729	AddInitializerToStaticVarDecl(const VarDecl &D,
4730	llvm::GlobalVariable *GV);
4731
4732	// Emit an @llvm.invariant.start call for the given memory region.
4733	void EmitInvariantStart(llvm::Constant *Addr, CharUnits Size);
4734
4735	/// EmitCXXGlobalVarDeclInit - Create the initializer for a C++
4736	/// variable with global storage.
4737	void EmitCXXGlobalVarDeclInit(const VarDecl &D, llvm::GlobalVariable *GV,
4738	bool PerformInit);
4739
4740	llvm::Function *createAtExitStub(const VarDecl &VD, llvm::FunctionCallee Dtor,
4741	llvm::Constant *Addr);
4742
4743	llvm::Function *createTLSAtExitStub(const VarDecl &VD,
4744	llvm::FunctionCallee Dtor,
4745	llvm::Constant *Addr,
4746	llvm::FunctionCallee &AtExit);
4747
4748	/// Call atexit() with a function that passes the given argument to
4749	/// the given function.
4750	void registerGlobalDtorWithAtExit(const VarDecl &D, llvm::FunctionCallee fn,
4751	llvm::Constant *addr);
4752
4753	/// Registers the dtor using 'llvm.global_dtors' for platforms that do not
4754	/// support an 'atexit()' function.
4755	void registerGlobalDtorWithLLVM(const VarDecl &D, llvm::FunctionCallee fn,
4756	llvm::Constant *addr);
4757
4758	/// Call atexit() with function dtorStub.
4759	void registerGlobalDtorWithAtExit(llvm::Constant *dtorStub);
4760
4761	/// Call unatexit() with function dtorStub.
4762	llvm::Value *unregisterGlobalDtorWithUnAtExit(llvm::Constant *dtorStub);
4763
4764	/// Emit code in this function to perform a guarded variable
4765	/// initialization. Guarded initializations are used when it's not
4766	/// possible to prove that an initialization will be done exactly
4767	/// once, e.g. with a static local variable or a static data member
4768	/// of a class template.
4769	void EmitCXXGuardedInit(const VarDecl &D, llvm::GlobalVariable *DeclPtr,
4770	bool PerformInit);
4771
4772	enum class GuardKind { VariableGuard, TlsGuard };
4773
4774	/// Emit a branch to select whether or not to perform guarded initialization.
4775	void EmitCXXGuardedInitBranch(llvm::Value *NeedsInit,
4776	llvm::BasicBlock *InitBlock,
4777	llvm::BasicBlock *NoInitBlock,
4778	GuardKind Kind, const VarDecl *D);
4779
4780	/// GenerateCXXGlobalInitFunc - Generates code for initializing global
4781	/// variables.
4782	void
4783	GenerateCXXGlobalInitFunc(llvm::Function *Fn,
4784	ArrayRef<llvm::Function *> CXXThreadLocals,
4785	ConstantAddress Guard = ConstantAddress::invalid());
4786
4787	/// GenerateCXXGlobalCleanUpFunc - Generates code for cleaning up global
4788	/// variables.
4789	void GenerateCXXGlobalCleanUpFunc(
4790	llvm::Function *Fn,
4791	ArrayRef<std::tuple<llvm::FunctionType *, llvm::WeakTrackingVH,
4792	llvm::Constant *>>
4793	DtorsOrStermFinalizers);
4794
4795	void GenerateCXXGlobalVarDeclInitFunc(llvm::Function *Fn,
4796	const VarDecl *D,
4797	llvm::GlobalVariable *Addr,
4798	bool PerformInit);
4799
4800	void EmitCXXConstructExpr(const CXXConstructExpr *E, AggValueSlot Dest);
4801
4802	void EmitSynthesizedCXXCopyCtor(Address Dest, Address Src, const Expr *Exp);
4803
4804	void EmitCXXThrowExpr(const CXXThrowExpr *E, bool KeepInsertionPoint = true);
4805
4806	RValue EmitAtomicExpr(AtomicExpr *E);
4807
4808	//===--------------------------------------------------------------------===//
4809	// Annotations Emission
4810	//===--------------------------------------------------------------------===//
4811
4812	/// Emit an annotation call (intrinsic).
4813	llvm::Value *EmitAnnotationCall(llvm::Function *AnnotationFn,
4814	llvm::Value *AnnotatedVal,
4815	StringRef AnnotationStr,
4816	SourceLocation Location,
4817	const AnnotateAttr *Attr);
4818
4819	/// Emit local annotations for the local variable V, declared by D.
4820	void EmitVarAnnotations(const VarDecl *D, llvm::Value *V);
4821
4822	/// Emit field annotations for the given field & value. Returns the
4823	/// annotation result.
4824	Address EmitFieldAnnotations(const FieldDecl *D, Address V);
4825
4826	//===--------------------------------------------------------------------===//
4827	// Internal Helpers
4828	//===--------------------------------------------------------------------===//
4829
4830	/// ContainsLabel - Return true if the statement contains a label in it. If
4831	/// this statement is not executed normally, it not containing a label means
4832	/// that we can just remove the code.
4833	static bool ContainsLabel(const Stmt *S, bool IgnoreCaseStmts = false);
4834
4835	/// containsBreak - Return true if the statement contains a break out of it.
4836	/// If the statement (recursively) contains a switch or loop with a break
4837	/// inside of it, this is fine.
4838	static bool containsBreak(const Stmt *S);
4839
4840	/// Determine if the given statement might introduce a declaration into the
4841	/// current scope, by being a (possibly-labelled) DeclStmt.
4842	static bool mightAddDeclToScope(const Stmt *S);
4843
4844	/// ConstantFoldsToSimpleInteger - If the specified expression does not fold
4845	/// to a constant, or if it does but contains a label, return false. If it
4846	/// constant folds return true and set the boolean result in Result.
4847	bool ConstantFoldsToSimpleInteger(const Expr *Cond, bool &Result,
4848	bool AllowLabels = false);
4849
4850	/// ConstantFoldsToSimpleInteger - If the specified expression does not fold
4851	/// to a constant, or if it does but contains a label, return false. If it
4852	/// constant folds return true and set the folded value.
4853	bool ConstantFoldsToSimpleInteger(const Expr *Cond, llvm::APSInt &Result,
4854	bool AllowLabels = false);
4855
4856	/// Ignore parentheses and logical-NOT to track conditions consistently.
4857	static const Expr *stripCond(const Expr *C);
4858
4859	/// isInstrumentedCondition - Determine whether the given condition is an
4860	/// instrumentable condition (i.e. no "&&" or "||").
4861	static bool isInstrumentedCondition(const Expr *C);
4862
4863	/// EmitBranchToCounterBlock - Emit a conditional branch to a new block that
4864	/// increments a profile counter based on the semantics of the given logical
4865	/// operator opcode. This is used to instrument branch condition coverage
4866	/// for logical operators.
4867	void EmitBranchToCounterBlock(const Expr *Cond, BinaryOperator::Opcode LOp,
4868	llvm::BasicBlock *TrueBlock,
4869	llvm::BasicBlock *FalseBlock,
4870	uint64_t TrueCount = 0,
4871	Stmt::Likelihood LH = Stmt::LH_None,
4872	const Expr *CntrIdx = nullptr);
4873
4874	/// EmitBranchOnBoolExpr - Emit a branch on a boolean condition (e.g. for an
4875	/// if statement) to the specified blocks. Based on the condition, this might
4876	/// try to simplify the codegen of the conditional based on the branch.
4877	/// TrueCount should be the number of times we expect the condition to
4878	/// evaluate to true based on PGO data.
4879	void EmitBranchOnBoolExpr(const Expr *Cond, llvm::BasicBlock *TrueBlock,
4880	llvm::BasicBlock *FalseBlock, uint64_t TrueCount,
4881	Stmt::Likelihood LH = Stmt::LH_None,
4882	const Expr *ConditionalOp = nullptr);
4883
4884	/// Given an assignment `*LHS = RHS`, emit a test that checks if \p RHS is
4885	/// nonnull, if \p LHS is marked _Nonnull.
4886	void EmitNullabilityCheck(LValue LHS, llvm::Value *RHS, SourceLocation Loc);
4887
4888	/// An enumeration which makes it easier to specify whether or not an
4889	/// operation is a subtraction.
4890	enum { NotSubtraction = false, IsSubtraction = true };
4891
4892	/// Same as IRBuilder::CreateInBoundsGEP, but additionally emits a check to
4893	/// detect undefined behavior when the pointer overflow sanitizer is enabled.
4894	/// \p SignedIndices indicates whether any of the GEP indices are signed.
4895	/// \p IsSubtraction indicates whether the expression used to form the GEP
4896	/// is a subtraction.
4897	llvm::Value *EmitCheckedInBoundsGEP(llvm::Type *ElemTy, llvm::Value *Ptr,
4898	ArrayRef<llvm::Value *> IdxList,
4899	bool SignedIndices,
4900	bool IsSubtraction,
4901	SourceLocation Loc,
4902	const Twine &Name = "");
4903
4904	Address EmitCheckedInBoundsGEP(Address Addr, ArrayRef<llvm::Value *> IdxList,
4905	llvm::Type *elementType, bool SignedIndices,
4906	bool IsSubtraction, SourceLocation Loc,
4907	CharUnits Align, const Twine &Name = "");
4908
4909	/// Specifies which type of sanitizer check to apply when handling a
4910	/// particular builtin.
4911	enum BuiltinCheckKind {
4912	BCK_CTZPassedZero,
4913	BCK_CLZPassedZero,
4914	};
4915
4916	/// Emits an argument for a call to a builtin. If the builtin sanitizer is
4917	/// enabled, a runtime check specified by \p Kind is also emitted.
4918	llvm::Value *EmitCheckedArgForBuiltin(const Expr *E, BuiltinCheckKind Kind);
4919
4920	/// Emit a description of a type in a format suitable for passing to
4921	/// a runtime sanitizer handler.
4922	llvm::Constant *EmitCheckTypeDescriptor(QualType T);
4923
4924	/// Convert a value into a format suitable for passing to a runtime
4925	/// sanitizer handler.
4926	llvm::Value *EmitCheckValue(llvm::Value *V);
4927
4928	/// Emit a description of a source location in a format suitable for
4929	/// passing to a runtime sanitizer handler.
4930	llvm::Constant *EmitCheckSourceLocation(SourceLocation Loc);
4931
4932	void EmitKCFIOperandBundle(const CGCallee &Callee,
4933	SmallVectorImpl<llvm::OperandBundleDef> &Bundles);
4934
4935	/// Create a basic block that will either trap or call a handler function in
4936	/// the UBSan runtime with the provided arguments, and create a conditional
4937	/// branch to it.
4938	void EmitCheck(ArrayRef<std::pair<llvm::Value *, SanitizerMask>> Checked,
4939	SanitizerHandler Check, ArrayRef<llvm::Constant *> StaticArgs,
4940	ArrayRef<llvm::Value *> DynamicArgs);
4941
4942	/// Emit a slow path cross-DSO CFI check which calls __cfi_slowpath
4943	/// if Cond if false.
4944	void EmitCfiSlowPathCheck(SanitizerMask Kind, llvm::Value *Cond,
4945	llvm::ConstantInt *TypeId, llvm::Value *Ptr,
4946	ArrayRef<llvm::Constant *> StaticArgs);
4947
4948	/// Emit a reached-unreachable diagnostic if \p Loc is valid and runtime
4949	/// checking is enabled. Otherwise, just emit an unreachable instruction.
4950	void EmitUnreachable(SourceLocation Loc);
4951
4952	/// Create a basic block that will call the trap intrinsic, and emit a
4953	/// conditional branch to it, for the -ftrapv checks.
4954	void EmitTrapCheck(llvm::Value *Checked, SanitizerHandler CheckHandlerID);
4955
4956	/// Emit a call to trap or debugtrap and attach function attribute
4957	/// "trap-func-name" if specified.
4958	llvm::CallInst *EmitTrapCall(llvm::Intrinsic::ID IntrID);
4959
4960	/// Emit a stub for the cross-DSO CFI check function.
4961	void EmitCfiCheckStub();
4962
4963	/// Emit a cross-DSO CFI failure handling function.
4964	void EmitCfiCheckFail();
4965
4966	/// Create a check for a function parameter that may potentially be
4967	/// declared as non-null.
4968	void EmitNonNullArgCheck(RValue RV, QualType ArgType, SourceLocation ArgLoc,
4969	AbstractCallee AC, unsigned ParmNum);
4970
4971	void EmitNonNullArgCheck(Address Addr, QualType ArgType,
4972	SourceLocation ArgLoc, AbstractCallee AC,
4973	unsigned ParmNum);
4974
4975	/// EmitCallArg - Emit a single call argument.
4976	void EmitCallArg(CallArgList &args, const Expr *E, QualType ArgType);
4977
4978	/// EmitDelegateCallArg - We are performing a delegate call; that
4979	/// is, the current function is delegating to another one. Produce
4980	/// a r-value suitable for passing the given parameter.
4981	void EmitDelegateCallArg(CallArgList &args, const VarDecl *param,
4982	SourceLocation loc);
4983
4984	/// SetFPAccuracy - Set the minimum required accuracy of the given floating
4985	/// point operation, expressed as the maximum relative error in ulp.
4986	void SetFPAccuracy(llvm::Value *Val, float Accuracy);
4987
4988	/// Set the minimum required accuracy of the given sqrt operation
4989	/// based on CodeGenOpts.
4990	void SetSqrtFPAccuracy(llvm::Value *Val);
4991
4992	/// Set the minimum required accuracy of the given sqrt operation based on
4993	/// CodeGenOpts.
4994	void SetDivFPAccuracy(llvm::Value *Val);
4995
4996	/// Set the codegen fast-math flags.
4997	void SetFastMathFlags(FPOptions FPFeatures);
4998
4999	// Truncate or extend a boolean vector to the requested number of elements.
5000	llvm::Value *emitBoolVecConversion(llvm::Value *SrcVec,
5001	unsigned NumElementsDst,
5002	const llvm::Twine &Name = "");
5003	// Adds a convergence_ctrl token to |Input| and emits the required parent
5004	// convergence instructions.
5005	llvm::CallBase *addControlledConvergenceToken(llvm::CallBase *Input);
5006
5007	private:
5008	// Emits a convergence_loop instruction for the given |BB|, with |ParentToken|
5009	// as it's parent convergence instr.
5010	llvm::IntrinsicInst *emitConvergenceLoopToken(llvm::BasicBlock *BB,
5011	llvm::Value *ParentToken);
5012	// Adds a convergence_ctrl token with |ParentToken| as parent convergence
5013	// instr to the call |Input|.
5014	llvm::CallBase *addConvergenceControlToken(llvm::CallBase *Input,
5015	llvm::Value *ParentToken);
5016	// Find the convergence_entry instruction |F|, or emits ones if none exists.
5017	// Returns the convergence instruction.
5018	llvm::IntrinsicInst *getOrEmitConvergenceEntryToken(llvm::Function *F);
5019	// Find the convergence_loop instruction for the loop defined by |LI|, or
5020	// emits one if none exists. Returns the convergence instruction.
5021	llvm::IntrinsicInst *getOrEmitConvergenceLoopToken(const LoopInfo *LI);
5022
5023	private:
5024	llvm::MDNode *getRangeForLoadFromType(QualType Ty);
5025	void EmitReturnOfRValue(RValue RV, QualType Ty);
5026
5027	void deferPlaceholderReplacement(llvm::Instruction *Old, llvm::Value *New);
5028
5029	llvm::SmallVector<std::pair<llvm::WeakTrackingVH, llvm::Value *>, 4>
5030	DeferredReplacements;
5031
5032	/// Set the address of a local variable.
5033	void setAddrOfLocalVar(const VarDecl *VD, Address Addr) {
5034	assert(!LocalDeclMap.count(VD) && "Decl already exists in LocalDeclMap!");
5035	LocalDeclMap.insert({VD, Addr});
5036	}
5037
5038	/// ExpandTypeFromArgs - Reconstruct a structure of type \arg Ty
5039	/// from function arguments into \arg Dst. See ABIArgInfo::Expand.
5040	///
5041	/// \param AI - The first function argument of the expansion.
5042	void ExpandTypeFromArgs(QualType Ty, LValue Dst,
5043	llvm::Function::arg_iterator &AI);
5044
5045	/// ExpandTypeToArgs - Expand an CallArg \arg Arg, with the LLVM type for \arg
5046	/// Ty, into individual arguments on the provided vector \arg IRCallArgs,
5047	/// starting at index \arg IRCallArgPos. See ABIArgInfo::Expand.
5048	void ExpandTypeToArgs(QualType Ty, CallArg Arg, llvm::FunctionType *IRFuncTy,
5049	SmallVectorImpl<llvm::Value *> &IRCallArgs,
5050	unsigned &IRCallArgPos);
5051
5052	std::pair<llvm::Value *, llvm::Type *>
5053	EmitAsmInput(const TargetInfo::ConstraintInfo &Info, const Expr *InputExpr,
5054	std::string &ConstraintStr);
5055
5056	std::pair<llvm::Value *, llvm::Type *>
5057	EmitAsmInputLValue(const TargetInfo::ConstraintInfo &Info, LValue InputValue,
5058	QualType InputType, std::string &ConstraintStr,
5059	SourceLocation Loc);
5060
5061	/// Attempts to statically evaluate the object size of E. If that
5062	/// fails, emits code to figure the size of E out for us. This is
5063	/// pass_object_size aware.
5064	///
5065	/// If EmittedExpr is non-null, this will use that instead of re-emitting E.
5066	llvm::Value *evaluateOrEmitBuiltinObjectSize(const Expr *E, unsigned Type,
5067	llvm::IntegerType *ResType,
5068	llvm::Value *EmittedE,
5069	bool IsDynamic);
5070
5071	/// Emits the size of E, as required by __builtin_object_size. This
5072	/// function is aware of pass_object_size parameters, and will act accordingly
5073	/// if E is a parameter with the pass_object_size attribute.
5074	llvm::Value *emitBuiltinObjectSize(const Expr *E, unsigned Type,
5075	llvm::IntegerType *ResType,
5076	llvm::Value *EmittedE,
5077	bool IsDynamic);
5078
5079	llvm::Value *emitFlexibleArrayMemberSize(const Expr *E, unsigned Type,
5080	llvm::IntegerType *ResType);
5081
5082	void emitZeroOrPatternForAutoVarInit(QualType type, const VarDecl &D,
5083	Address Loc);
5084
5085	public:
5086	enum class EvaluationOrder {
5087	///! No language constraints on evaluation order.
5088	Default,
5089	///! Language semantics require left-to-right evaluation.
5090	ForceLeftToRight,
5091	///! Language semantics require right-to-left evaluation.
5092	ForceRightToLeft
5093	};
5094
5095	// Wrapper for function prototype sources. Wraps either a FunctionProtoType or
5096	// an ObjCMethodDecl.
5097	struct PrototypeWrapper {
5098	llvm::PointerUnion<const FunctionProtoType *, const ObjCMethodDecl *> P;
5099
5100	PrototypeWrapper(const FunctionProtoType *FT) : P(FT) {}
5101	PrototypeWrapper(const ObjCMethodDecl *MD) : P(MD) {}
5102	};
5103
5104	void EmitCallArgs(CallArgList &Args, PrototypeWrapper Prototype,
5105	llvm::iterator_range<CallExpr::const_arg_iterator> ArgRange,
5106	AbstractCallee AC = AbstractCallee(),
5107	unsigned ParamsToSkip = 0,
5108	EvaluationOrder Order = EvaluationOrder::Default);
5109
5110	/// EmitPointerWithAlignment - Given an expression with a pointer type,
5111	/// emit the value and compute our best estimate of the alignment of the
5112	/// pointee.
5113	///
5114	/// \param BaseInfo - If non-null, this will be initialized with
5115	/// information about the source of the alignment and the may-alias
5116	/// attribute. Note that this function will conservatively fall back on
5117	/// the type when it doesn't recognize the expression and may-alias will
5118	/// be set to false.
5119	///
5120	/// One reasonable way to use this information is when there's a language
5121	/// guarantee that the pointer must be aligned to some stricter value, and
5122	/// we're simply trying to ensure that sufficiently obvious uses of under-
5123	/// aligned objects don't get miscompiled; for example, a placement new
5124	/// into the address of a local variable. In such a case, it's quite
5125	/// reasonable to just ignore the returned alignment when it isn't from an
5126	/// explicit source.
5127	Address
5128	EmitPointerWithAlignment(const Expr *Addr, LValueBaseInfo *BaseInfo = nullptr,
5129	TBAAAccessInfo *TBAAInfo = nullptr,
5130	KnownNonNull_t IsKnownNonNull = NotKnownNonNull);
5131
5132	/// If \p E references a parameter with pass_object_size info or a constant
5133	/// array size modifier, emit the object size divided by the size of \p EltTy.
5134	/// Otherwise return null.
5135	llvm::Value *LoadPassedObjectSize(const Expr *E, QualType EltTy);
5136
5137	void EmitSanitizerStatReport(llvm::SanitizerStatKind SSK);
5138
5139	struct MultiVersionResolverOption {
5140	llvm::Function *Function;
5141	struct Conds {
5142	StringRef Architecture;
5143	llvm::SmallVector<StringRef, 8> Features;
5144
5145	Conds(StringRef Arch, ArrayRef<StringRef> Feats)
5146	: Architecture(Arch), Features(Feats.begin(), Feats.end()) {}
5147	} Conditions;
5148
5149	MultiVersionResolverOption(llvm::Function *F, StringRef Arch,
5150	ArrayRef<StringRef> Feats)
5151	: Function(F), Conditions(Arch, Feats) {}
5152	};
5153
5154	// Emits the body of a multiversion function's resolver. Assumes that the
5155	// options are already sorted in the proper order, with the 'default' option
5156	// last (if it exists).
5157	void EmitMultiVersionResolver(llvm::Function *Resolver,
5158	ArrayRef<MultiVersionResolverOption> Options);
5159	void
5160	EmitX86MultiVersionResolver(llvm::Function *Resolver,
5161	ArrayRef<MultiVersionResolverOption> Options);
5162	void
5163	EmitAArch64MultiVersionResolver(llvm::Function *Resolver,
5164	ArrayRef<MultiVersionResolverOption> Options);
5165
5166	private:
5167	QualType getVarArgType(const Expr *Arg);
5168
5169	void EmitDeclMetadata();
5170
5171	BlockByrefHelpers *buildByrefHelpers(llvm::StructType &byrefType,
5172	const AutoVarEmission &emission);
5173
5174	void AddObjCARCExceptionMetadata(llvm::Instruction *Inst);
5175
5176	llvm::Value *GetValueForARMHint(unsigned BuiltinID);
5177	llvm::Value *EmitX86CpuIs(const CallExpr *E);
5178	llvm::Value *EmitX86CpuIs(StringRef CPUStr);
5179	llvm::Value *EmitX86CpuSupports(const CallExpr *E);
5180	llvm::Value *EmitX86CpuSupports(ArrayRef<StringRef> FeatureStrs);
5181	llvm::Value *EmitX86CpuSupports(std::array<uint32_t, 4> FeatureMask);
5182	llvm::Value *EmitX86CpuInit();
5183	llvm::Value *FormX86ResolverCondition(const MultiVersionResolverOption &RO);
5184	llvm::Value *EmitAArch64CpuInit();
5185	llvm::Value *
5186	FormAArch64ResolverCondition(const MultiVersionResolverOption &RO);
5187	llvm::Value *EmitAArch64CpuSupports(const CallExpr *E);
5188	llvm::Value *EmitAArch64CpuSupports(ArrayRef<StringRef> FeatureStrs);
5189	};
5190
5191	inline DominatingLLVMValue::saved_type
5192	DominatingLLVMValue::save(CodeGenFunction &CGF, llvm::Value *value) {
5193	if (!needsSaving(value)) return saved_type(value, false);
5194
5195	// Otherwise, we need an alloca.
5196	auto align = CharUnits::fromQuantity(
5197	Quantity: CGF.CGM.getDataLayout().getPrefTypeAlign(Ty: value->getType()));
5198	Address alloca =
5199	CGF.CreateTempAlloca(Ty: value->getType(), align, Name: "cond-cleanup.save");
5200	CGF.Builder.CreateStore(Val: value, Addr: alloca);
5201
5202	return saved_type(alloca.emitRawPointer(CGF), true);
5203	}
5204
5205	inline llvm::Value *DominatingLLVMValue::restore(CodeGenFunction &CGF,
5206	saved_type value) {
5207	// If the value says it wasn't saved, trust that it's still dominating.
5208	if (!value.getInt()) return value.getPointer();
5209
5210	// Otherwise, it should be an alloca instruction, as set up in save().
5211	auto alloca = cast<llvm::AllocaInst>(Val: value.getPointer());
5212	return CGF.Builder.CreateAlignedLoad(Ty: alloca->getAllocatedType(), Ptr: alloca,
5213	Align: alloca->getAlign());
5214	}
5215
5216	} // end namespace CodeGen
5217
5218	// Map the LangOption for floating point exception behavior into
5219	// the corresponding enum in the IR.
5220	llvm::fp::ExceptionBehavior
5221	ToConstrainedExceptMD(LangOptions::FPExceptionModeKind Kind);
5222	} // end namespace clang
5223
5224	#endif
5225