1	//===----- CGOpenMPRuntime.h - Interface to OpenMP Runtimes -----*- C++ -*-===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This provides a class for OpenMP runtime code generation.
10	//
11	//===----------------------------------------------------------------------===//
12
13	#ifndef LLVM_CLANG_LIB_CODEGEN_CGOPENMPRUNTIME_H
14	#define LLVM_CLANG_LIB_CODEGEN_CGOPENMPRUNTIME_H
15
16	#include "CGValue.h"
17	#include "clang/AST/DeclOpenMP.h"
18	#include "clang/AST/GlobalDecl.h"
19	#include "clang/AST/Type.h"
20	#include "clang/Basic/OpenMPKinds.h"
21	#include "clang/Basic/SourceLocation.h"
22	#include "llvm/ADT/DenseMap.h"
23	#include "llvm/ADT/PointerIntPair.h"
24	#include "llvm/ADT/SmallPtrSet.h"
25	#include "llvm/ADT/StringMap.h"
26	#include "llvm/ADT/StringSet.h"
27	#include "llvm/Frontend/OpenMP/OMPConstants.h"
28	#include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
29	#include "llvm/IR/Function.h"
30	#include "llvm/IR/ValueHandle.h"
31	#include "llvm/Support/AtomicOrdering.h"
32
33	namespace llvm {
34	class ArrayType;
35	class Constant;
36	class FunctionType;
37	class GlobalVariable;
38	class Type;
39	class Value;
40	class OpenMPIRBuilder;
41	} // namespace llvm
42
43	namespace clang {
44	class Expr;
45	class OMPDependClause;
46	class OMPExecutableDirective;
47	class OMPLoopDirective;
48	class VarDecl;
49	class OMPDeclareReductionDecl;
50
51	namespace CodeGen {
52	class Address;
53	class CodeGenFunction;
54	class CodeGenModule;
55
56	/// A basic class for pre|post-action for advanced codegen sequence for OpenMP
57	/// region.
58	class PrePostActionTy {
59	public:
60	explicit PrePostActionTy() {}
61	virtual void Enter(CodeGenFunction &CGF) {}
62	virtual void Exit(CodeGenFunction &CGF) {}
63	virtual ~PrePostActionTy() {}
64	};
65
66	/// Class provides a way to call simple version of codegen for OpenMP region, or
67	/// an advanced with possible pre|post-actions in codegen.
68	class RegionCodeGenTy final {
69	intptr_t CodeGen;
70	typedef void (*CodeGenTy)(intptr_t, CodeGenFunction &, PrePostActionTy &);
71	CodeGenTy Callback;
72	mutable PrePostActionTy *PrePostAction;
73	RegionCodeGenTy() = delete;
74	template <typename Callable>
75	static void CallbackFn(intptr_t CodeGen, CodeGenFunction &CGF,
76	PrePostActionTy &Action) {
77	return (*reinterpret_cast<Callable *>(CodeGen))(CGF, Action);
78	}
79
80	public:
81	template <typename Callable>
82	RegionCodeGenTy(
83	Callable &&CodeGen,
84	std::enable_if_t<!std::is_same<std::remove_reference_t<Callable>,
85	RegionCodeGenTy>::value> * = nullptr)
86	: CodeGen(reinterpret_cast<intptr_t>(&CodeGen)),
87	Callback(CallbackFn<std::remove_reference_t<Callable>>),
88	PrePostAction(nullptr) {}
89	void setAction(PrePostActionTy &Action) const { PrePostAction = &Action; }
90	void operator()(CodeGenFunction &CGF) const;
91	};
92
93	struct OMPTaskDataTy final {
94	SmallVector<const Expr *, 4> PrivateVars;
95	SmallVector<const Expr *, 4> PrivateCopies;
96	SmallVector<const Expr *, 4> FirstprivateVars;
97	SmallVector<const Expr *, 4> FirstprivateCopies;
98	SmallVector<const Expr *, 4> FirstprivateInits;
99	SmallVector<const Expr *, 4> LastprivateVars;
100	SmallVector<const Expr *, 4> LastprivateCopies;
101	SmallVector<const Expr *, 4> ReductionVars;
102	SmallVector<const Expr *, 4> ReductionOrigs;
103	SmallVector<const Expr *, 4> ReductionCopies;
104	SmallVector<const Expr *, 4> ReductionOps;
105	SmallVector<CanonicalDeclPtr<const VarDecl>, 4> PrivateLocals;
106	struct DependData {
107	OpenMPDependClauseKind DepKind = OMPC_DEPEND_unknown;
108	const Expr *IteratorExpr = nullptr;
109	SmallVector<const Expr *, 4> DepExprs;
110	explicit DependData() = default;
111	DependData(OpenMPDependClauseKind DepKind, const Expr *IteratorExpr)
112	: DepKind(DepKind), IteratorExpr(IteratorExpr) {}
113	};
114	SmallVector<DependData, 4> Dependences;
115	llvm::PointerIntPair<llvm::Value *, 1, bool> Final;
116	llvm::PointerIntPair<llvm::Value *, 1, bool> Schedule;
117	llvm::PointerIntPair<llvm::Value *, 1, bool> Priority;
118	llvm::Value *Reductions = nullptr;
119	unsigned NumberOfParts = 0;
120	bool Tied = true;
121	bool Nogroup = false;
122	bool IsReductionWithTaskMod = false;
123	bool IsWorksharingReduction = false;
124	bool HasNowaitClause = false;
125	bool HasModifier = false;
126	};
127
128	/// Class intended to support codegen of all kind of the reduction clauses.
129	class ReductionCodeGen {
130	private:
131	/// Data required for codegen of reduction clauses.
132	struct ReductionData {
133	/// Reference to the item shared between tasks to reduce into.
134	const Expr *Shared = nullptr;
135	/// Reference to the original item.
136	const Expr *Ref = nullptr;
137	/// Helper expression for generation of private copy.
138	const Expr *Private = nullptr;
139	/// Helper expression for generation reduction operation.
140	const Expr *ReductionOp = nullptr;
141	ReductionData(const Expr *Shared, const Expr *Ref, const Expr *Private,
142	const Expr *ReductionOp)
143	: Shared(Shared), Ref(Ref), Private(Private), ReductionOp(ReductionOp) {
144	}
145	};
146	/// List of reduction-based clauses.
147	SmallVector<ReductionData, 4> ClausesData;
148
149	/// List of addresses of shared variables/expressions.
150	SmallVector<std::pair<LValue, LValue>, 4> SharedAddresses;
151	/// List of addresses of original variables/expressions.
152	SmallVector<std::pair<LValue, LValue>, 4> OrigAddresses;
153	/// Sizes of the reduction items in chars.
154	SmallVector<std::pair<llvm::Value *, llvm::Value *>, 4> Sizes;
155	/// Base declarations for the reduction items.
156	SmallVector<const VarDecl *, 4> BaseDecls;
157
158	/// Emits lvalue for shared expression.
159	LValue emitSharedLValue(CodeGenFunction &CGF, const Expr *E);
160	/// Emits upper bound for shared expression (if array section).
161	LValue emitSharedLValueUB(CodeGenFunction &CGF, const Expr *E);
162	/// Performs aggregate initialization.
163	/// \param N Number of reduction item in the common list.
164	/// \param PrivateAddr Address of the corresponding private item.
165	/// \param SharedAddr Address of the original shared variable.
166	/// \param DRD Declare reduction construct used for reduction item.
167	void emitAggregateInitialization(CodeGenFunction &CGF, unsigned N,
168	Address PrivateAddr, Address SharedAddr,
169	const OMPDeclareReductionDecl *DRD);
170
171	public:
172	ReductionCodeGen(ArrayRef<const Expr *> Shareds, ArrayRef<const Expr *> Origs,
173	ArrayRef<const Expr *> Privates,
174	ArrayRef<const Expr *> ReductionOps);
175	/// Emits lvalue for the shared and original reduction item.
176	/// \param N Number of the reduction item.
177	void emitSharedOrigLValue(CodeGenFunction &CGF, unsigned N);
178	/// Emits the code for the variable-modified type, if required.
179	/// \param N Number of the reduction item.
180	void emitAggregateType(CodeGenFunction &CGF, unsigned N);
181	/// Emits the code for the variable-modified type, if required.
182	/// \param N Number of the reduction item.
183	/// \param Size Size of the type in chars.
184	void emitAggregateType(CodeGenFunction &CGF, unsigned N, llvm::Value *Size);
185	/// Performs initialization of the private copy for the reduction item.
186	/// \param N Number of the reduction item.
187	/// \param PrivateAddr Address of the corresponding private item.
188	/// \param DefaultInit Default initialization sequence that should be
189	/// performed if no reduction specific initialization is found.
190	/// \param SharedAddr Address of the original shared variable.
191	void
192	emitInitialization(CodeGenFunction &CGF, unsigned N, Address PrivateAddr,
193	Address SharedAddr,
194	llvm::function_ref<bool(CodeGenFunction &)> DefaultInit);
195	/// Returns true if the private copy requires cleanups.
196	bool needCleanups(unsigned N);
197	/// Emits cleanup code for the reduction item.
198	/// \param N Number of the reduction item.
199	/// \param PrivateAddr Address of the corresponding private item.
200	void emitCleanups(CodeGenFunction &CGF, unsigned N, Address PrivateAddr);
201	/// Adjusts \p PrivatedAddr for using instead of the original variable
202	/// address in normal operations.
203	/// \param N Number of the reduction item.
204	/// \param PrivateAddr Address of the corresponding private item.
205	Address adjustPrivateAddress(CodeGenFunction &CGF, unsigned N,
206	Address PrivateAddr);
207	/// Returns LValue for the reduction item.
208	LValue getSharedLValue(unsigned N) const { return SharedAddresses[N].first; }
209	/// Returns LValue for the original reduction item.
210	LValue getOrigLValue(unsigned N) const { return OrigAddresses[N].first; }
211	/// Returns the size of the reduction item (in chars and total number of
212	/// elements in the item), or nullptr, if the size is a constant.
213	std::pair<llvm::Value *, llvm::Value *> getSizes(unsigned N) const {
214	return Sizes[N];
215	}
216	/// Returns the base declaration of the reduction item.
217	const VarDecl *getBaseDecl(unsigned N) const { return BaseDecls[N]; }
218	/// Returns the base declaration of the reduction item.
219	const Expr *getRefExpr(unsigned N) const { return ClausesData[N].Ref; }
220	/// Returns true if the initialization of the reduction item uses initializer
221	/// from declare reduction construct.
222	bool usesReductionInitializer(unsigned N) const;
223	/// Return the type of the private item.
224	QualType getPrivateType(unsigned N) const {
225	return cast<VarDecl>(Val: cast<DeclRefExpr>(Val: ClausesData[N].Private)->getDecl())
226	->getType();
227	}
228	};
229
230	class CGOpenMPRuntime {
231	public:
232	/// Allows to disable automatic handling of functions used in target regions
233	/// as those marked as `omp declare target`.
234	class DisableAutoDeclareTargetRAII {
235	CodeGenModule &CGM;
236	bool SavedShouldMarkAsGlobal = false;
237
238	public:
239	DisableAutoDeclareTargetRAII(CodeGenModule &CGM);
240	~DisableAutoDeclareTargetRAII();
241	};
242
243	/// Manages list of nontemporal decls for the specified directive.
244	class NontemporalDeclsRAII {
245	CodeGenModule &CGM;
246	const bool NeedToPush;
247
248	public:
249	NontemporalDeclsRAII(CodeGenModule &CGM, const OMPLoopDirective &S);
250	~NontemporalDeclsRAII();
251	};
252
253	/// Manages list of nontemporal decls for the specified directive.
254	class UntiedTaskLocalDeclsRAII {
255	CodeGenModule &CGM;
256	const bool NeedToPush;
257
258	public:
259	UntiedTaskLocalDeclsRAII(
260	CodeGenFunction &CGF,
261	const llvm::MapVector<CanonicalDeclPtr<const VarDecl>,
262	std::pair<Address, Address>> &LocalVars);
263	~UntiedTaskLocalDeclsRAII();
264	};
265
266	/// Maps the expression for the lastprivate variable to the global copy used
267	/// to store new value because original variables are not mapped in inner
268	/// parallel regions. Only private copies are captured but we need also to
269	/// store private copy in shared address.
270	/// Also, stores the expression for the private loop counter and it
271	/// threaprivate name.
272	struct LastprivateConditionalData {
273	llvm::MapVector<CanonicalDeclPtr<const Decl>, SmallString<16>>
274	DeclToUniqueName;
275	LValue IVLVal;
276	llvm::Function *Fn = nullptr;
277	bool Disabled = false;
278	};
279	/// Manages list of lastprivate conditional decls for the specified directive.
280	class LastprivateConditionalRAII {
281	enum class ActionToDo {
282	DoNotPush,
283	PushAsLastprivateConditional,
284	DisableLastprivateConditional,
285	};
286	CodeGenModule &CGM;
287	ActionToDo Action = ActionToDo::DoNotPush;
288
289	/// Check and try to disable analysis of inner regions for changes in
290	/// lastprivate conditional.
291	void tryToDisableInnerAnalysis(const OMPExecutableDirective &S,
292	llvm::DenseSet<CanonicalDeclPtr<const Decl>>
293	&NeedToAddForLPCsAsDisabled) const;
294
295	LastprivateConditionalRAII(CodeGenFunction &CGF,
296	const OMPExecutableDirective &S);
297
298	public:
299	explicit LastprivateConditionalRAII(CodeGenFunction &CGF,
300	const OMPExecutableDirective &S,
301	LValue IVLVal);
302	static LastprivateConditionalRAII disable(CodeGenFunction &CGF,
303	const OMPExecutableDirective &S);
304	~LastprivateConditionalRAII();
305	};
306
307	llvm::OpenMPIRBuilder &getOMPBuilder() { return OMPBuilder; }
308
309	protected:
310	CodeGenModule &CGM;
311
312	/// An OpenMP-IR-Builder instance.
313	llvm::OpenMPIRBuilder OMPBuilder;
314
315	/// Helper to determine the min/max number of threads/teams for \p D.
316	void computeMinAndMaxThreadsAndTeams(
317	const OMPExecutableDirective &D, CodeGenFunction &CGF,
318	llvm::OpenMPIRBuilder::TargetKernelDefaultAttrs &Attrs);
319
320	/// Helper to emit outlined function for 'target' directive.
321	/// \param D Directive to emit.
322	/// \param ParentName Name of the function that encloses the target region.
323	/// \param OutlinedFn Outlined function value to be defined by this call.
324	/// \param OutlinedFnID Outlined function ID value to be defined by this call.
325	/// \param IsOffloadEntry True if the outlined function is an offload entry.
326	/// \param CodeGen Lambda codegen specific to an accelerator device.
327	/// An outlined function may not be an entry if, e.g. the if clause always
328	/// evaluates to false.
329	virtual void emitTargetOutlinedFunctionHelper(const OMPExecutableDirective &D,
330	StringRef ParentName,
331	llvm::Function *&OutlinedFn,
332	llvm::Constant *&OutlinedFnID,
333	bool IsOffloadEntry,
334	const RegionCodeGenTy &CodeGen);
335
336	/// Returns pointer to ident_t type.
337	llvm::Type *getIdentTyPointerTy();
338
339	/// Gets thread id value for the current thread.
340	///
341	llvm::Value *getThreadID(CodeGenFunction &CGF, SourceLocation Loc);
342
343	/// Get the function name of an outlined region.
344	std::string getOutlinedHelperName(StringRef Name) const;
345	std::string getOutlinedHelperName(CodeGenFunction &CGF) const;
346
347	/// Get the function name of a reduction function.
348	std::string getReductionFuncName(StringRef Name) const;
349
350	/// Emits \p Callee function call with arguments \p Args with location \p Loc.
351	void emitCall(CodeGenFunction &CGF, SourceLocation Loc,
352	llvm::FunctionCallee Callee,
353	ArrayRef<llvm::Value *> Args = {}) const;
354
355	/// Emits address of the word in a memory where current thread id is
356	/// stored.
357	virtual Address emitThreadIDAddress(CodeGenFunction &CGF, SourceLocation Loc);
358
359	void setLocThreadIdInsertPt(CodeGenFunction &CGF,
360	bool AtCurrentPoint = false);
361	void clearLocThreadIdInsertPt(CodeGenFunction &CGF);
362
363	/// Check if the default location must be constant.
364	/// Default is false to support OMPT/OMPD.
365	virtual bool isDefaultLocationConstant() const { return false; }
366
367	/// Returns additional flags that can be stored in reserved_2 field of the
368	/// default location.
369	virtual unsigned getDefaultLocationReserved2Flags() const { return 0; }
370
371	/// Returns default flags for the barriers depending on the directive, for
372	/// which this barier is going to be emitted.
373	static unsigned getDefaultFlagsForBarriers(OpenMPDirectiveKind Kind);
374
375	/// Get the LLVM type for the critical name.
376	llvm::ArrayType *getKmpCriticalNameTy() const {return KmpCriticalNameTy;}
377
378	/// Returns corresponding lock object for the specified critical region
379	/// name. If the lock object does not exist it is created, otherwise the
380	/// reference to the existing copy is returned.
381	/// \param CriticalName Name of the critical region.
382	///
383	llvm::Value *getCriticalRegionLock(StringRef CriticalName);
384
385	protected:
386	/// Map for SourceLocation and OpenMP runtime library debug locations.
387	typedef llvm::DenseMap<SourceLocation, llvm::Value *> OpenMPDebugLocMapTy;
388	OpenMPDebugLocMapTy OpenMPDebugLocMap;
389	/// Stores debug location and ThreadID for the function.
390	struct DebugLocThreadIdTy {
391	llvm::Value *DebugLoc;
392	llvm::Value *ThreadID;
393	/// Insert point for the service instructions.
394	llvm::AssertingVH<llvm::Instruction> ServiceInsertPt = nullptr;
395	};
396	/// Map of local debug location, ThreadId and functions.
397	typedef llvm::DenseMap<llvm::Function *, DebugLocThreadIdTy>
398	OpenMPLocThreadIDMapTy;
399	OpenMPLocThreadIDMapTy OpenMPLocThreadIDMap;
400	/// Map of UDRs and corresponding combiner/initializer.
401	typedef llvm::DenseMap<const OMPDeclareReductionDecl *,
402	std::pair<llvm::Function *, llvm::Function *>>
403	UDRMapTy;
404	UDRMapTy UDRMap;
405	/// Map of functions and locally defined UDRs.
406	typedef llvm::DenseMap<llvm::Function *,
407	SmallVector<const OMPDeclareReductionDecl *, 4>>
408	FunctionUDRMapTy;
409	FunctionUDRMapTy FunctionUDRMap;
410	/// Map from the user-defined mapper declaration to its corresponding
411	/// functions.
412	llvm::DenseMap<const OMPDeclareMapperDecl *, llvm::Function *> UDMMap;
413	/// Map of functions and their local user-defined mappers.
414	using FunctionUDMMapTy =
415	llvm::DenseMap<llvm::Function *,
416	SmallVector<const OMPDeclareMapperDecl *, 4>>;
417	FunctionUDMMapTy FunctionUDMMap;
418	/// Maps local variables marked as lastprivate conditional to their internal
419	/// types.
420	llvm::DenseMap<llvm::Function *,
421	llvm::DenseMap<CanonicalDeclPtr<const Decl>,
422	std::tuple<QualType, const FieldDecl *,
423	const FieldDecl *, LValue>>>
424	LastprivateConditionalToTypes;
425	/// Maps function to the position of the untied task locals stack.
426	llvm::DenseMap<llvm::Function *, unsigned> FunctionToUntiedTaskStackMap;
427	/// Type kmp_critical_name, originally defined as typedef kmp_int32
428	/// kmp_critical_name[8];
429	llvm::ArrayType *KmpCriticalNameTy;
430	/// An ordered map of auto-generated variables to their unique names.
431	/// It stores variables with the following names: 1) ".gomp_critical_user_" +
432	/// <critical_section_name> + ".var" for "omp critical" directives; 2)
433	/// <mangled_name_for_global_var> + ".cache." for cache for threadprivate
434	/// variables.
435	llvm::StringMap<llvm::AssertingVH<llvm::GlobalVariable>,
436	llvm::BumpPtrAllocator> InternalVars;
437	/// Type typedef kmp_int32 (* kmp_routine_entry_t)(kmp_int32, void *);
438	llvm::Type *KmpRoutineEntryPtrTy = nullptr;
439	QualType KmpRoutineEntryPtrQTy;
440	/// Type typedef struct kmp_task {
441	/// void * shareds; /**< pointer to block of pointers to
442	/// shared vars */
443	/// kmp_routine_entry_t routine; /**< pointer to routine to call for
444	/// executing task */
445	/// kmp_int32 part_id; /**< part id for the task */
446	/// kmp_routine_entry_t destructors; /* pointer to function to invoke
447	/// deconstructors of firstprivate C++ objects */
448	/// } kmp_task_t;
449	QualType KmpTaskTQTy;
450	/// Saved kmp_task_t for task directive.
451	QualType SavedKmpTaskTQTy;
452	/// Saved kmp_task_t for taskloop-based directive.
453	QualType SavedKmpTaskloopTQTy;
454	/// Type typedef struct kmp_depend_info {
455	/// kmp_intptr_t base_addr;
456	/// size_t len;
457	/// struct {
458	/// bool in:1;
459	/// bool out:1;
460	/// } flags;
461	/// } kmp_depend_info_t;
462	QualType KmpDependInfoTy;
463	/// Type typedef struct kmp_task_affinity_info {
464	/// kmp_intptr_t base_addr;
465	/// size_t len;
466	/// struct {
467	/// bool flag1 : 1;
468	/// bool flag2 : 1;
469	/// kmp_int32 reserved : 30;
470	/// } flags;
471	/// } kmp_task_affinity_info_t;
472	QualType KmpTaskAffinityInfoTy;
473	/// struct kmp_dim { // loop bounds info casted to kmp_int64
474	/// kmp_int64 lo; // lower
475	/// kmp_int64 up; // upper
476	/// kmp_int64 st; // stride
477	/// };
478	QualType KmpDimTy;
479
480	bool ShouldMarkAsGlobal = true;
481	/// List of the emitted declarations.
482	llvm::DenseSet<CanonicalDeclPtr<const Decl>> AlreadyEmittedTargetDecls;
483	/// List of the global variables with their addresses that should not be
484	/// emitted for the target.
485	llvm::StringMap<llvm::WeakTrackingVH> EmittedNonTargetVariables;
486
487	/// List of variables that can become declare target implicitly and, thus,
488	/// must be emitted.
489	llvm::SmallDenseSet<const VarDecl *> DeferredGlobalVariables;
490
491	using NontemporalDeclsSet = llvm::SmallDenseSet<CanonicalDeclPtr<const Decl>>;
492	/// Stack for list of declarations in current context marked as nontemporal.
493	/// The set is the union of all current stack elements.
494	llvm::SmallVector<NontemporalDeclsSet, 4> NontemporalDeclsStack;
495
496	using UntiedLocalVarsAddressesMap =
497	llvm::MapVector<CanonicalDeclPtr<const VarDecl>,
498	std::pair<Address, Address>>;
499	llvm::SmallVector<UntiedLocalVarsAddressesMap, 4> UntiedLocalVarsStack;
500
501	/// Stack for list of addresses of declarations in current context marked as
502	/// lastprivate conditional. The set is the union of all current stack
503	/// elements.
504	llvm::SmallVector<LastprivateConditionalData, 4> LastprivateConditionalStack;
505
506	/// Flag for keeping track of weather a requires unified_shared_memory
507	/// directive is present.
508	bool HasRequiresUnifiedSharedMemory = false;
509
510	/// Atomic ordering from the omp requires directive.
511	llvm::AtomicOrdering RequiresAtomicOrdering = llvm::AtomicOrdering::Monotonic;
512
513	/// Flag for keeping track of weather a target region has been emitted.
514	bool HasEmittedTargetRegion = false;
515
516	/// Flag for keeping track of weather a device routine has been emitted.
517	/// Device routines are specific to the
518	bool HasEmittedDeclareTargetRegion = false;
519
520	/// Start scanning from statement \a S and emit all target regions
521	/// found along the way.
522	/// \param S Starting statement.
523	/// \param ParentName Name of the function declaration that is being scanned.
524	void scanForTargetRegionsFunctions(const Stmt *S, StringRef ParentName);
525
526	/// Build type kmp_routine_entry_t (if not built yet).
527	void emitKmpRoutineEntryT(QualType KmpInt32Ty);
528
529	/// If the specified mangled name is not in the module, create and
530	/// return threadprivate cache object. This object is a pointer's worth of
531	/// storage that's reserved for use by the OpenMP runtime.
532	/// \param VD Threadprivate variable.
533	/// \return Cache variable for the specified threadprivate.
534	llvm::Constant *getOrCreateThreadPrivateCache(const VarDecl *VD);
535
536	/// Set of threadprivate variables with the generated initializer.
537	llvm::StringSet<> ThreadPrivateWithDefinition;
538
539	/// Set of declare target variables with the generated initializer.
540	llvm::StringSet<> DeclareTargetWithDefinition;
541
542	/// Emits initialization code for the threadprivate variables.
543	/// \param VDAddr Address of the global variable \a VD.
544	/// \param Ctor Pointer to a global init function for \a VD.
545	/// \param CopyCtor Pointer to a global copy function for \a VD.
546	/// \param Dtor Pointer to a global destructor function for \a VD.
547	/// \param Loc Location of threadprivate declaration.
548	void emitThreadPrivateVarInit(CodeGenFunction &CGF, Address VDAddr,
549	llvm::Value *Ctor, llvm::Value *CopyCtor,
550	llvm::Value *Dtor, SourceLocation Loc);
551
552	struct TaskResultTy {
553	llvm::Value *NewTask = nullptr;
554	llvm::Function *TaskEntry = nullptr;
555	llvm::Value *NewTaskNewTaskTTy = nullptr;
556	LValue TDBase;
557	const RecordDecl *KmpTaskTQTyRD = nullptr;
558	llvm::Value *TaskDupFn = nullptr;
559	};
560	/// Emit task region for the task directive. The task region is emitted in
561	/// several steps:
562	/// 1. Emit a call to kmp_task_t *__kmpc_omp_task_alloc(ident_t *, kmp_int32
563	/// gtid, kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
564	/// kmp_routine_entry_t *task_entry). Here task_entry is a pointer to the
565	/// function:
566	/// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) {
567	/// TaskFunction(gtid, tt->part_id, tt->shareds);
568	/// return 0;
569	/// }
570	/// 2. Copy a list of shared variables to field shareds of the resulting
571	/// structure kmp_task_t returned by the previous call (if any).
572	/// 3. Copy a pointer to destructions function to field destructions of the
573	/// resulting structure kmp_task_t.
574	/// \param D Current task directive.
575	/// \param TaskFunction An LLVM function with type void (*)(i32 /*gtid*/, i32
576	/// /*part_id*/, captured_struct */*__context*/);
577	/// \param SharedsTy A type which contains references the shared variables.
578	/// \param Shareds Context with the list of shared variables from the \p
579	/// TaskFunction.
580	/// \param Data Additional data for task generation like tiednsee, final
581	/// state, list of privates etc.
582	TaskResultTy emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc,
583	const OMPExecutableDirective &D,
584	llvm::Function *TaskFunction, QualType SharedsTy,
585	Address Shareds, const OMPTaskDataTy &Data);
586
587	/// Emit update for lastprivate conditional data.
588	void emitLastprivateConditionalUpdate(CodeGenFunction &CGF, LValue IVLVal,
589	StringRef UniqueDeclName, LValue LVal,
590	SourceLocation Loc);
591
592	/// Returns the number of the elements and the address of the depobj
593	/// dependency array.
594	/// \return Number of elements in depobj array and the pointer to the array of
595	/// dependencies.
596	std::pair<llvm::Value *, LValue> getDepobjElements(CodeGenFunction &CGF,
597	LValue DepobjLVal,
598	SourceLocation Loc);
599
600	SmallVector<llvm::Value *, 4>
601	emitDepobjElementsSizes(CodeGenFunction &CGF, QualType &KmpDependInfoTy,
602	const OMPTaskDataTy::DependData &Data);
603
604	void emitDepobjElements(CodeGenFunction &CGF, QualType &KmpDependInfoTy,
605	LValue PosLVal, const OMPTaskDataTy::DependData &Data,
606	Address DependenciesArray);
607
608	public:
609	explicit CGOpenMPRuntime(CodeGenModule &CGM);
610	virtual ~CGOpenMPRuntime() {}
611	virtual void clear();
612
613	/// Emits object of ident_t type with info for source location.
614	/// \param Flags Flags for OpenMP location.
615	/// \param EmitLoc emit source location with debug-info is off.
616	///
617	llvm::Value *emitUpdateLocation(CodeGenFunction &CGF, SourceLocation Loc,
618	unsigned Flags = 0, bool EmitLoc = false);
619
620	/// Emit the number of teams for a target directive. Inspect the num_teams
621	/// clause associated with a teams construct combined or closely nested
622	/// with the target directive.
623	///
624	/// Emit a team of size one for directives such as 'target parallel' that
625	/// have no associated teams construct.
626	///
627	/// Otherwise, return nullptr.
628	const Expr *getNumTeamsExprForTargetDirective(CodeGenFunction &CGF,
629	const OMPExecutableDirective &D,
630	int32_t &MinTeamsVal,
631	int32_t &MaxTeamsVal);
632	llvm::Value *emitNumTeamsForTargetDirective(CodeGenFunction &CGF,
633	const OMPExecutableDirective &D);
634
635	/// Check for a number of threads upper bound constant value (stored in \p
636	/// UpperBound), or expression (returned). If the value is conditional (via an
637	/// if-clause), store the condition in \p CondExpr. Similarly, a potential
638	/// thread limit expression is stored in \p ThreadLimitExpr. If \p
639	/// UpperBoundOnly is true, no expression evaluation is perfomed.
640	const Expr *getNumThreadsExprForTargetDirective(
641	CodeGenFunction &CGF, const OMPExecutableDirective &D,
642	int32_t &UpperBound, bool UpperBoundOnly,
643	llvm::Value **CondExpr = nullptr, const Expr **ThreadLimitExpr = nullptr);
644
645	/// Emit an expression that denotes the number of threads a target region
646	/// shall use. Will generate "i32 0" to allow the runtime to choose.
647	llvm::Value *
648	emitNumThreadsForTargetDirective(CodeGenFunction &CGF,
649	const OMPExecutableDirective &D);
650
651	/// Return the trip count of loops associated with constructs / 'target teams
652	/// distribute' and 'teams distribute parallel for'. \param SizeEmitter Emits
653	/// the int64 value for the number of iterations of the associated loop.
654	llvm::Value *emitTargetNumIterationsCall(
655	CodeGenFunction &CGF, const OMPExecutableDirective &D,
656	llvm::function_ref<llvm::Value *(CodeGenFunction &CGF,
657	const OMPLoopDirective &D)>
658	SizeEmitter);
659
660	/// Returns true if the current target is a GPU.
661	virtual bool isGPU() const { return false; }
662
663	/// Check if the variable length declaration is delayed:
664	virtual bool isDelayedVariableLengthDecl(CodeGenFunction &CGF,
665	const VarDecl *VD) const {
666	return false;
667	};
668
669	/// Get call to __kmpc_alloc_shared
670	virtual std::pair<llvm::Value *, llvm::Value *>
671	getKmpcAllocShared(CodeGenFunction &CGF, const VarDecl *VD) {
672	llvm_unreachable("not implemented");
673	}
674
675	/// Get call to __kmpc_free_shared
676	virtual void getKmpcFreeShared(
677	CodeGenFunction &CGF,
678	const std::pair<llvm::Value *, llvm::Value *> &AddrSizePair) {
679	llvm_unreachable("not implemented");
680	}
681
682	/// Emits code for OpenMP 'if' clause using specified \a CodeGen
683	/// function. Here is the logic:
684	/// if (Cond) {
685	/// ThenGen();
686	/// } else {
687	/// ElseGen();
688	/// }
689	void emitIfClause(CodeGenFunction &CGF, const Expr *Cond,
690	const RegionCodeGenTy &ThenGen,
691	const RegionCodeGenTy &ElseGen);
692
693	/// Checks if the \p Body is the \a CompoundStmt and returns its child
694	/// statement iff there is only one that is not evaluatable at the compile
695	/// time.
696	static const Stmt *getSingleCompoundChild(ASTContext &Ctx, const Stmt *Body);
697
698	/// Get the platform-specific name separator.
699	std::string getName(ArrayRef<StringRef> Parts) const;
700
701	/// Emit code for the specified user defined reduction construct.
702	virtual void emitUserDefinedReduction(CodeGenFunction *CGF,
703	const OMPDeclareReductionDecl *D);
704	/// Get combiner/initializer for the specified user-defined reduction, if any.
705	virtual std::pair<llvm::Function *, llvm::Function *>
706	getUserDefinedReduction(const OMPDeclareReductionDecl *D);
707
708	/// Emit the function for the user defined mapper construct.
709	void emitUserDefinedMapper(const OMPDeclareMapperDecl *D,
710	CodeGenFunction *CGF = nullptr);
711	/// Get the function for the specified user-defined mapper. If it does not
712	/// exist, create one.
713	llvm::Function *
714	getOrCreateUserDefinedMapperFunc(const OMPDeclareMapperDecl *D);
715
716	/// Emits outlined function for the specified OpenMP parallel directive
717	/// \a D. This outlined function has type void(*)(kmp_int32 *ThreadID,
718	/// kmp_int32 BoundID, struct context_vars*).
719	/// \param CGF Reference to current CodeGenFunction.
720	/// \param D OpenMP directive.
721	/// \param ThreadIDVar Variable for thread id in the current OpenMP region.
722	/// \param InnermostKind Kind of innermost directive (for simple directives it
723	/// is a directive itself, for combined - its innermost directive).
724	/// \param CodeGen Code generation sequence for the \a D directive.
725	virtual llvm::Function *emitParallelOutlinedFunction(
726	CodeGenFunction &CGF, const OMPExecutableDirective &D,
727	const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,
728	const RegionCodeGenTy &CodeGen);
729
730	/// Emits outlined function for the specified OpenMP teams directive
731	/// \a D. This outlined function has type void(*)(kmp_int32 *ThreadID,
732	/// kmp_int32 BoundID, struct context_vars*).
733	/// \param CGF Reference to current CodeGenFunction.
734	/// \param D OpenMP directive.
735	/// \param ThreadIDVar Variable for thread id in the current OpenMP region.
736	/// \param InnermostKind Kind of innermost directive (for simple directives it
737	/// is a directive itself, for combined - its innermost directive).
738	/// \param CodeGen Code generation sequence for the \a D directive.
739	virtual llvm::Function *emitTeamsOutlinedFunction(
740	CodeGenFunction &CGF, const OMPExecutableDirective &D,
741	const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,
742	const RegionCodeGenTy &CodeGen);
743
744	/// Emits outlined function for the OpenMP task directive \a D. This
745	/// outlined function has type void(*)(kmp_int32 ThreadID, struct task_t*
746	/// TaskT).
747	/// \param D OpenMP directive.
748	/// \param ThreadIDVar Variable for thread id in the current OpenMP region.
749	/// \param PartIDVar Variable for partition id in the current OpenMP untied
750	/// task region.
751	/// \param TaskTVar Variable for task_t argument.
752	/// \param InnermostKind Kind of innermost directive (for simple directives it
753	/// is a directive itself, for combined - its innermost directive).
754	/// \param CodeGen Code generation sequence for the \a D directive.
755	/// \param Tied true if task is generated for tied task, false otherwise.
756	/// \param NumberOfParts Number of parts in untied task. Ignored for tied
757	/// tasks.
758	///
759	virtual llvm::Function *emitTaskOutlinedFunction(
760	const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
761	const VarDecl *PartIDVar, const VarDecl *TaskTVar,
762	OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
763	bool Tied, unsigned &NumberOfParts);
764
765	/// Cleans up references to the objects in finished function.
766	///
767	virtual void functionFinished(CodeGenFunction &CGF);
768
769	/// Emits code for parallel or serial call of the \a OutlinedFn with
770	/// variables captured in a record which address is stored in \a
771	/// CapturedStruct.
772	/// \param OutlinedFn Outlined function to be run in parallel threads. Type of
773	/// this function is void(*)(kmp_int32 *, kmp_int32, struct context_vars*).
774	/// \param CapturedVars A pointer to the record with the references to
775	/// variables used in \a OutlinedFn function.
776	/// \param IfCond Condition in the associated 'if' clause, if it was
777	/// specified, nullptr otherwise.
778	/// \param NumThreads The value corresponding to the num_threads clause, if
779	/// any, or nullptr.
780	///
781	virtual void emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc,
782	llvm::Function *OutlinedFn,
783	ArrayRef<llvm::Value *> CapturedVars,
784	const Expr *IfCond, llvm::Value *NumThreads);
785
786	/// Emits a critical region.
787	/// \param CriticalName Name of the critical region.
788	/// \param CriticalOpGen Generator for the statement associated with the given
789	/// critical region.
790	/// \param Hint Value of the 'hint' clause (optional).
791	virtual void emitCriticalRegion(CodeGenFunction &CGF, StringRef CriticalName,
792	const RegionCodeGenTy &CriticalOpGen,
793	SourceLocation Loc,
794	const Expr *Hint = nullptr);
795
796	/// Emits a master region.
797	/// \param MasterOpGen Generator for the statement associated with the given
798	/// master region.
799	virtual void emitMasterRegion(CodeGenFunction &CGF,
800	const RegionCodeGenTy &MasterOpGen,
801	SourceLocation Loc);
802
803	/// Emits a masked region.
804	/// \param MaskedOpGen Generator for the statement associated with the given
805	/// masked region.
806	virtual void emitMaskedRegion(CodeGenFunction &CGF,
807	const RegionCodeGenTy &MaskedOpGen,
808	SourceLocation Loc,
809	const Expr *Filter = nullptr);
810
811	/// Emits code for a taskyield directive.
812	virtual void emitTaskyieldCall(CodeGenFunction &CGF, SourceLocation Loc);
813
814	/// Emit __kmpc_error call for error directive
815	/// extern void __kmpc_error(ident_t *loc, int severity, const char *message);
816	virtual void emitErrorCall(CodeGenFunction &CGF, SourceLocation Loc, Expr *ME,
817	bool IsFatal);
818
819	/// Emit a taskgroup region.
820	/// \param TaskgroupOpGen Generator for the statement associated with the
821	/// given taskgroup region.
822	virtual void emitTaskgroupRegion(CodeGenFunction &CGF,
823	const RegionCodeGenTy &TaskgroupOpGen,
824	SourceLocation Loc);
825
826	/// Emits a single region.
827	/// \param SingleOpGen Generator for the statement associated with the given
828	/// single region.
829	virtual void emitSingleRegion(CodeGenFunction &CGF,
830	const RegionCodeGenTy &SingleOpGen,
831	SourceLocation Loc,
832	ArrayRef<const Expr *> CopyprivateVars,
833	ArrayRef<const Expr *> DestExprs,
834	ArrayRef<const Expr *> SrcExprs,
835	ArrayRef<const Expr *> AssignmentOps);
836
837	/// Emit an ordered region.
838	/// \param OrderedOpGen Generator for the statement associated with the given
839	/// ordered region.
840	virtual void emitOrderedRegion(CodeGenFunction &CGF,
841	const RegionCodeGenTy &OrderedOpGen,
842	SourceLocation Loc, bool IsThreads);
843
844	/// Emit an implicit/explicit barrier for OpenMP threads.
845	/// \param Kind Directive for which this implicit barrier call must be
846	/// generated. Must be OMPD_barrier for explicit barrier generation.
847	/// \param EmitChecks true if need to emit checks for cancellation barriers.
848	/// \param ForceSimpleCall true simple barrier call must be emitted, false if
849	/// runtime class decides which one to emit (simple or with cancellation
850	/// checks).
851	///
852	virtual void emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc,
853	OpenMPDirectiveKind Kind,
854	bool EmitChecks = true,
855	bool ForceSimpleCall = false);
856
857	/// Check if the specified \a ScheduleKind is static non-chunked.
858	/// This kind of worksharing directive is emitted without outer loop.
859	/// \param ScheduleKind Schedule kind specified in the 'schedule' clause.
860	/// \param Chunked True if chunk is specified in the clause.
861	///
862	virtual bool isStaticNonchunked(OpenMPScheduleClauseKind ScheduleKind,
863	bool Chunked) const;
864
865	/// Check if the specified \a ScheduleKind is static non-chunked.
866	/// This kind of distribute directive is emitted without outer loop.
867	/// \param ScheduleKind Schedule kind specified in the 'dist_schedule' clause.
868	/// \param Chunked True if chunk is specified in the clause.
869	///
870	virtual bool isStaticNonchunked(OpenMPDistScheduleClauseKind ScheduleKind,
871	bool Chunked) const;
872
873	/// Check if the specified \a ScheduleKind is static chunked.
874	/// \param ScheduleKind Schedule kind specified in the 'schedule' clause.
875	/// \param Chunked True if chunk is specified in the clause.
876	///
877	virtual bool isStaticChunked(OpenMPScheduleClauseKind ScheduleKind,
878	bool Chunked) const;
879
880	/// Check if the specified \a ScheduleKind is static non-chunked.
881	/// \param ScheduleKind Schedule kind specified in the 'dist_schedule' clause.
882	/// \param Chunked True if chunk is specified in the clause.
883	///
884	virtual bool isStaticChunked(OpenMPDistScheduleClauseKind ScheduleKind,
885	bool Chunked) const;
886
887	/// Check if the specified \a ScheduleKind is dynamic.
888	/// This kind of worksharing directive is emitted without outer loop.
889	/// \param ScheduleKind Schedule Kind specified in the 'schedule' clause.
890	///
891	virtual bool isDynamic(OpenMPScheduleClauseKind ScheduleKind) const;
892
893	/// struct with the values to be passed to the dispatch runtime function
894	struct DispatchRTInput {
895	/// Loop lower bound
896	llvm::Value *LB = nullptr;
897	/// Loop upper bound
898	llvm::Value *UB = nullptr;
899	/// Chunk size specified using 'schedule' clause (nullptr if chunk
900	/// was not specified)
901	llvm::Value *Chunk = nullptr;
902	DispatchRTInput() = default;
903	DispatchRTInput(llvm::Value *LB, llvm::Value *UB, llvm::Value *Chunk)
904	: LB(LB), UB(UB), Chunk(Chunk) {}
905	};
906
907	/// Call the appropriate runtime routine to initialize it before start
908	/// of loop.
909
910	/// This is used for non static scheduled types and when the ordered
911	/// clause is present on the loop construct.
912	/// Depending on the loop schedule, it is necessary to call some runtime
913	/// routine before start of the OpenMP loop to get the loop upper / lower
914	/// bounds \a LB and \a UB and stride \a ST.
915	///
916	/// \param CGF Reference to current CodeGenFunction.
917	/// \param Loc Clang source location.
918	/// \param ScheduleKind Schedule kind, specified by the 'schedule' clause.
919	/// \param IVSize Size of the iteration variable in bits.
920	/// \param IVSigned Sign of the iteration variable.
921	/// \param Ordered true if loop is ordered, false otherwise.
922	/// \param DispatchValues struct containing llvm values for lower bound, upper
923	/// bound, and chunk expression.
924	/// For the default (nullptr) value, the chunk 1 will be used.
925	///
926	virtual void emitForDispatchInit(CodeGenFunction &CGF, SourceLocation Loc,
927	const OpenMPScheduleTy &ScheduleKind,
928	unsigned IVSize, bool IVSigned, bool Ordered,
929	const DispatchRTInput &DispatchValues);
930
931	/// This is used for non static scheduled types and when the ordered
932	/// clause is present on the loop construct.
933	///
934	/// \param CGF Reference to current CodeGenFunction.
935	/// \param Loc Clang source location.
936	///
937	virtual void emitForDispatchDeinit(CodeGenFunction &CGF, SourceLocation Loc);
938
939	/// Struct with the values to be passed to the static runtime function
940	struct StaticRTInput {
941	/// Size of the iteration variable in bits.
942	unsigned IVSize = 0;
943	/// Sign of the iteration variable.
944	bool IVSigned = false;
945	/// true if loop is ordered, false otherwise.
946	bool Ordered = false;
947	/// Address of the output variable in which the flag of the last iteration
948	/// is returned.
949	Address IL = Address::invalid();
950	/// Address of the output variable in which the lower iteration number is
951	/// returned.
952	Address LB = Address::invalid();
953	/// Address of the output variable in which the upper iteration number is
954	/// returned.
955	Address UB = Address::invalid();
956	/// Address of the output variable in which the stride value is returned
957	/// necessary to generated the static_chunked scheduled loop.
958	Address ST = Address::invalid();
959	/// Value of the chunk for the static_chunked scheduled loop. For the
960	/// default (nullptr) value, the chunk 1 will be used.
961	llvm::Value *Chunk = nullptr;
962	StaticRTInput(unsigned IVSize, bool IVSigned, bool Ordered, Address IL,
963	Address LB, Address UB, Address ST,
964	llvm::Value *Chunk = nullptr)
965	: IVSize(IVSize), IVSigned(IVSigned), Ordered(Ordered), IL(IL), LB(LB),
966	UB(UB), ST(ST), Chunk(Chunk) {}
967	};
968	/// Call the appropriate runtime routine to initialize it before start
969	/// of loop.
970	///
971	/// This is used only in case of static schedule, when the user did not
972	/// specify a ordered clause on the loop construct.
973	/// Depending on the loop schedule, it is necessary to call some runtime
974	/// routine before start of the OpenMP loop to get the loop upper / lower
975	/// bounds LB and UB and stride ST.
976	///
977	/// \param CGF Reference to current CodeGenFunction.
978	/// \param Loc Clang source location.
979	/// \param DKind Kind of the directive.
980	/// \param ScheduleKind Schedule kind, specified by the 'schedule' clause.
981	/// \param Values Input arguments for the construct.
982	///
983	virtual void emitForStaticInit(CodeGenFunction &CGF, SourceLocation Loc,
984	OpenMPDirectiveKind DKind,
985	const OpenMPScheduleTy &ScheduleKind,
986	const StaticRTInput &Values);
987
988	///
989	/// \param CGF Reference to current CodeGenFunction.
990	/// \param Loc Clang source location.
991	/// \param SchedKind Schedule kind, specified by the 'dist_schedule' clause.
992	/// \param Values Input arguments for the construct.
993	///
994	virtual void emitDistributeStaticInit(CodeGenFunction &CGF,
995	SourceLocation Loc,
996	OpenMPDistScheduleClauseKind SchedKind,
997	const StaticRTInput &Values);
998
999	/// Call the appropriate runtime routine to notify that we finished
1000	/// iteration of the ordered loop with the dynamic scheduling.
1001	///
1002	/// \param CGF Reference to current CodeGenFunction.
1003	/// \param Loc Clang source location.
1004	/// \param IVSize Size of the iteration variable in bits.
1005	/// \param IVSigned Sign of the iteration variable.
1006	///
1007	virtual void emitForOrderedIterationEnd(CodeGenFunction &CGF,
1008	SourceLocation Loc, unsigned IVSize,
1009	bool IVSigned);
1010
1011	/// Call the appropriate runtime routine to notify that we finished
1012	/// all the work with current loop.
1013	///
1014	/// \param CGF Reference to current CodeGenFunction.
1015	/// \param Loc Clang source location.
1016	/// \param DKind Kind of the directive for which the static finish is emitted.
1017	///
1018	virtual void emitForStaticFinish(CodeGenFunction &CGF, SourceLocation Loc,
1019	OpenMPDirectiveKind DKind);
1020
1021	/// Call __kmpc_dispatch_next(
1022	/// ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter,
1023	/// kmp_int[32|64] *p_lower, kmp_int[32|64] *p_upper,
1024	/// kmp_int[32|64] *p_stride);
1025	/// \param IVSize Size of the iteration variable in bits.
1026	/// \param IVSigned Sign of the iteration variable.
1027	/// \param IL Address of the output variable in which the flag of the
1028	/// last iteration is returned.
1029	/// \param LB Address of the output variable in which the lower iteration
1030	/// number is returned.
1031	/// \param UB Address of the output variable in which the upper iteration
1032	/// number is returned.
1033	/// \param ST Address of the output variable in which the stride value is
1034	/// returned.
1035	virtual llvm::Value *emitForNext(CodeGenFunction &CGF, SourceLocation Loc,
1036	unsigned IVSize, bool IVSigned,
1037	Address IL, Address LB,
1038	Address UB, Address ST);
1039
1040	/// Emits call to void __kmpc_push_num_threads(ident_t *loc, kmp_int32
1041	/// global_tid, kmp_int32 num_threads) to generate code for 'num_threads'
1042	/// clause.
1043	/// \param NumThreads An integer value of threads.
1044	virtual void emitNumThreadsClause(CodeGenFunction &CGF,
1045	llvm::Value *NumThreads,
1046	SourceLocation Loc);
1047
1048	/// Emit call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32
1049	/// global_tid, int proc_bind) to generate code for 'proc_bind' clause.
1050	virtual void emitProcBindClause(CodeGenFunction &CGF,
1051	llvm::omp::ProcBindKind ProcBind,
1052	SourceLocation Loc);
1053
1054	/// Returns address of the threadprivate variable for the current
1055	/// thread.
1056	/// \param VD Threadprivate variable.
1057	/// \param VDAddr Address of the global variable \a VD.
1058	/// \param Loc Location of the reference to threadprivate var.
1059	/// \return Address of the threadprivate variable for the current thread.
1060	virtual Address getAddrOfThreadPrivate(CodeGenFunction &CGF,
1061	const VarDecl *VD, Address VDAddr,
1062	SourceLocation Loc);
1063
1064	/// Returns the address of the variable marked as declare target with link
1065	/// clause OR as declare target with to clause and unified memory.
1066	virtual ConstantAddress getAddrOfDeclareTargetVar(const VarDecl *VD);
1067
1068	/// Emit a code for initialization of threadprivate variable. It emits
1069	/// a call to runtime library which adds initial value to the newly created
1070	/// threadprivate variable (if it is not constant) and registers destructor
1071	/// for the variable (if any).
1072	/// \param VD Threadprivate variable.
1073	/// \param VDAddr Address of the global variable \a VD.
1074	/// \param Loc Location of threadprivate declaration.
1075	/// \param PerformInit true if initialization expression is not constant.
1076	virtual llvm::Function *
1077	emitThreadPrivateVarDefinition(const VarDecl *VD, Address VDAddr,
1078	SourceLocation Loc, bool PerformInit,
1079	CodeGenFunction *CGF = nullptr);
1080
1081	/// Emit code for handling declare target functions in the runtime.
1082	/// \param FD Declare target function.
1083	/// \param Addr Address of the global \a FD.
1084	/// \param PerformInit true if initialization expression is not constant.
1085	virtual void emitDeclareTargetFunction(const FunctionDecl *FD,
1086	llvm::GlobalValue *GV);
1087
1088	/// Creates artificial threadprivate variable with name \p Name and type \p
1089	/// VarType.
1090	/// \param VarType Type of the artificial threadprivate variable.
1091	/// \param Name Name of the artificial threadprivate variable.
1092	virtual Address getAddrOfArtificialThreadPrivate(CodeGenFunction &CGF,
1093	QualType VarType,
1094	StringRef Name);
1095
1096	/// Emit flush of the variables specified in 'omp flush' directive.
1097	/// \param Vars List of variables to flush.
1098	virtual void emitFlush(CodeGenFunction &CGF, ArrayRef<const Expr *> Vars,
1099	SourceLocation Loc, llvm::AtomicOrdering AO);
1100
1101	/// Emit task region for the task directive. The task region is
1102	/// emitted in several steps:
1103	/// 1. Emit a call to kmp_task_t *__kmpc_omp_task_alloc(ident_t *, kmp_int32
1104	/// gtid, kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
1105	/// kmp_routine_entry_t *task_entry). Here task_entry is a pointer to the
1106	/// function:
1107	/// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) {
1108	/// TaskFunction(gtid, tt->part_id, tt->shareds);
1109	/// return 0;
1110	/// }
1111	/// 2. Copy a list of shared variables to field shareds of the resulting
1112	/// structure kmp_task_t returned by the previous call (if any).
1113	/// 3. Copy a pointer to destructions function to field destructions of the
1114	/// resulting structure kmp_task_t.
1115	/// 4. Emit a call to kmp_int32 __kmpc_omp_task(ident_t *, kmp_int32 gtid,
1116	/// kmp_task_t *new_task), where new_task is a resulting structure from
1117	/// previous items.
1118	/// \param D Current task directive.
1119	/// \param TaskFunction An LLVM function with type void (*)(i32 /*gtid*/, i32
1120	/// /*part_id*/, captured_struct */*__context*/);
1121	/// \param SharedsTy A type which contains references the shared variables.
1122	/// \param Shareds Context with the list of shared variables from the \p
1123	/// TaskFunction.
1124	/// \param IfCond Not a nullptr if 'if' clause was specified, nullptr
1125	/// otherwise.
1126	/// \param Data Additional data for task generation like tiednsee, final
1127	/// state, list of privates etc.
1128	virtual void emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc,
1129	const OMPExecutableDirective &D,
1130	llvm::Function *TaskFunction, QualType SharedsTy,
1131	Address Shareds, const Expr *IfCond,
1132	const OMPTaskDataTy &Data);
1133
1134	/// Emit task region for the taskloop directive. The taskloop region is
1135	/// emitted in several steps:
1136	/// 1. Emit a call to kmp_task_t *__kmpc_omp_task_alloc(ident_t *, kmp_int32
1137	/// gtid, kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
1138	/// kmp_routine_entry_t *task_entry). Here task_entry is a pointer to the
1139	/// function:
1140	/// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) {
1141	/// TaskFunction(gtid, tt->part_id, tt->shareds);
1142	/// return 0;
1143	/// }
1144	/// 2. Copy a list of shared variables to field shareds of the resulting
1145	/// structure kmp_task_t returned by the previous call (if any).
1146	/// 3. Copy a pointer to destructions function to field destructions of the
1147	/// resulting structure kmp_task_t.
1148	/// 4. Emit a call to void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t
1149	/// *task, int if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int
1150	/// nogroup, int sched, kmp_uint64 grainsize, void *task_dup ), where new_task
1151	/// is a resulting structure from
1152	/// previous items.
1153	/// \param D Current task directive.
1154	/// \param TaskFunction An LLVM function with type void (*)(i32 /*gtid*/, i32
1155	/// /*part_id*/, captured_struct */*__context*/);
1156	/// \param SharedsTy A type which contains references the shared variables.
1157	/// \param Shareds Context with the list of shared variables from the \p
1158	/// TaskFunction.
1159	/// \param IfCond Not a nullptr if 'if' clause was specified, nullptr
1160	/// otherwise.
1161	/// \param Data Additional data for task generation like tiednsee, final
1162	/// state, list of privates etc.
1163	virtual void emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc,
1164	const OMPLoopDirective &D,
1165	llvm::Function *TaskFunction,
1166	QualType SharedsTy, Address Shareds,
1167	const Expr *IfCond, const OMPTaskDataTy &Data);
1168
1169	/// Emit code for the directive that does not require outlining.
1170	///
1171	/// \param InnermostKind Kind of innermost directive (for simple directives it
1172	/// is a directive itself, for combined - its innermost directive).
1173	/// \param CodeGen Code generation sequence for the \a D directive.
1174	/// \param HasCancel true if region has inner cancel directive, false
1175	/// otherwise.
1176	virtual void emitInlinedDirective(CodeGenFunction &CGF,
1177	OpenMPDirectiveKind InnermostKind,
1178	const RegionCodeGenTy &CodeGen,
1179	bool HasCancel = false);
1180
1181	/// Emits reduction function.
1182	/// \param ReducerName Name of the function calling the reduction.
1183	/// \param ArgsElemType Array type containing pointers to reduction variables.
1184	/// \param Privates List of private copies for original reduction arguments.
1185	/// \param LHSExprs List of LHS in \a ReductionOps reduction operations.
1186	/// \param RHSExprs List of RHS in \a ReductionOps reduction operations.
1187	/// \param ReductionOps List of reduction operations in form 'LHS binop RHS'
1188	/// or 'operator binop(LHS, RHS)'.
1189	llvm::Function *emitReductionFunction(
1190	StringRef ReducerName, SourceLocation Loc, llvm::Type *ArgsElemType,
1191	ArrayRef<const Expr *> Privates, ArrayRef<const Expr *> LHSExprs,
1192	ArrayRef<const Expr *> RHSExprs, ArrayRef<const Expr *> ReductionOps);
1193
1194	/// Emits single reduction combiner
1195	void emitSingleReductionCombiner(CodeGenFunction &CGF,
1196	const Expr *ReductionOp,
1197	const Expr *PrivateRef,
1198	const DeclRefExpr *LHS,
1199	const DeclRefExpr *RHS);
1200
1201	struct ReductionOptionsTy {
1202	bool WithNowait;
1203	bool SimpleReduction;
1204	llvm::SmallVector<bool, 8> IsPrivateVarReduction;
1205	OpenMPDirectiveKind ReductionKind;
1206	};
1207
1208	/// Emits code for private variable reduction
1209	/// \param Privates List of private copies for original reduction arguments.
1210	/// \param LHSExprs List of LHS in \a ReductionOps reduction operations.
1211	/// \param RHSExprs List of RHS in \a ReductionOps reduction operations.
1212	/// \param ReductionOps List of reduction operations in form 'LHS binop RHS'
1213	/// or 'operator binop(LHS, RHS)'.
1214	void emitPrivateReduction(CodeGenFunction &CGF, SourceLocation Loc,
1215	const Expr *Privates, const Expr *LHSExprs,
1216	const Expr *RHSExprs, const Expr *ReductionOps);
1217
1218	/// Emit a code for reduction clause. Next code should be emitted for
1219	/// reduction:
1220	/// \code
1221	///
1222	/// static kmp_critical_name lock = { 0 };
1223	///
1224	/// void reduce_func(void *lhs[<n>], void *rhs[<n>]) {
1225	/// ...
1226	/// *(Type<i>*)lhs[i] = RedOp<i>(*(Type<i>*)lhs[i], *(Type<i>*)rhs[i]);
1227	/// ...
1228	/// }
1229	///
1230	/// ...
1231	/// void *RedList[<n>] = {&<RHSExprs>[0], ..., &<RHSExprs>[<n>-1]};
1232	/// switch (__kmpc_reduce{_nowait}(<loc>, <gtid>, <n>, sizeof(RedList),
1233	/// RedList, reduce_func, &<lock>)) {
1234	/// case 1:
1235	/// ...
1236	/// <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]);
1237	/// ...
1238	/// __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>);
1239	/// break;
1240	/// case 2:
1241	/// ...
1242	/// Atomic(<LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]));
1243	/// ...
1244	/// break;
1245	/// default:;
1246	/// }
1247	/// \endcode
1248	///
1249	/// \param Privates List of private copies for original reduction arguments.
1250	/// \param LHSExprs List of LHS in \a ReductionOps reduction operations.
1251	/// \param RHSExprs List of RHS in \a ReductionOps reduction operations.
1252	/// \param ReductionOps List of reduction operations in form 'LHS binop RHS'
1253	/// or 'operator binop(LHS, RHS)'.
1254	/// \param Options List of options for reduction codegen:
1255	/// WithNowait true if parent directive has also nowait clause, false
1256	/// otherwise.
1257	/// SimpleReduction Emit reduction operation only. Used for omp simd
1258	/// directive on the host.
1259	/// ReductionKind The kind of reduction to perform.
1260	virtual void emitReduction(CodeGenFunction &CGF, SourceLocation Loc,
1261	ArrayRef<const Expr *> Privates,
1262	ArrayRef<const Expr *> LHSExprs,
1263	ArrayRef<const Expr *> RHSExprs,
1264	ArrayRef<const Expr *> ReductionOps,
1265	ReductionOptionsTy Options);
1266
1267	/// Emit a code for initialization of task reduction clause. Next code
1268	/// should be emitted for reduction:
1269	/// \code
1270	///
1271	/// _taskred_item_t red_data[n];
1272	/// ...
1273	/// red_data[i].shar = &shareds[i];
1274	/// red_data[i].orig = &origs[i];
1275	/// red_data[i].size = sizeof(origs[i]);
1276	/// red_data[i].f_init = (void*)RedInit<i>;
1277	/// red_data[i].f_fini = (void*)RedDest<i>;
1278	/// red_data[i].f_comb = (void*)RedOp<i>;
1279	/// red_data[i].flags = <Flag_i>;
1280	/// ...
1281	/// void* tg1 = __kmpc_taskred_init(gtid, n, red_data);
1282	/// \endcode
1283	/// For reduction clause with task modifier it emits the next call:
1284	/// \code
1285	///
1286	/// _taskred_item_t red_data[n];
1287	/// ...
1288	/// red_data[i].shar = &shareds[i];
1289	/// red_data[i].orig = &origs[i];
1290	/// red_data[i].size = sizeof(origs[i]);
1291	/// red_data[i].f_init = (void*)RedInit<i>;
1292	/// red_data[i].f_fini = (void*)RedDest<i>;
1293	/// red_data[i].f_comb = (void*)RedOp<i>;
1294	/// red_data[i].flags = <Flag_i>;
1295	/// ...
1296	/// void* tg1 = __kmpc_taskred_modifier_init(loc, gtid, is_worksharing, n,
1297	/// red_data);
1298	/// \endcode
1299	/// \param LHSExprs List of LHS in \a Data.ReductionOps reduction operations.
1300	/// \param RHSExprs List of RHS in \a Data.ReductionOps reduction operations.
1301	/// \param Data Additional data for task generation like tiedness, final
1302	/// state, list of privates, reductions etc.
1303	virtual llvm::Value *emitTaskReductionInit(CodeGenFunction &CGF,
1304	SourceLocation Loc,
1305	ArrayRef<const Expr *> LHSExprs,
1306	ArrayRef<const Expr *> RHSExprs,
1307	const OMPTaskDataTy &Data);
1308
1309	/// Emits the following code for reduction clause with task modifier:
1310	/// \code
1311	/// __kmpc_task_reduction_modifier_fini(loc, gtid, is_worksharing);
1312	/// \endcode
1313	virtual void emitTaskReductionFini(CodeGenFunction &CGF, SourceLocation Loc,
1314	bool IsWorksharingReduction);
1315
1316	/// Required to resolve existing problems in the runtime. Emits threadprivate
1317	/// variables to store the size of the VLAs/array sections for
1318	/// initializer/combiner/finalizer functions.
1319	/// \param RCG Allows to reuse an existing data for the reductions.
1320	/// \param N Reduction item for which fixups must be emitted.
1321	virtual void emitTaskReductionFixups(CodeGenFunction &CGF, SourceLocation Loc,
1322	ReductionCodeGen &RCG, unsigned N);
1323
1324	/// Get the address of `void *` type of the privatue copy of the reduction
1325	/// item specified by the \p SharedLVal.
1326	/// \param ReductionsPtr Pointer to the reduction data returned by the
1327	/// emitTaskReductionInit function.
1328	/// \param SharedLVal Address of the original reduction item.
1329	virtual Address getTaskReductionItem(CodeGenFunction &CGF, SourceLocation Loc,
1330	llvm::Value *ReductionsPtr,
1331	LValue SharedLVal);
1332
1333	/// Emit code for 'taskwait' directive.
1334	virtual void emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc,
1335	const OMPTaskDataTy &Data);
1336
1337	/// Emit code for 'cancellation point' construct.
1338	/// \param CancelRegion Region kind for which the cancellation point must be
1339	/// emitted.
1340	///
1341	virtual void emitCancellationPointCall(CodeGenFunction &CGF,
1342	SourceLocation Loc,
1343	OpenMPDirectiveKind CancelRegion);
1344
1345	/// Emit code for 'cancel' construct.
1346	/// \param IfCond Condition in the associated 'if' clause, if it was
1347	/// specified, nullptr otherwise.
1348	/// \param CancelRegion Region kind for which the cancel must be emitted.
1349	///
1350	virtual void emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc,
1351	const Expr *IfCond,
1352	OpenMPDirectiveKind CancelRegion);
1353
1354	/// Emit outilined function for 'target' directive.
1355	/// \param D Directive to emit.
1356	/// \param ParentName Name of the function that encloses the target region.
1357	/// \param OutlinedFn Outlined function value to be defined by this call.
1358	/// \param OutlinedFnID Outlined function ID value to be defined by this call.
1359	/// \param IsOffloadEntry True if the outlined function is an offload entry.
1360	/// \param CodeGen Code generation sequence for the \a D directive.
1361	/// An outlined function may not be an entry if, e.g. the if clause always
1362	/// evaluates to false.
1363	virtual void emitTargetOutlinedFunction(const OMPExecutableDirective &D,
1364	StringRef ParentName,
1365	llvm::Function *&OutlinedFn,
1366	llvm::Constant *&OutlinedFnID,
1367	bool IsOffloadEntry,
1368	const RegionCodeGenTy &CodeGen);
1369
1370	/// Emit the target offloading code associated with \a D. The emitted
1371	/// code attempts offloading the execution to the device, an the event of
1372	/// a failure it executes the host version outlined in \a OutlinedFn.
1373	/// \param D Directive to emit.
1374	/// \param OutlinedFn Host version of the code to be offloaded.
1375	/// \param OutlinedFnID ID of host version of the code to be offloaded.
1376	/// \param IfCond Expression evaluated in if clause associated with the target
1377	/// directive, or null if no if clause is used.
1378	/// \param Device Expression evaluated in device clause associated with the
1379	/// target directive, or null if no device clause is used and device modifier.
1380	/// \param SizeEmitter Callback to emit number of iterations for loop-based
1381	/// directives.
1382	virtual void emitTargetCall(
1383	CodeGenFunction &CGF, const OMPExecutableDirective &D,
1384	llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID, const Expr *IfCond,
1385	llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier> Device,
1386	llvm::function_ref<llvm::Value *(CodeGenFunction &CGF,
1387	const OMPLoopDirective &D)>
1388	SizeEmitter);
1389
1390	/// Emit the target regions enclosed in \a GD function definition or
1391	/// the function itself in case it is a valid device function. Returns true if
1392	/// \a GD was dealt with successfully.
1393	/// \param GD Function to scan.
1394	virtual bool emitTargetFunctions(GlobalDecl GD);
1395
1396	/// Emit the global variable if it is a valid device global variable.
1397	/// Returns true if \a GD was dealt with successfully.
1398	/// \param GD Variable declaration to emit.
1399	virtual bool emitTargetGlobalVariable(GlobalDecl GD);
1400
1401	/// Checks if the provided global decl \a GD is a declare target variable and
1402	/// registers it when emitting code for the host.
1403	virtual void registerTargetGlobalVariable(const VarDecl *VD,
1404	llvm::Constant *Addr);
1405
1406	/// Emit the global \a GD if it is meaningful for the target. Returns
1407	/// if it was emitted successfully.
1408	/// \param GD Global to scan.
1409	virtual bool emitTargetGlobal(GlobalDecl GD);
1410
1411	/// Creates all the offload entries in the current compilation unit
1412	/// along with the associated metadata.
1413	void createOffloadEntriesAndInfoMetadata();
1414
1415	/// Emits code for teams call of the \a OutlinedFn with
1416	/// variables captured in a record which address is stored in \a
1417	/// CapturedStruct.
1418	/// \param OutlinedFn Outlined function to be run by team masters. Type of
1419	/// this function is void(*)(kmp_int32 *, kmp_int32, struct context_vars*).
1420	/// \param CapturedVars A pointer to the record with the references to
1421	/// variables used in \a OutlinedFn function.
1422	///
1423	virtual void emitTeamsCall(CodeGenFunction &CGF,
1424	const OMPExecutableDirective &D,
1425	SourceLocation Loc, llvm::Function *OutlinedFn,
1426	ArrayRef<llvm::Value *> CapturedVars);
1427
1428	/// Emits call to void __kmpc_push_num_teams(ident_t *loc, kmp_int32
1429	/// global_tid, kmp_int32 num_teams, kmp_int32 thread_limit) to generate code
1430	/// for num_teams clause.
1431	/// \param NumTeams An integer expression of teams.
1432	/// \param ThreadLimit An integer expression of threads.
1433	virtual void emitNumTeamsClause(CodeGenFunction &CGF, const Expr *NumTeams,
1434	const Expr *ThreadLimit, SourceLocation Loc);
1435
1436	/// Emits call to void __kmpc_set_thread_limit(ident_t *loc, kmp_int32
1437	/// global_tid, kmp_int32 thread_limit) to generate code for
1438	/// thread_limit clause on target directive
1439	/// \param ThreadLimit An integer expression of threads.
1440	virtual void emitThreadLimitClause(CodeGenFunction &CGF,
1441	const Expr *ThreadLimit,
1442	SourceLocation Loc);
1443
1444	/// Struct that keeps all the relevant information that should be kept
1445	/// throughout a 'target data' region.
1446	class TargetDataInfo : public llvm::OpenMPIRBuilder::TargetDataInfo {
1447	public:
1448	explicit TargetDataInfo() : llvm::OpenMPIRBuilder::TargetDataInfo() {}
1449	explicit TargetDataInfo(bool RequiresDevicePointerInfo,
1450	bool SeparateBeginEndCalls)
1451	: llvm::OpenMPIRBuilder::TargetDataInfo(RequiresDevicePointerInfo,
1452	SeparateBeginEndCalls) {}
1453	/// Map between the a declaration of a capture and the corresponding new
1454	/// llvm address where the runtime returns the device pointers.
1455	llvm::DenseMap<const ValueDecl *, llvm::Value *> CaptureDeviceAddrMap;
1456	};
1457
1458	/// Emit the target data mapping code associated with \a D.
1459	/// \param D Directive to emit.
1460	/// \param IfCond Expression evaluated in if clause associated with the
1461	/// target directive, or null if no device clause is used.
1462	/// \param Device Expression evaluated in device clause associated with the
1463	/// target directive, or null if no device clause is used.
1464	/// \param Info A record used to store information that needs to be preserved
1465	/// until the region is closed.
1466	virtual void emitTargetDataCalls(CodeGenFunction &CGF,
1467	const OMPExecutableDirective &D,
1468	const Expr *IfCond, const Expr *Device,
1469	const RegionCodeGenTy &CodeGen,
1470	CGOpenMPRuntime::TargetDataInfo &Info);
1471
1472	/// Emit the data mapping/movement code associated with the directive
1473	/// \a D that should be of the form 'target [{enter|exit} data | update]'.
1474	/// \param D Directive to emit.
1475	/// \param IfCond Expression evaluated in if clause associated with the target
1476	/// directive, or null if no if clause is used.
1477	/// \param Device Expression evaluated in device clause associated with the
1478	/// target directive, or null if no device clause is used.
1479	virtual void emitTargetDataStandAloneCall(CodeGenFunction &CGF,
1480	const OMPExecutableDirective &D,
1481	const Expr *IfCond,
1482	const Expr *Device);
1483
1484	/// Marks function \a Fn with properly mangled versions of vector functions.
1485	/// \param FD Function marked as 'declare simd'.
1486	/// \param Fn LLVM function that must be marked with 'declare simd'
1487	/// attributes.
1488	virtual void emitDeclareSimdFunction(const FunctionDecl *FD,
1489	llvm::Function *Fn);
1490
1491	/// Emit initialization for doacross loop nesting support.
1492	/// \param D Loop-based construct used in doacross nesting construct.
1493	virtual void emitDoacrossInit(CodeGenFunction &CGF, const OMPLoopDirective &D,
1494	ArrayRef<Expr *> NumIterations);
1495
1496	/// Emit code for doacross ordered directive with 'depend' clause.
1497	/// \param C 'depend' clause with 'sink|source' dependency kind.
1498	virtual void emitDoacrossOrdered(CodeGenFunction &CGF,
1499	const OMPDependClause *C);
1500
1501	/// Emit code for doacross ordered directive with 'doacross' clause.
1502	/// \param C 'doacross' clause with 'sink|source' dependence type.
1503	virtual void emitDoacrossOrdered(CodeGenFunction &CGF,
1504	const OMPDoacrossClause *C);
1505
1506	/// Translates the native parameter of outlined function if this is required
1507	/// for target.
1508	/// \param FD Field decl from captured record for the parameter.
1509	/// \param NativeParam Parameter itself.
1510	virtual const VarDecl *translateParameter(const FieldDecl *FD,
1511	const VarDecl *NativeParam) const {
1512	return NativeParam;
1513	}
1514
1515	/// Gets the address of the native argument basing on the address of the
1516	/// target-specific parameter.
1517	/// \param NativeParam Parameter itself.
1518	/// \param TargetParam Corresponding target-specific parameter.
1519	virtual Address getParameterAddress(CodeGenFunction &CGF,
1520	const VarDecl *NativeParam,
1521	const VarDecl *TargetParam) const;
1522
1523	/// Choose default schedule type and chunk value for the
1524	/// dist_schedule clause.
1525	virtual void getDefaultDistScheduleAndChunk(CodeGenFunction &CGF,
1526	const OMPLoopDirective &S, OpenMPDistScheduleClauseKind &ScheduleKind,
1527	llvm::Value *&Chunk) const {}
1528
1529	/// Choose default schedule type and chunk value for the
1530	/// schedule clause.
1531	virtual void getDefaultScheduleAndChunk(CodeGenFunction &CGF,
1532	const OMPLoopDirective &S, OpenMPScheduleClauseKind &ScheduleKind,
1533	const Expr *&ChunkExpr) const;
1534
1535	/// Emits call of the outlined function with the provided arguments,
1536	/// translating these arguments to correct target-specific arguments.
1537	virtual void
1538	emitOutlinedFunctionCall(CodeGenFunction &CGF, SourceLocation Loc,
1539	llvm::FunctionCallee OutlinedFn,
1540	ArrayRef<llvm::Value *> Args = {}) const;
1541
1542	/// Emits OpenMP-specific function prolog.
1543	/// Required for device constructs.
1544	virtual void emitFunctionProlog(CodeGenFunction &CGF, const Decl *D);
1545
1546	/// Gets the OpenMP-specific address of the local variable.
1547	virtual Address getAddressOfLocalVariable(CodeGenFunction &CGF,
1548	const VarDecl *VD);
1549
1550	/// Marks the declaration as already emitted for the device code and returns
1551	/// true, if it was marked already, and false, otherwise.
1552	bool markAsGlobalTarget(GlobalDecl GD);
1553
1554	/// Emit deferred declare target variables marked for deferred emission.
1555	void emitDeferredTargetDecls() const;
1556
1557	/// Adjust some parameters for the target-based directives, like addresses of
1558	/// the variables captured by reference in lambdas.
1559	virtual void
1560	adjustTargetSpecificDataForLambdas(CodeGenFunction &CGF,
1561	const OMPExecutableDirective &D) const;
1562
1563	/// Perform check on requires decl to ensure that target architecture
1564	/// supports unified addressing
1565	virtual void processRequiresDirective(const OMPRequiresDecl *D);
1566
1567	/// Gets default memory ordering as specified in requires directive.
1568	llvm::AtomicOrdering getDefaultMemoryOrdering() const;
1569
1570	/// Checks if the variable has associated OMPAllocateDeclAttr attribute with
1571	/// the predefined allocator and translates it into the corresponding address
1572	/// space.
1573	virtual bool hasAllocateAttributeForGlobalVar(const VarDecl *VD, LangAS &AS);
1574
1575	/// Return whether the unified_shared_memory has been specified.
1576	bool hasRequiresUnifiedSharedMemory() const;
1577
1578	/// Checks if the \p VD variable is marked as nontemporal declaration in
1579	/// current context.
1580	bool isNontemporalDecl(const ValueDecl *VD) const;
1581
1582	/// Create specialized alloca to handle lastprivate conditionals.
1583	Address emitLastprivateConditionalInit(CodeGenFunction &CGF,
1584	const VarDecl *VD);
1585
1586	/// Checks if the provided \p LVal is lastprivate conditional and emits the
1587	/// code to update the value of the original variable.
1588	/// \code
1589	/// lastprivate(conditional: a)
1590	/// ...
1591	/// <type> a;
1592	/// lp_a = ...;
1593	/// #pragma omp critical(a)
1594	/// if (last_iv_a <= iv) {
1595	/// last_iv_a = iv;
1596	/// global_a = lp_a;
1597	/// }
1598	/// \endcode
1599	virtual void checkAndEmitLastprivateConditional(CodeGenFunction &CGF,
1600	const Expr *LHS);
1601
1602	/// Checks if the lastprivate conditional was updated in inner region and
1603	/// writes the value.
1604	/// \code
1605	/// lastprivate(conditional: a)
1606	/// ...
1607	/// <type> a;bool Fired = false;
1608	/// #pragma omp ... shared(a)
1609	/// {
1610	/// lp_a = ...;
1611	/// Fired = true;
1612	/// }
1613	/// if (Fired) {
1614	/// #pragma omp critical(a)
1615	/// if (last_iv_a <= iv) {
1616	/// last_iv_a = iv;
1617	/// global_a = lp_a;
1618	/// }
1619	/// Fired = false;
1620	/// }
1621	/// \endcode
1622	virtual void checkAndEmitSharedLastprivateConditional(
1623	CodeGenFunction &CGF, const OMPExecutableDirective &D,
1624	const llvm::DenseSet<CanonicalDeclPtr<const VarDecl>> &IgnoredDecls);
1625
1626	/// Gets the address of the global copy used for lastprivate conditional
1627	/// update, if any.
1628	/// \param PrivLVal LValue for the private copy.
1629	/// \param VD Original lastprivate declaration.
1630	virtual void emitLastprivateConditionalFinalUpdate(CodeGenFunction &CGF,
1631	LValue PrivLVal,
1632	const VarDecl *VD,
1633	SourceLocation Loc);
1634
1635	/// Emits list of dependecies based on the provided data (array of
1636	/// dependence/expression pairs).
1637	/// \returns Pointer to the first element of the array casted to VoidPtr type.
1638	std::pair<llvm::Value *, Address>
1639	emitDependClause(CodeGenFunction &CGF,
1640	ArrayRef<OMPTaskDataTy::DependData> Dependencies,
1641	SourceLocation Loc);
1642
1643	/// Emits list of dependecies based on the provided data (array of
1644	/// dependence/expression pairs) for depobj construct. In this case, the
1645	/// variable is allocated in dynamically. \returns Pointer to the first
1646	/// element of the array casted to VoidPtr type.
1647	Address emitDepobjDependClause(CodeGenFunction &CGF,
1648	const OMPTaskDataTy::DependData &Dependencies,
1649	SourceLocation Loc);
1650
1651	/// Emits the code to destroy the dependency object provided in depobj
1652	/// directive.
1653	void emitDestroyClause(CodeGenFunction &CGF, LValue DepobjLVal,
1654	SourceLocation Loc);
1655
1656	/// Updates the dependency kind in the specified depobj object.
1657	/// \param DepobjLVal LValue for the main depobj object.
1658	/// \param NewDepKind New dependency kind.
1659	void emitUpdateClause(CodeGenFunction &CGF, LValue DepobjLVal,
1660	OpenMPDependClauseKind NewDepKind, SourceLocation Loc);
1661
1662	/// Initializes user defined allocators specified in the uses_allocators
1663	/// clauses.
1664	void emitUsesAllocatorsInit(CodeGenFunction &CGF, const Expr *Allocator,
1665	const Expr *AllocatorTraits);
1666
1667	/// Destroys user defined allocators specified in the uses_allocators clause.
1668	void emitUsesAllocatorsFini(CodeGenFunction &CGF, const Expr *Allocator);
1669
1670	/// Returns true if the variable is a local variable in untied task.
1671	bool isLocalVarInUntiedTask(CodeGenFunction &CGF, const VarDecl *VD) const;
1672	};
1673
1674	/// Class supports emissionof SIMD-only code.
1675	class CGOpenMPSIMDRuntime final : public CGOpenMPRuntime {
1676	public:
1677	explicit CGOpenMPSIMDRuntime(CodeGenModule &CGM) : CGOpenMPRuntime(CGM) {}
1678	~CGOpenMPSIMDRuntime() override {}
1679
1680	/// Emits outlined function for the specified OpenMP parallel directive
1681	/// \a D. This outlined function has type void(*)(kmp_int32 *ThreadID,
1682	/// kmp_int32 BoundID, struct context_vars*).
1683	/// \param CGF Reference to current CodeGenFunction.
1684	/// \param D OpenMP directive.
1685	/// \param ThreadIDVar Variable for thread id in the current OpenMP region.
1686	/// \param InnermostKind Kind of innermost directive (for simple directives it
1687	/// is a directive itself, for combined - its innermost directive).
1688	/// \param CodeGen Code generation sequence for the \a D directive.
1689	llvm::Function *emitParallelOutlinedFunction(
1690	CodeGenFunction &CGF, const OMPExecutableDirective &D,
1691	const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,
1692	const RegionCodeGenTy &CodeGen) override;
1693
1694	/// Emits outlined function for the specified OpenMP teams directive
1695	/// \a D. This outlined function has type void(*)(kmp_int32 *ThreadID,
1696	/// kmp_int32 BoundID, struct context_vars*).
1697	/// \param CGF Reference to current CodeGenFunction.
1698	/// \param D OpenMP directive.
1699	/// \param ThreadIDVar Variable for thread id in the current OpenMP region.
1700	/// \param InnermostKind Kind of innermost directive (for simple directives it
1701	/// is a directive itself, for combined - its innermost directive).
1702	/// \param CodeGen Code generation sequence for the \a D directive.
1703	llvm::Function *emitTeamsOutlinedFunction(
1704	CodeGenFunction &CGF, const OMPExecutableDirective &D,
1705	const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,
1706	const RegionCodeGenTy &CodeGen) override;
1707
1708	/// Emits outlined function for the OpenMP task directive \a D. This
1709	/// outlined function has type void(*)(kmp_int32 ThreadID, struct task_t*
1710	/// TaskT).
1711	/// \param D OpenMP directive.
1712	/// \param ThreadIDVar Variable for thread id in the current OpenMP region.
1713	/// \param PartIDVar Variable for partition id in the current OpenMP untied
1714	/// task region.
1715	/// \param TaskTVar Variable for task_t argument.
1716	/// \param InnermostKind Kind of innermost directive (for simple directives it
1717	/// is a directive itself, for combined - its innermost directive).
1718	/// \param CodeGen Code generation sequence for the \a D directive.
1719	/// \param Tied true if task is generated for tied task, false otherwise.
1720	/// \param NumberOfParts Number of parts in untied task. Ignored for tied
1721	/// tasks.
1722	///
1723	llvm::Function *emitTaskOutlinedFunction(
1724	const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
1725	const VarDecl *PartIDVar, const VarDecl *TaskTVar,
1726	OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
1727	bool Tied, unsigned &NumberOfParts) override;
1728
1729	/// Emits code for parallel or serial call of the \a OutlinedFn with
1730	/// variables captured in a record which address is stored in \a
1731	/// CapturedStruct.
1732	/// \param OutlinedFn Outlined function to be run in parallel threads. Type of
1733	/// this function is void(*)(kmp_int32 *, kmp_int32, struct context_vars*).
1734	/// \param CapturedVars A pointer to the record with the references to
1735	/// variables used in \a OutlinedFn function.
1736	/// \param IfCond Condition in the associated 'if' clause, if it was
1737	/// specified, nullptr otherwise.
1738	/// \param NumThreads The value corresponding to the num_threads clause, if
1739	/// any, or nullptr.
1740	///
1741	void emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc,
1742	llvm::Function *OutlinedFn,
1743	ArrayRef<llvm::Value *> CapturedVars,
1744	const Expr *IfCond, llvm::Value *NumThreads) override;
1745
1746	/// Emits a critical region.
1747	/// \param CriticalName Name of the critical region.
1748	/// \param CriticalOpGen Generator for the statement associated with the given
1749	/// critical region.
1750	/// \param Hint Value of the 'hint' clause (optional).
1751	void emitCriticalRegion(CodeGenFunction &CGF, StringRef CriticalName,
1752	const RegionCodeGenTy &CriticalOpGen,
1753	SourceLocation Loc,
1754	const Expr *Hint = nullptr) override;
1755
1756	/// Emits a master region.
1757	/// \param MasterOpGen Generator for the statement associated with the given
1758	/// master region.
1759	void emitMasterRegion(CodeGenFunction &CGF,
1760	const RegionCodeGenTy &MasterOpGen,
1761	SourceLocation Loc) override;
1762
1763	/// Emits a masked region.
1764	/// \param MaskedOpGen Generator for the statement associated with the given
1765	/// masked region.
1766	void emitMaskedRegion(CodeGenFunction &CGF,
1767	const RegionCodeGenTy &MaskedOpGen, SourceLocation Loc,
1768	const Expr *Filter = nullptr) override;
1769
1770	/// Emits a masked region.
1771	/// \param MaskedOpGen Generator for the statement associated with the given
1772	/// masked region.
1773
1774	/// Emits code for a taskyield directive.
1775	void emitTaskyieldCall(CodeGenFunction &CGF, SourceLocation Loc) override;
1776
1777	/// Emit a taskgroup region.
1778	/// \param TaskgroupOpGen Generator for the statement associated with the
1779	/// given taskgroup region.
1780	void emitTaskgroupRegion(CodeGenFunction &CGF,
1781	const RegionCodeGenTy &TaskgroupOpGen,
1782	SourceLocation Loc) override;
1783
1784	/// Emits a single region.
1785	/// \param SingleOpGen Generator for the statement associated with the given
1786	/// single region.
1787	void emitSingleRegion(CodeGenFunction &CGF,
1788	const RegionCodeGenTy &SingleOpGen, SourceLocation Loc,
1789	ArrayRef<const Expr *> CopyprivateVars,
1790	ArrayRef<const Expr *> DestExprs,
1791	ArrayRef<const Expr *> SrcExprs,
1792	ArrayRef<const Expr *> AssignmentOps) override;
1793
1794	/// Emit an ordered region.
1795	/// \param OrderedOpGen Generator for the statement associated with the given
1796	/// ordered region.
1797	void emitOrderedRegion(CodeGenFunction &CGF,
1798	const RegionCodeGenTy &OrderedOpGen,
1799	SourceLocation Loc, bool IsThreads) override;
1800
1801	/// Emit an implicit/explicit barrier for OpenMP threads.
1802	/// \param Kind Directive for which this implicit barrier call must be
1803	/// generated. Must be OMPD_barrier for explicit barrier generation.
1804	/// \param EmitChecks true if need to emit checks for cancellation barriers.
1805	/// \param ForceSimpleCall true simple barrier call must be emitted, false if
1806	/// runtime class decides which one to emit (simple or with cancellation
1807	/// checks).
1808	///
1809	void emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc,
1810	OpenMPDirectiveKind Kind, bool EmitChecks = true,
1811	bool ForceSimpleCall = false) override;
1812
1813	/// This is used for non static scheduled types and when the ordered
1814	/// clause is present on the loop construct.
1815	/// Depending on the loop schedule, it is necessary to call some runtime
1816	/// routine before start of the OpenMP loop to get the loop upper / lower
1817	/// bounds \a LB and \a UB and stride \a ST.
1818	///
1819	/// \param CGF Reference to current CodeGenFunction.
1820	/// \param Loc Clang source location.
1821	/// \param ScheduleKind Schedule kind, specified by the 'schedule' clause.
1822	/// \param IVSize Size of the iteration variable in bits.
1823	/// \param IVSigned Sign of the iteration variable.
1824	/// \param Ordered true if loop is ordered, false otherwise.
1825	/// \param DispatchValues struct containing llvm values for lower bound, upper
1826	/// bound, and chunk expression.
1827	/// For the default (nullptr) value, the chunk 1 will be used.
1828	///
1829	void emitForDispatchInit(CodeGenFunction &CGF, SourceLocation Loc,
1830	const OpenMPScheduleTy &ScheduleKind,
1831	unsigned IVSize, bool IVSigned, bool Ordered,
1832	const DispatchRTInput &DispatchValues) override;
1833
1834	/// This is used for non static scheduled types and when the ordered
1835	/// clause is present on the loop construct.
1836	///
1837	/// \param CGF Reference to current CodeGenFunction.
1838	/// \param Loc Clang source location.
1839	///
1840	void emitForDispatchDeinit(CodeGenFunction &CGF, SourceLocation Loc) override;
1841
1842	/// Call the appropriate runtime routine to initialize it before start
1843	/// of loop.
1844	///
1845	/// This is used only in case of static schedule, when the user did not
1846	/// specify a ordered clause on the loop construct.
1847	/// Depending on the loop schedule, it is necessary to call some runtime
1848	/// routine before start of the OpenMP loop to get the loop upper / lower
1849	/// bounds LB and UB and stride ST.
1850	///
1851	/// \param CGF Reference to current CodeGenFunction.
1852	/// \param Loc Clang source location.
1853	/// \param DKind Kind of the directive.
1854	/// \param ScheduleKind Schedule kind, specified by the 'schedule' clause.
1855	/// \param Values Input arguments for the construct.
1856	///
1857	void emitForStaticInit(CodeGenFunction &CGF, SourceLocation Loc,
1858	OpenMPDirectiveKind DKind,
1859	const OpenMPScheduleTy &ScheduleKind,
1860	const StaticRTInput &Values) override;
1861
1862	///
1863	/// \param CGF Reference to current CodeGenFunction.
1864	/// \param Loc Clang source location.
1865	/// \param SchedKind Schedule kind, specified by the 'dist_schedule' clause.
1866	/// \param Values Input arguments for the construct.
1867	///
1868	void emitDistributeStaticInit(CodeGenFunction &CGF, SourceLocation Loc,
1869	OpenMPDistScheduleClauseKind SchedKind,
1870	const StaticRTInput &Values) override;
1871
1872	/// Call the appropriate runtime routine to notify that we finished
1873	/// iteration of the ordered loop with the dynamic scheduling.
1874	///
1875	/// \param CGF Reference to current CodeGenFunction.
1876	/// \param Loc Clang source location.
1877	/// \param IVSize Size of the iteration variable in bits.
1878	/// \param IVSigned Sign of the iteration variable.
1879	///
1880	void emitForOrderedIterationEnd(CodeGenFunction &CGF, SourceLocation Loc,
1881	unsigned IVSize, bool IVSigned) override;
1882
1883	/// Call the appropriate runtime routine to notify that we finished
1884	/// all the work with current loop.
1885	///
1886	/// \param CGF Reference to current CodeGenFunction.
1887	/// \param Loc Clang source location.
1888	/// \param DKind Kind of the directive for which the static finish is emitted.
1889	///
1890	void emitForStaticFinish(CodeGenFunction &CGF, SourceLocation Loc,
1891	OpenMPDirectiveKind DKind) override;
1892
1893	/// Call __kmpc_dispatch_next(
1894	/// ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter,
1895	/// kmp_int[32|64] *p_lower, kmp_int[32|64] *p_upper,
1896	/// kmp_int[32|64] *p_stride);
1897	/// \param IVSize Size of the iteration variable in bits.
1898	/// \param IVSigned Sign of the iteration variable.
1899	/// \param IL Address of the output variable in which the flag of the
1900	/// last iteration is returned.
1901	/// \param LB Address of the output variable in which the lower iteration
1902	/// number is returned.
1903	/// \param UB Address of the output variable in which the upper iteration
1904	/// number is returned.
1905	/// \param ST Address of the output variable in which the stride value is
1906	/// returned.
1907	llvm::Value *emitForNext(CodeGenFunction &CGF, SourceLocation Loc,
1908	unsigned IVSize, bool IVSigned, Address IL,
1909	Address LB, Address UB, Address ST) override;
1910
1911	/// Emits call to void __kmpc_push_num_threads(ident_t *loc, kmp_int32
1912	/// global_tid, kmp_int32 num_threads) to generate code for 'num_threads'
1913	/// clause.
1914	/// \param NumThreads An integer value of threads.
1915	void emitNumThreadsClause(CodeGenFunction &CGF, llvm::Value *NumThreads,
1916	SourceLocation Loc) override;
1917
1918	/// Emit call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32
1919	/// global_tid, int proc_bind) to generate code for 'proc_bind' clause.
1920	void emitProcBindClause(CodeGenFunction &CGF,
1921	llvm::omp::ProcBindKind ProcBind,
1922	SourceLocation Loc) override;
1923
1924	/// Returns address of the threadprivate variable for the current
1925	/// thread.
1926	/// \param VD Threadprivate variable.
1927	/// \param VDAddr Address of the global variable \a VD.
1928	/// \param Loc Location of the reference to threadprivate var.
1929	/// \return Address of the threadprivate variable for the current thread.
1930	Address getAddrOfThreadPrivate(CodeGenFunction &CGF, const VarDecl *VD,
1931	Address VDAddr, SourceLocation Loc) override;
1932
1933	/// Emit a code for initialization of threadprivate variable. It emits
1934	/// a call to runtime library which adds initial value to the newly created
1935	/// threadprivate variable (if it is not constant) and registers destructor
1936	/// for the variable (if any).
1937	/// \param VD Threadprivate variable.
1938	/// \param VDAddr Address of the global variable \a VD.
1939	/// \param Loc Location of threadprivate declaration.
1940	/// \param PerformInit true if initialization expression is not constant.
1941	llvm::Function *
1942	emitThreadPrivateVarDefinition(const VarDecl *VD, Address VDAddr,
1943	SourceLocation Loc, bool PerformInit,
1944	CodeGenFunction *CGF = nullptr) override;
1945
1946	/// Creates artificial threadprivate variable with name \p Name and type \p
1947	/// VarType.
1948	/// \param VarType Type of the artificial threadprivate variable.
1949	/// \param Name Name of the artificial threadprivate variable.
1950	Address getAddrOfArtificialThreadPrivate(CodeGenFunction &CGF,
1951	QualType VarType,
1952	StringRef Name) override;
1953
1954	/// Emit flush of the variables specified in 'omp flush' directive.
1955	/// \param Vars List of variables to flush.
1956	void emitFlush(CodeGenFunction &CGF, ArrayRef<const Expr *> Vars,
1957	SourceLocation Loc, llvm::AtomicOrdering AO) override;
1958
1959	/// Emit task region for the task directive. The task region is
1960	/// emitted in several steps:
1961	/// 1. Emit a call to kmp_task_t *__kmpc_omp_task_alloc(ident_t *, kmp_int32
1962	/// gtid, kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
1963	/// kmp_routine_entry_t *task_entry). Here task_entry is a pointer to the
1964	/// function:
1965	/// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) {
1966	/// TaskFunction(gtid, tt->part_id, tt->shareds);
1967	/// return 0;
1968	/// }
1969	/// 2. Copy a list of shared variables to field shareds of the resulting
1970	/// structure kmp_task_t returned by the previous call (if any).
1971	/// 3. Copy a pointer to destructions function to field destructions of the
1972	/// resulting structure kmp_task_t.
1973	/// 4. Emit a call to kmp_int32 __kmpc_omp_task(ident_t *, kmp_int32 gtid,
1974	/// kmp_task_t *new_task), where new_task is a resulting structure from
1975	/// previous items.
1976	/// \param D Current task directive.
1977	/// \param TaskFunction An LLVM function with type void (*)(i32 /*gtid*/, i32
1978	/// /*part_id*/, captured_struct */*__context*/);
1979	/// \param SharedsTy A type which contains references the shared variables.
1980	/// \param Shareds Context with the list of shared variables from the \p
1981	/// TaskFunction.
1982	/// \param IfCond Not a nullptr if 'if' clause was specified, nullptr
1983	/// otherwise.
1984	/// \param Data Additional data for task generation like tiednsee, final
1985	/// state, list of privates etc.
1986	void emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc,
1987	const OMPExecutableDirective &D,
1988	llvm::Function *TaskFunction, QualType SharedsTy,
1989	Address Shareds, const Expr *IfCond,
1990	const OMPTaskDataTy &Data) override;
1991
1992	/// Emit task region for the taskloop directive. The taskloop region is
1993	/// emitted in several steps:
1994	/// 1. Emit a call to kmp_task_t *__kmpc_omp_task_alloc(ident_t *, kmp_int32
1995	/// gtid, kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
1996	/// kmp_routine_entry_t *task_entry). Here task_entry is a pointer to the
1997	/// function:
1998	/// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) {
1999	/// TaskFunction(gtid, tt->part_id, tt->shareds);
2000	/// return 0;
2001	/// }
2002	/// 2. Copy a list of shared variables to field shareds of the resulting
2003	/// structure kmp_task_t returned by the previous call (if any).
2004	/// 3. Copy a pointer to destructions function to field destructions of the
2005	/// resulting structure kmp_task_t.
2006	/// 4. Emit a call to void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t
2007	/// *task, int if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int
2008	/// nogroup, int sched, kmp_uint64 grainsize, void *task_dup ), where new_task
2009	/// is a resulting structure from
2010	/// previous items.
2011	/// \param D Current task directive.
2012	/// \param TaskFunction An LLVM function with type void (*)(i32 /*gtid*/, i32
2013	/// /*part_id*/, captured_struct */*__context*/);
2014	/// \param SharedsTy A type which contains references the shared variables.
2015	/// \param Shareds Context with the list of shared variables from the \p
2016	/// TaskFunction.
2017	/// \param IfCond Not a nullptr if 'if' clause was specified, nullptr
2018	/// otherwise.
2019	/// \param Data Additional data for task generation like tiednsee, final
2020	/// state, list of privates etc.
2021	void emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc,
2022	const OMPLoopDirective &D, llvm::Function *TaskFunction,
2023	QualType SharedsTy, Address Shareds, const Expr *IfCond,
2024	const OMPTaskDataTy &Data) override;
2025
2026	/// Emit a code for reduction clause. Next code should be emitted for
2027	/// reduction:
2028	/// \code
2029	///
2030	/// static kmp_critical_name lock = { 0 };
2031	///
2032	/// void reduce_func(void *lhs[<n>], void *rhs[<n>]) {
2033	/// ...
2034	/// *(Type<i>*)lhs[i] = RedOp<i>(*(Type<i>*)lhs[i], *(Type<i>*)rhs[i]);
2035	/// ...
2036	/// }
2037	///
2038	/// ...
2039	/// void *RedList[<n>] = {&<RHSExprs>[0], ..., &<RHSExprs>[<n>-1]};
2040	/// switch (__kmpc_reduce{_nowait}(<loc>, <gtid>, <n>, sizeof(RedList),
2041	/// RedList, reduce_func, &<lock>)) {
2042	/// case 1:
2043	/// ...
2044	/// <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]);
2045	/// ...
2046	/// __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>);
2047	/// break;
2048	/// case 2:
2049	/// ...
2050	/// Atomic(<LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]));
2051	/// ...
2052	/// break;
2053	/// default:;
2054	/// }
2055	/// \endcode
2056	///
2057	/// \param Privates List of private copies for original reduction arguments.
2058	/// \param LHSExprs List of LHS in \a ReductionOps reduction operations.
2059	/// \param RHSExprs List of RHS in \a ReductionOps reduction operations.
2060	/// \param ReductionOps List of reduction operations in form 'LHS binop RHS'
2061	/// or 'operator binop(LHS, RHS)'.
2062	/// \param Options List of options for reduction codegen:
2063	/// WithNowait true if parent directive has also nowait clause, false
2064	/// otherwise.
2065	/// SimpleReduction Emit reduction operation only. Used for omp simd
2066	/// directive on the host.
2067	/// ReductionKind The kind of reduction to perform.
2068	void emitReduction(CodeGenFunction &CGF, SourceLocation Loc,
2069	ArrayRef<const Expr *> Privates,
2070	ArrayRef<const Expr *> LHSExprs,
2071	ArrayRef<const Expr *> RHSExprs,
2072	ArrayRef<const Expr *> ReductionOps,
2073	ReductionOptionsTy Options) override;
2074
2075	/// Emit a code for initialization of task reduction clause. Next code
2076	/// should be emitted for reduction:
2077	/// \code
2078	///
2079	/// _taskred_item_t red_data[n];
2080	/// ...
2081	/// red_data[i].shar = &shareds[i];
2082	/// red_data[i].orig = &origs[i];
2083	/// red_data[i].size = sizeof(origs[i]);
2084	/// red_data[i].f_init = (void*)RedInit<i>;
2085	/// red_data[i].f_fini = (void*)RedDest<i>;
2086	/// red_data[i].f_comb = (void*)RedOp<i>;
2087	/// red_data[i].flags = <Flag_i>;
2088	/// ...
2089	/// void* tg1 = __kmpc_taskred_init(gtid, n, red_data);
2090	/// \endcode
2091	/// For reduction clause with task modifier it emits the next call:
2092	/// \code
2093	///
2094	/// _taskred_item_t red_data[n];
2095	/// ...
2096	/// red_data[i].shar = &shareds[i];
2097	/// red_data[i].orig = &origs[i];
2098	/// red_data[i].size = sizeof(origs[i]);
2099	/// red_data[i].f_init = (void*)RedInit<i>;
2100	/// red_data[i].f_fini = (void*)RedDest<i>;
2101	/// red_data[i].f_comb = (void*)RedOp<i>;
2102	/// red_data[i].flags = <Flag_i>;
2103	/// ...
2104	/// void* tg1 = __kmpc_taskred_modifier_init(loc, gtid, is_worksharing, n,
2105	/// red_data);
2106	/// \endcode
2107	/// \param LHSExprs List of LHS in \a Data.ReductionOps reduction operations.
2108	/// \param RHSExprs List of RHS in \a Data.ReductionOps reduction operations.
2109	/// \param Data Additional data for task generation like tiedness, final
2110	/// state, list of privates, reductions etc.
2111	llvm::Value *emitTaskReductionInit(CodeGenFunction &CGF, SourceLocation Loc,
2112	ArrayRef<const Expr *> LHSExprs,
2113	ArrayRef<const Expr *> RHSExprs,
2114	const OMPTaskDataTy &Data) override;
2115
2116	/// Emits the following code for reduction clause with task modifier:
2117	/// \code
2118	/// __kmpc_task_reduction_modifier_fini(loc, gtid, is_worksharing);
2119	/// \endcode
2120	void emitTaskReductionFini(CodeGenFunction &CGF, SourceLocation Loc,
2121	bool IsWorksharingReduction) override;
2122
2123	/// Required to resolve existing problems in the runtime. Emits threadprivate
2124	/// variables to store the size of the VLAs/array sections for
2125	/// initializer/combiner/finalizer functions + emits threadprivate variable to
2126	/// store the pointer to the original reduction item for the custom
2127	/// initializer defined by declare reduction construct.
2128	/// \param RCG Allows to reuse an existing data for the reductions.
2129	/// \param N Reduction item for which fixups must be emitted.
2130	void emitTaskReductionFixups(CodeGenFunction &CGF, SourceLocation Loc,
2131	ReductionCodeGen &RCG, unsigned N) override;
2132
2133	/// Get the address of `void *` type of the privatue copy of the reduction
2134	/// item specified by the \p SharedLVal.
2135	/// \param ReductionsPtr Pointer to the reduction data returned by the
2136	/// emitTaskReductionInit function.
2137	/// \param SharedLVal Address of the original reduction item.
2138	Address getTaskReductionItem(CodeGenFunction &CGF, SourceLocation Loc,
2139	llvm::Value *ReductionsPtr,
2140	LValue SharedLVal) override;
2141
2142	/// Emit code for 'taskwait' directive.
2143	void emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc,
2144	const OMPTaskDataTy &Data) override;
2145
2146	/// Emit code for 'cancellation point' construct.
2147	/// \param CancelRegion Region kind for which the cancellation point must be
2148	/// emitted.
2149	///
2150	void emitCancellationPointCall(CodeGenFunction &CGF, SourceLocation Loc,
2151	OpenMPDirectiveKind CancelRegion) override;
2152
2153	/// Emit code for 'cancel' construct.
2154	/// \param IfCond Condition in the associated 'if' clause, if it was
2155	/// specified, nullptr otherwise.
2156	/// \param CancelRegion Region kind for which the cancel must be emitted.
2157	///
2158	void emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc,
2159	const Expr *IfCond,
2160	OpenMPDirectiveKind CancelRegion) override;
2161
2162	/// Emit outilined function for 'target' directive.
2163	/// \param D Directive to emit.
2164	/// \param ParentName Name of the function that encloses the target region.
2165	/// \param OutlinedFn Outlined function value to be defined by this call.
2166	/// \param OutlinedFnID Outlined function ID value to be defined by this call.
2167	/// \param IsOffloadEntry True if the outlined function is an offload entry.
2168	/// \param CodeGen Code generation sequence for the \a D directive.
2169	/// An outlined function may not be an entry if, e.g. the if clause always
2170	/// evaluates to false.
2171	void emitTargetOutlinedFunction(const OMPExecutableDirective &D,
2172	StringRef ParentName,
2173	llvm::Function *&OutlinedFn,
2174	llvm::Constant *&OutlinedFnID,
2175	bool IsOffloadEntry,
2176	const RegionCodeGenTy &CodeGen) override;
2177
2178	/// Emit the target offloading code associated with \a D. The emitted
2179	/// code attempts offloading the execution to the device, an the event of
2180	/// a failure it executes the host version outlined in \a OutlinedFn.
2181	/// \param D Directive to emit.
2182	/// \param OutlinedFn Host version of the code to be offloaded.
2183	/// \param OutlinedFnID ID of host version of the code to be offloaded.
2184	/// \param IfCond Expression evaluated in if clause associated with the target
2185	/// directive, or null if no if clause is used.
2186	/// \param Device Expression evaluated in device clause associated with the
2187	/// target directive, or null if no device clause is used and device modifier.
2188	void emitTargetCall(
2189	CodeGenFunction &CGF, const OMPExecutableDirective &D,
2190	llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID, const Expr *IfCond,
2191	llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier> Device,
2192	llvm::function_ref<llvm::Value *(CodeGenFunction &CGF,
2193	const OMPLoopDirective &D)>
2194	SizeEmitter) override;
2195
2196	/// Emit the target regions enclosed in \a GD function definition or
2197	/// the function itself in case it is a valid device function. Returns true if
2198	/// \a GD was dealt with successfully.
2199	/// \param GD Function to scan.
2200	bool emitTargetFunctions(GlobalDecl GD) override;
2201
2202	/// Emit the global variable if it is a valid device global variable.
2203	/// Returns true if \a GD was dealt with successfully.
2204	/// \param GD Variable declaration to emit.
2205	bool emitTargetGlobalVariable(GlobalDecl GD) override;
2206
2207	/// Emit the global \a GD if it is meaningful for the target. Returns
2208	/// if it was emitted successfully.
2209	/// \param GD Global to scan.
2210	bool emitTargetGlobal(GlobalDecl GD) override;
2211
2212	/// Emits code for teams call of the \a OutlinedFn with
2213	/// variables captured in a record which address is stored in \a
2214	/// CapturedStruct.
2215	/// \param OutlinedFn Outlined function to be run by team masters. Type of
2216	/// this function is void(*)(kmp_int32 *, kmp_int32, struct context_vars*).
2217	/// \param CapturedVars A pointer to the record with the references to
2218	/// variables used in \a OutlinedFn function.
2219	///
2220	void emitTeamsCall(CodeGenFunction &CGF, const OMPExecutableDirective &D,
2221	SourceLocation Loc, llvm::Function *OutlinedFn,
2222	ArrayRef<llvm::Value *> CapturedVars) override;
2223
2224	/// Emits call to void __kmpc_push_num_teams(ident_t *loc, kmp_int32
2225	/// global_tid, kmp_int32 num_teams, kmp_int32 thread_limit) to generate code
2226	/// for num_teams clause.
2227	/// \param NumTeams An integer expression of teams.
2228	/// \param ThreadLimit An integer expression of threads.
2229	void emitNumTeamsClause(CodeGenFunction &CGF, const Expr *NumTeams,
2230	const Expr *ThreadLimit, SourceLocation Loc) override;
2231
2232	/// Emit the target data mapping code associated with \a D.
2233	/// \param D Directive to emit.
2234	/// \param IfCond Expression evaluated in if clause associated with the
2235	/// target directive, or null if no device clause is used.
2236	/// \param Device Expression evaluated in device clause associated with the
2237	/// target directive, or null if no device clause is used.
2238	/// \param Info A record used to store information that needs to be preserved
2239	/// until the region is closed.
2240	void emitTargetDataCalls(CodeGenFunction &CGF,
2241	const OMPExecutableDirective &D, const Expr *IfCond,
2242	const Expr *Device, const RegionCodeGenTy &CodeGen,
2243	CGOpenMPRuntime::TargetDataInfo &Info) override;
2244
2245	/// Emit the data mapping/movement code associated with the directive
2246	/// \a D that should be of the form 'target [{enter|exit} data | update]'.
2247	/// \param D Directive to emit.
2248	/// \param IfCond Expression evaluated in if clause associated with the target
2249	/// directive, or null if no if clause is used.
2250	/// \param Device Expression evaluated in device clause associated with the
2251	/// target directive, or null if no device clause is used.
2252	void emitTargetDataStandAloneCall(CodeGenFunction &CGF,
2253	const OMPExecutableDirective &D,
2254	const Expr *IfCond,
2255	const Expr *Device) override;
2256
2257	/// Emit initialization for doacross loop nesting support.
2258	/// \param D Loop-based construct used in doacross nesting construct.
2259	void emitDoacrossInit(CodeGenFunction &CGF, const OMPLoopDirective &D,
2260	ArrayRef<Expr *> NumIterations) override;
2261
2262	/// Emit code for doacross ordered directive with 'depend' clause.
2263	/// \param C 'depend' clause with 'sink|source' dependency kind.
2264	void emitDoacrossOrdered(CodeGenFunction &CGF,
2265	const OMPDependClause *C) override;
2266
2267	/// Emit code for doacross ordered directive with 'doacross' clause.
2268	/// \param C 'doacross' clause with 'sink|source' dependence type.
2269	void emitDoacrossOrdered(CodeGenFunction &CGF,
2270	const OMPDoacrossClause *C) override;
2271
2272	/// Translates the native parameter of outlined function if this is required
2273	/// for target.
2274	/// \param FD Field decl from captured record for the parameter.
2275	/// \param NativeParam Parameter itself.
2276	const VarDecl *translateParameter(const FieldDecl *FD,
2277	const VarDecl *NativeParam) const override;
2278
2279	/// Gets the address of the native argument basing on the address of the
2280	/// target-specific parameter.
2281	/// \param NativeParam Parameter itself.
2282	/// \param TargetParam Corresponding target-specific parameter.
2283	Address getParameterAddress(CodeGenFunction &CGF, const VarDecl *NativeParam,
2284	const VarDecl *TargetParam) const override;
2285
2286	/// Gets the OpenMP-specific address of the local variable.
2287	Address getAddressOfLocalVariable(CodeGenFunction &CGF,
2288	const VarDecl *VD) override {
2289	return Address::invalid();
2290	}
2291	};
2292
2293	} // namespace CodeGen
2294	// Utility for openmp doacross clause kind
2295	namespace {
2296	template <typename T> class OMPDoacrossKind {
2297	public:
2298	bool isSink(const T *) { return false; }
2299	bool isSource(const T *) { return false; }
2300	};
2301	template <> class OMPDoacrossKind<OMPDependClause> {
2302	public:
2303	bool isSink(const OMPDependClause *C) {
2304	return C->getDependencyKind() == OMPC_DEPEND_sink;
2305	}
2306	bool isSource(const OMPDependClause *C) {
2307	return C->getDependencyKind() == OMPC_DEPEND_source;
2308	}
2309	};
2310	template <> class OMPDoacrossKind<OMPDoacrossClause> {
2311	public:
2312	bool isSource(const OMPDoacrossClause *C) {
2313	return C->getDependenceType() == OMPC_DOACROSS_source ||
2314	C->getDependenceType() == OMPC_DOACROSS_source_omp_cur_iteration;
2315	}
2316	bool isSink(const OMPDoacrossClause *C) {
2317	return C->getDependenceType() == OMPC_DOACROSS_sink ||
2318	C->getDependenceType() == OMPC_DOACROSS_sink_omp_cur_iteration;
2319	}
2320	};
2321	} // namespace
2322	} // namespace clang
2323
2324	#endif
2325