1	//===----- CGOpenMPRuntime.h - Interface to OpenMP Runtimes -----*- C++ -*-===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This provides a class for OpenMP runtime code generation.
10	//
11	//===----------------------------------------------------------------------===//
12
13	#ifndef LLVM_CLANG_LIB_CODEGEN_CGOPENMPRUNTIME_H
14	#define LLVM_CLANG_LIB_CODEGEN_CGOPENMPRUNTIME_H
15
16	#include "CGValue.h"
17	#include "clang/AST/DeclOpenMP.h"
18	#include "clang/AST/GlobalDecl.h"
19	#include "clang/AST/Type.h"
20	#include "clang/Basic/OpenMPKinds.h"
21	#include "clang/Basic/SourceLocation.h"
22	#include "llvm/ADT/DenseMap.h"
23	#include "llvm/ADT/PointerIntPair.h"
24	#include "llvm/ADT/SmallPtrSet.h"
25	#include "llvm/ADT/StringMap.h"
26	#include "llvm/ADT/StringSet.h"
27	#include "llvm/Frontend/OpenMP/OMPConstants.h"
28	#include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
29	#include "llvm/IR/Function.h"
30	#include "llvm/IR/ValueHandle.h"
31	#include "llvm/Support/AtomicOrdering.h"
32
33	namespace llvm {
34	class ArrayType;
35	class Constant;
36	class FunctionType;
37	class GlobalVariable;
38	class Type;
39	class Value;
40	class OpenMPIRBuilder;
41	} // namespace llvm
42
43	namespace clang {
44	class Expr;
45	class OMPDependClause;
46	class OMPExecutableDirective;
47	class OMPLoopDirective;
48	class VarDecl;
49	class OMPDeclareReductionDecl;
50
51	namespace CodeGen {
52	class Address;
53	class CodeGenFunction;
54	class CodeGenModule;
55
56	/// A basic class for pre|post-action for advanced codegen sequence for OpenMP
57	/// region.
58	class PrePostActionTy {
59	public:
60	explicit PrePostActionTy() {}
61	virtual void Enter(CodeGenFunction &CGF) {}
62	virtual void Exit(CodeGenFunction &CGF) {}
63	virtual ~PrePostActionTy() {}
64	};
65
66	/// Class provides a way to call simple version of codegen for OpenMP region, or
67	/// an advanced with possible pre|post-actions in codegen.
68	class RegionCodeGenTy final {
69	intptr_t CodeGen;
70	typedef void (*CodeGenTy)(intptr_t, CodeGenFunction &, PrePostActionTy &);
71	CodeGenTy Callback;
72	mutable PrePostActionTy *PrePostAction;
73	RegionCodeGenTy() = delete;
74	template <typename Callable>
75	static void CallbackFn(intptr_t CodeGen, CodeGenFunction &CGF,
76	PrePostActionTy &Action) {
77	return (*reinterpret_cast<Callable *>(CodeGen))(CGF, Action);
78	}
79
80	public:
81	template <typename Callable>
82	RegionCodeGenTy(
83	Callable &&CodeGen,
84	std::enable_if_t<!std::is_same<std::remove_reference_t<Callable>,
85	RegionCodeGenTy>::value> * = nullptr)
86	: CodeGen(reinterpret_cast<intptr_t>(&CodeGen)),
87	Callback(CallbackFn<std::remove_reference_t<Callable>>),
88	PrePostAction(nullptr) {}
89	void setAction(PrePostActionTy &Action) const { PrePostAction = &Action; }
90	void operator()(CodeGenFunction &CGF) const;
91	};
92
93	struct OMPTaskDataTy final {
94	SmallVector<const Expr *, 4> PrivateVars;
95	SmallVector<const Expr *, 4> PrivateCopies;
96	SmallVector<const Expr *, 4> FirstprivateVars;
97	SmallVector<const Expr *, 4> FirstprivateCopies;
98	SmallVector<const Expr *, 4> FirstprivateInits;
99	SmallVector<const Expr *, 4> LastprivateVars;
100	SmallVector<const Expr *, 4> LastprivateCopies;
101	SmallVector<const Expr *, 4> ReductionVars;
102	SmallVector<const Expr *, 4> ReductionOrigs;
103	SmallVector<const Expr *, 4> ReductionCopies;
104	SmallVector<const Expr *, 4> ReductionOps;
105	SmallVector<CanonicalDeclPtr<const VarDecl>, 4> PrivateLocals;
106	struct DependData {
107	OpenMPDependClauseKind DepKind = OMPC_DEPEND_unknown;
108	const Expr *IteratorExpr = nullptr;
109	SmallVector<const Expr *, 4> DepExprs;
110	explicit DependData() = default;
111	DependData(OpenMPDependClauseKind DepKind, const Expr *IteratorExpr)
112	: DepKind(DepKind), IteratorExpr(IteratorExpr) {}
113	};
114	SmallVector<DependData, 4> Dependences;
115	llvm::PointerIntPair<llvm::Value *, 1, bool> Final;
116	llvm::PointerIntPair<llvm::Value *, 1, bool> Schedule;
117	llvm::PointerIntPair<llvm::Value *, 1, bool> Priority;
118	llvm::Value *Reductions = nullptr;
119	unsigned NumberOfParts = 0;
120	bool Tied = true;
121	bool Nogroup = false;
122	bool IsReductionWithTaskMod = false;
123	bool IsWorksharingReduction = false;
124	bool HasNowaitClause = false;
125	};
126
127	/// Class intended to support codegen of all kind of the reduction clauses.
128	class ReductionCodeGen {
129	private:
130	/// Data required for codegen of reduction clauses.
131	struct ReductionData {
132	/// Reference to the item shared between tasks to reduce into.
133	const Expr *Shared = nullptr;
134	/// Reference to the original item.
135	const Expr *Ref = nullptr;
136	/// Helper expression for generation of private copy.
137	const Expr *Private = nullptr;
138	/// Helper expression for generation reduction operation.
139	const Expr *ReductionOp = nullptr;
140	ReductionData(const Expr *Shared, const Expr *Ref, const Expr *Private,
141	const Expr *ReductionOp)
142	: Shared(Shared), Ref(Ref), Private(Private), ReductionOp(ReductionOp) {
143	}
144	};
145	/// List of reduction-based clauses.
146	SmallVector<ReductionData, 4> ClausesData;
147
148	/// List of addresses of shared variables/expressions.
149	SmallVector<std::pair<LValue, LValue>, 4> SharedAddresses;
150	/// List of addresses of original variables/expressions.
151	SmallVector<std::pair<LValue, LValue>, 4> OrigAddresses;
152	/// Sizes of the reduction items in chars.
153	SmallVector<std::pair<llvm::Value *, llvm::Value *>, 4> Sizes;
154	/// Base declarations for the reduction items.
155	SmallVector<const VarDecl *, 4> BaseDecls;
156
157	/// Emits lvalue for shared expression.
158	LValue emitSharedLValue(CodeGenFunction &CGF, const Expr *E);
159	/// Emits upper bound for shared expression (if array section).
160	LValue emitSharedLValueUB(CodeGenFunction &CGF, const Expr *E);
161	/// Performs aggregate initialization.
162	/// \param N Number of reduction item in the common list.
163	/// \param PrivateAddr Address of the corresponding private item.
164	/// \param SharedAddr Address of the original shared variable.
165	/// \param DRD Declare reduction construct used for reduction item.
166	void emitAggregateInitialization(CodeGenFunction &CGF, unsigned N,
167	Address PrivateAddr, Address SharedAddr,
168	const OMPDeclareReductionDecl *DRD);
169
170	public:
171	ReductionCodeGen(ArrayRef<const Expr *> Shareds, ArrayRef<const Expr *> Origs,
172	ArrayRef<const Expr *> Privates,
173	ArrayRef<const Expr *> ReductionOps);
174	/// Emits lvalue for the shared and original reduction item.
175	/// \param N Number of the reduction item.
176	void emitSharedOrigLValue(CodeGenFunction &CGF, unsigned N);
177	/// Emits the code for the variable-modified type, if required.
178	/// \param N Number of the reduction item.
179	void emitAggregateType(CodeGenFunction &CGF, unsigned N);
180	/// Emits the code for the variable-modified type, if required.
181	/// \param N Number of the reduction item.
182	/// \param Size Size of the type in chars.
183	void emitAggregateType(CodeGenFunction &CGF, unsigned N, llvm::Value *Size);
184	/// Performs initialization of the private copy for the reduction item.
185	/// \param N Number of the reduction item.
186	/// \param PrivateAddr Address of the corresponding private item.
187	/// \param DefaultInit Default initialization sequence that should be
188	/// performed if no reduction specific initialization is found.
189	/// \param SharedAddr Address of the original shared variable.
190	void
191	emitInitialization(CodeGenFunction &CGF, unsigned N, Address PrivateAddr,
192	Address SharedAddr,
193	llvm::function_ref<bool(CodeGenFunction &)> DefaultInit);
194	/// Returns true if the private copy requires cleanups.
195	bool needCleanups(unsigned N);
196	/// Emits cleanup code for the reduction item.
197	/// \param N Number of the reduction item.
198	/// \param PrivateAddr Address of the corresponding private item.
199	void emitCleanups(CodeGenFunction &CGF, unsigned N, Address PrivateAddr);
200	/// Adjusts \p PrivatedAddr for using instead of the original variable
201	/// address in normal operations.
202	/// \param N Number of the reduction item.
203	/// \param PrivateAddr Address of the corresponding private item.
204	Address adjustPrivateAddress(CodeGenFunction &CGF, unsigned N,
205	Address PrivateAddr);
206	/// Returns LValue for the reduction item.
207	LValue getSharedLValue(unsigned N) const { return SharedAddresses[N].first; }
208	/// Returns LValue for the original reduction item.
209	LValue getOrigLValue(unsigned N) const { return OrigAddresses[N].first; }
210	/// Returns the size of the reduction item (in chars and total number of
211	/// elements in the item), or nullptr, if the size is a constant.
212	std::pair<llvm::Value *, llvm::Value *> getSizes(unsigned N) const {
213	return Sizes[N];
214	}
215	/// Returns the base declaration of the reduction item.
216	const VarDecl *getBaseDecl(unsigned N) const { return BaseDecls[N]; }
217	/// Returns the base declaration of the reduction item.
218	const Expr *getRefExpr(unsigned N) const { return ClausesData[N].Ref; }
219	/// Returns true if the initialization of the reduction item uses initializer
220	/// from declare reduction construct.
221	bool usesReductionInitializer(unsigned N) const;
222	/// Return the type of the private item.
223	QualType getPrivateType(unsigned N) const {
224	return cast<VarDecl>(Val: cast<DeclRefExpr>(Val: ClausesData[N].Private)->getDecl())
225	->getType();
226	}
227	};
228
229	class CGOpenMPRuntime {
230	public:
231	/// Allows to disable automatic handling of functions used in target regions
232	/// as those marked as `omp declare target`.
233	class DisableAutoDeclareTargetRAII {
234	CodeGenModule &CGM;
235	bool SavedShouldMarkAsGlobal = false;
236
237	public:
238	DisableAutoDeclareTargetRAII(CodeGenModule &CGM);
239	~DisableAutoDeclareTargetRAII();
240	};
241
242	/// Manages list of nontemporal decls for the specified directive.
243	class NontemporalDeclsRAII {
244	CodeGenModule &CGM;
245	const bool NeedToPush;
246
247	public:
248	NontemporalDeclsRAII(CodeGenModule &CGM, const OMPLoopDirective &S);
249	~NontemporalDeclsRAII();
250	};
251
252	/// Manages list of nontemporal decls for the specified directive.
253	class UntiedTaskLocalDeclsRAII {
254	CodeGenModule &CGM;
255	const bool NeedToPush;
256
257	public:
258	UntiedTaskLocalDeclsRAII(
259	CodeGenFunction &CGF,
260	const llvm::MapVector<CanonicalDeclPtr<const VarDecl>,
261	std::pair<Address, Address>> &LocalVars);
262	~UntiedTaskLocalDeclsRAII();
263	};
264
265	/// Maps the expression for the lastprivate variable to the global copy used
266	/// to store new value because original variables are not mapped in inner
267	/// parallel regions. Only private copies are captured but we need also to
268	/// store private copy in shared address.
269	/// Also, stores the expression for the private loop counter and it
270	/// threaprivate name.
271	struct LastprivateConditionalData {
272	llvm::MapVector<CanonicalDeclPtr<const Decl>, SmallString<16>>
273	DeclToUniqueName;
274	LValue IVLVal;
275	llvm::Function *Fn = nullptr;
276	bool Disabled = false;
277	};
278	/// Manages list of lastprivate conditional decls for the specified directive.
279	class LastprivateConditionalRAII {
280	enum class ActionToDo {
281	DoNotPush,
282	PushAsLastprivateConditional,
283	DisableLastprivateConditional,
284	};
285	CodeGenModule &CGM;
286	ActionToDo Action = ActionToDo::DoNotPush;
287
288	/// Check and try to disable analysis of inner regions for changes in
289	/// lastprivate conditional.
290	void tryToDisableInnerAnalysis(const OMPExecutableDirective &S,
291	llvm::DenseSet<CanonicalDeclPtr<const Decl>>
292	&NeedToAddForLPCsAsDisabled) const;
293
294	LastprivateConditionalRAII(CodeGenFunction &CGF,
295	const OMPExecutableDirective &S);
296
297	public:
298	explicit LastprivateConditionalRAII(CodeGenFunction &CGF,
299	const OMPExecutableDirective &S,
300	LValue IVLVal);
301	static LastprivateConditionalRAII disable(CodeGenFunction &CGF,
302	const OMPExecutableDirective &S);
303	~LastprivateConditionalRAII();
304	};
305
306	llvm::OpenMPIRBuilder &getOMPBuilder() { return OMPBuilder; }
307
308	protected:
309	CodeGenModule &CGM;
310
311	/// An OpenMP-IR-Builder instance.
312	llvm::OpenMPIRBuilder OMPBuilder;
313
314	/// Helper to determine the min/max number of threads/teams for \p D.
315	void computeMinAndMaxThreadsAndTeams(const OMPExecutableDirective &D,
316	CodeGenFunction &CGF,
317	int32_t &MinThreadsVal,
318	int32_t &MaxThreadsVal,
319	int32_t &MinTeamsVal,
320	int32_t &MaxTeamsVal);
321
322	/// Helper to emit outlined function for 'target' directive.
323	/// \param D Directive to emit.
324	/// \param ParentName Name of the function that encloses the target region.
325	/// \param OutlinedFn Outlined function value to be defined by this call.
326	/// \param OutlinedFnID Outlined function ID value to be defined by this call.
327	/// \param IsOffloadEntry True if the outlined function is an offload entry.
328	/// \param CodeGen Lambda codegen specific to an accelerator device.
329	/// An outlined function may not be an entry if, e.g. the if clause always
330	/// evaluates to false.
331	virtual void emitTargetOutlinedFunctionHelper(const OMPExecutableDirective &D,
332	StringRef ParentName,
333	llvm::Function *&OutlinedFn,
334	llvm::Constant *&OutlinedFnID,
335	bool IsOffloadEntry,
336	const RegionCodeGenTy &CodeGen);
337
338	/// Returns pointer to ident_t type.
339	llvm::Type *getIdentTyPointerTy();
340
341	/// Gets thread id value for the current thread.
342	///
343	llvm::Value *getThreadID(CodeGenFunction &CGF, SourceLocation Loc);
344
345	/// Get the function name of an outlined region.
346	std::string getOutlinedHelperName(StringRef Name) const;
347	std::string getOutlinedHelperName(CodeGenFunction &CGF) const;
348
349	/// Get the function name of a reduction function.
350	std::string getReductionFuncName(StringRef Name) const;
351
352	/// Emits \p Callee function call with arguments \p Args with location \p Loc.
353	void emitCall(CodeGenFunction &CGF, SourceLocation Loc,
354	llvm::FunctionCallee Callee,
355	ArrayRef<llvm::Value *> Args = std::nullopt) const;
356
357	/// Emits address of the word in a memory where current thread id is
358	/// stored.
359	virtual Address emitThreadIDAddress(CodeGenFunction &CGF, SourceLocation Loc);
360
361	void setLocThreadIdInsertPt(CodeGenFunction &CGF,
362	bool AtCurrentPoint = false);
363	void clearLocThreadIdInsertPt(CodeGenFunction &CGF);
364
365	/// Check if the default location must be constant.
366	/// Default is false to support OMPT/OMPD.
367	virtual bool isDefaultLocationConstant() const { return false; }
368
369	/// Returns additional flags that can be stored in reserved_2 field of the
370	/// default location.
371	virtual unsigned getDefaultLocationReserved2Flags() const { return 0; }
372
373	/// Returns default flags for the barriers depending on the directive, for
374	/// which this barier is going to be emitted.
375	static unsigned getDefaultFlagsForBarriers(OpenMPDirectiveKind Kind);
376
377	/// Get the LLVM type for the critical name.
378	llvm::ArrayType *getKmpCriticalNameTy() const {return KmpCriticalNameTy;}
379
380	/// Returns corresponding lock object for the specified critical region
381	/// name. If the lock object does not exist it is created, otherwise the
382	/// reference to the existing copy is returned.
383	/// \param CriticalName Name of the critical region.
384	///
385	llvm::Value *getCriticalRegionLock(StringRef CriticalName);
386
387	protected:
388	/// Map for SourceLocation and OpenMP runtime library debug locations.
389	typedef llvm::DenseMap<SourceLocation, llvm::Value *> OpenMPDebugLocMapTy;
390	OpenMPDebugLocMapTy OpenMPDebugLocMap;
391	/// The type for a microtask which gets passed to __kmpc_fork_call().
392	/// Original representation is:
393	/// typedef void (kmpc_micro)(kmp_int32 global_tid, kmp_int32 bound_tid,...);
394	llvm::FunctionType *Kmpc_MicroTy = nullptr;
395	/// Stores debug location and ThreadID for the function.
396	struct DebugLocThreadIdTy {
397	llvm::Value *DebugLoc;
398	llvm::Value *ThreadID;
399	/// Insert point for the service instructions.
400	llvm::AssertingVH<llvm::Instruction> ServiceInsertPt = nullptr;
401	};
402	/// Map of local debug location, ThreadId and functions.
403	typedef llvm::DenseMap<llvm::Function *, DebugLocThreadIdTy>
404	OpenMPLocThreadIDMapTy;
405	OpenMPLocThreadIDMapTy OpenMPLocThreadIDMap;
406	/// Map of UDRs and corresponding combiner/initializer.
407	typedef llvm::DenseMap<const OMPDeclareReductionDecl *,
408	std::pair<llvm::Function *, llvm::Function *>>
409	UDRMapTy;
410	UDRMapTy UDRMap;
411	/// Map of functions and locally defined UDRs.
412	typedef llvm::DenseMap<llvm::Function *,
413	SmallVector<const OMPDeclareReductionDecl *, 4>>
414	FunctionUDRMapTy;
415	FunctionUDRMapTy FunctionUDRMap;
416	/// Map from the user-defined mapper declaration to its corresponding
417	/// functions.
418	llvm::DenseMap<const OMPDeclareMapperDecl *, llvm::Function *> UDMMap;
419	/// Map of functions and their local user-defined mappers.
420	using FunctionUDMMapTy =
421	llvm::DenseMap<llvm::Function *,
422	SmallVector<const OMPDeclareMapperDecl *, 4>>;
423	FunctionUDMMapTy FunctionUDMMap;
424	/// Maps local variables marked as lastprivate conditional to their internal
425	/// types.
426	llvm::DenseMap<llvm::Function *,
427	llvm::DenseMap<CanonicalDeclPtr<const Decl>,
428	std::tuple<QualType, const FieldDecl *,
429	const FieldDecl *, LValue>>>
430	LastprivateConditionalToTypes;
431	/// Maps function to the position of the untied task locals stack.
432	llvm::DenseMap<llvm::Function *, unsigned> FunctionToUntiedTaskStackMap;
433	/// Type kmp_critical_name, originally defined as typedef kmp_int32
434	/// kmp_critical_name[8];
435	llvm::ArrayType *KmpCriticalNameTy;
436	/// An ordered map of auto-generated variables to their unique names.
437	/// It stores variables with the following names: 1) ".gomp_critical_user_" +
438	/// <critical_section_name> + ".var" for "omp critical" directives; 2)
439	/// <mangled_name_for_global_var> + ".cache." for cache for threadprivate
440	/// variables.
441	llvm::StringMap<llvm::AssertingVH<llvm::GlobalVariable>,
442	llvm::BumpPtrAllocator> InternalVars;
443	/// Type typedef kmp_int32 (* kmp_routine_entry_t)(kmp_int32, void *);
444	llvm::Type *KmpRoutineEntryPtrTy = nullptr;
445	QualType KmpRoutineEntryPtrQTy;
446	/// Type typedef struct kmp_task {
447	/// void * shareds; /**< pointer to block of pointers to
448	/// shared vars */
449	/// kmp_routine_entry_t routine; /**< pointer to routine to call for
450	/// executing task */
451	/// kmp_int32 part_id; /**< part id for the task */
452	/// kmp_routine_entry_t destructors; /* pointer to function to invoke
453	/// deconstructors of firstprivate C++ objects */
454	/// } kmp_task_t;
455	QualType KmpTaskTQTy;
456	/// Saved kmp_task_t for task directive.
457	QualType SavedKmpTaskTQTy;
458	/// Saved kmp_task_t for taskloop-based directive.
459	QualType SavedKmpTaskloopTQTy;
460	/// Type typedef struct kmp_depend_info {
461	/// kmp_intptr_t base_addr;
462	/// size_t len;
463	/// struct {
464	/// bool in:1;
465	/// bool out:1;
466	/// } flags;
467	/// } kmp_depend_info_t;
468	QualType KmpDependInfoTy;
469	/// Type typedef struct kmp_task_affinity_info {
470	/// kmp_intptr_t base_addr;
471	/// size_t len;
472	/// struct {
473	/// bool flag1 : 1;
474	/// bool flag2 : 1;
475	/// kmp_int32 reserved : 30;
476	/// } flags;
477	/// } kmp_task_affinity_info_t;
478	QualType KmpTaskAffinityInfoTy;
479	/// struct kmp_dim { // loop bounds info casted to kmp_int64
480	/// kmp_int64 lo; // lower
481	/// kmp_int64 up; // upper
482	/// kmp_int64 st; // stride
483	/// };
484	QualType KmpDimTy;
485
486	bool ShouldMarkAsGlobal = true;
487	/// List of the emitted declarations.
488	llvm::DenseSet<CanonicalDeclPtr<const Decl>> AlreadyEmittedTargetDecls;
489	/// List of the global variables with their addresses that should not be
490	/// emitted for the target.
491	llvm::StringMap<llvm::WeakTrackingVH> EmittedNonTargetVariables;
492
493	/// List of variables that can become declare target implicitly and, thus,
494	/// must be emitted.
495	llvm::SmallDenseSet<const VarDecl *> DeferredGlobalVariables;
496
497	using NontemporalDeclsSet = llvm::SmallDenseSet<CanonicalDeclPtr<const Decl>>;
498	/// Stack for list of declarations in current context marked as nontemporal.
499	/// The set is the union of all current stack elements.
500	llvm::SmallVector<NontemporalDeclsSet, 4> NontemporalDeclsStack;
501
502	using UntiedLocalVarsAddressesMap =
503	llvm::MapVector<CanonicalDeclPtr<const VarDecl>,
504	std::pair<Address, Address>>;
505	llvm::SmallVector<UntiedLocalVarsAddressesMap, 4> UntiedLocalVarsStack;
506
507	/// Stack for list of addresses of declarations in current context marked as
508	/// lastprivate conditional. The set is the union of all current stack
509	/// elements.
510	llvm::SmallVector<LastprivateConditionalData, 4> LastprivateConditionalStack;
511
512	/// Flag for keeping track of weather a requires unified_shared_memory
513	/// directive is present.
514	bool HasRequiresUnifiedSharedMemory = false;
515
516	/// Atomic ordering from the omp requires directive.
517	llvm::AtomicOrdering RequiresAtomicOrdering = llvm::AtomicOrdering::Monotonic;
518
519	/// Flag for keeping track of weather a target region has been emitted.
520	bool HasEmittedTargetRegion = false;
521
522	/// Flag for keeping track of weather a device routine has been emitted.
523	/// Device routines are specific to the
524	bool HasEmittedDeclareTargetRegion = false;
525
526	/// Start scanning from statement \a S and emit all target regions
527	/// found along the way.
528	/// \param S Starting statement.
529	/// \param ParentName Name of the function declaration that is being scanned.
530	void scanForTargetRegionsFunctions(const Stmt *S, StringRef ParentName);
531
532	/// Build type kmp_routine_entry_t (if not built yet).
533	void emitKmpRoutineEntryT(QualType KmpInt32Ty);
534
535	/// Returns pointer to kmpc_micro type.
536	llvm::Type *getKmpc_MicroPointerTy();
537
538	/// If the specified mangled name is not in the module, create and
539	/// return threadprivate cache object. This object is a pointer's worth of
540	/// storage that's reserved for use by the OpenMP runtime.
541	/// \param VD Threadprivate variable.
542	/// \return Cache variable for the specified threadprivate.
543	llvm::Constant *getOrCreateThreadPrivateCache(const VarDecl *VD);
544
545	/// Set of threadprivate variables with the generated initializer.
546	llvm::StringSet<> ThreadPrivateWithDefinition;
547
548	/// Set of declare target variables with the generated initializer.
549	llvm::StringSet<> DeclareTargetWithDefinition;
550
551	/// Emits initialization code for the threadprivate variables.
552	/// \param VDAddr Address of the global variable \a VD.
553	/// \param Ctor Pointer to a global init function for \a VD.
554	/// \param CopyCtor Pointer to a global copy function for \a VD.
555	/// \param Dtor Pointer to a global destructor function for \a VD.
556	/// \param Loc Location of threadprivate declaration.
557	void emitThreadPrivateVarInit(CodeGenFunction &CGF, Address VDAddr,
558	llvm::Value *Ctor, llvm::Value *CopyCtor,
559	llvm::Value *Dtor, SourceLocation Loc);
560
561	/// Emit the array initialization or deletion portion for user-defined mapper
562	/// code generation.
563	void emitUDMapperArrayInitOrDel(CodeGenFunction &MapperCGF,
564	llvm::Value *Handle, llvm::Value *BasePtr,
565	llvm::Value *Ptr, llvm::Value *Size,
566	llvm::Value *MapType, llvm::Value *MapName,
567	CharUnits ElementSize,
568	llvm::BasicBlock *ExitBB, bool IsInit);
569
570	struct TaskResultTy {
571	llvm::Value *NewTask = nullptr;
572	llvm::Function *TaskEntry = nullptr;
573	llvm::Value *NewTaskNewTaskTTy = nullptr;
574	LValue TDBase;
575	const RecordDecl *KmpTaskTQTyRD = nullptr;
576	llvm::Value *TaskDupFn = nullptr;
577	};
578	/// Emit task region for the task directive. The task region is emitted in
579	/// several steps:
580	/// 1. Emit a call to kmp_task_t *__kmpc_omp_task_alloc(ident_t *, kmp_int32
581	/// gtid, kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
582	/// kmp_routine_entry_t *task_entry). Here task_entry is a pointer to the
583	/// function:
584	/// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) {
585	/// TaskFunction(gtid, tt->part_id, tt->shareds);
586	/// return 0;
587	/// }
588	/// 2. Copy a list of shared variables to field shareds of the resulting
589	/// structure kmp_task_t returned by the previous call (if any).
590	/// 3. Copy a pointer to destructions function to field destructions of the
591	/// resulting structure kmp_task_t.
592	/// \param D Current task directive.
593	/// \param TaskFunction An LLVM function with type void (*)(i32 /*gtid*/, i32
594	/// /*part_id*/, captured_struct */*__context*/);
595	/// \param SharedsTy A type which contains references the shared variables.
596	/// \param Shareds Context with the list of shared variables from the \p
597	/// TaskFunction.
598	/// \param Data Additional data for task generation like tiednsee, final
599	/// state, list of privates etc.
600	TaskResultTy emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc,
601	const OMPExecutableDirective &D,
602	llvm::Function *TaskFunction, QualType SharedsTy,
603	Address Shareds, const OMPTaskDataTy &Data);
604
605	/// Emit update for lastprivate conditional data.
606	void emitLastprivateConditionalUpdate(CodeGenFunction &CGF, LValue IVLVal,
607	StringRef UniqueDeclName, LValue LVal,
608	SourceLocation Loc);
609
610	/// Returns the number of the elements and the address of the depobj
611	/// dependency array.
612	/// \return Number of elements in depobj array and the pointer to the array of
613	/// dependencies.
614	std::pair<llvm::Value *, LValue> getDepobjElements(CodeGenFunction &CGF,
615	LValue DepobjLVal,
616	SourceLocation Loc);
617
618	SmallVector<llvm::Value *, 4>
619	emitDepobjElementsSizes(CodeGenFunction &CGF, QualType &KmpDependInfoTy,
620	const OMPTaskDataTy::DependData &Data);
621
622	void emitDepobjElements(CodeGenFunction &CGF, QualType &KmpDependInfoTy,
623	LValue PosLVal, const OMPTaskDataTy::DependData &Data,
624	Address DependenciesArray);
625
626	public:
627	explicit CGOpenMPRuntime(CodeGenModule &CGM);
628	virtual ~CGOpenMPRuntime() {}
629	virtual void clear();
630
631	/// Emits object of ident_t type with info for source location.
632	/// \param Flags Flags for OpenMP location.
633	/// \param EmitLoc emit source location with debug-info is off.
634	///
635	llvm::Value *emitUpdateLocation(CodeGenFunction &CGF, SourceLocation Loc,
636	unsigned Flags = 0, bool EmitLoc = false);
637
638	/// Emit the number of teams for a target directive. Inspect the num_teams
639	/// clause associated with a teams construct combined or closely nested
640	/// with the target directive.
641	///
642	/// Emit a team of size one for directives such as 'target parallel' that
643	/// have no associated teams construct.
644	///
645	/// Otherwise, return nullptr.
646	const Expr *getNumTeamsExprForTargetDirective(CodeGenFunction &CGF,
647	const OMPExecutableDirective &D,
648	int32_t &MinTeamsVal,
649	int32_t &MaxTeamsVal);
650	llvm::Value *emitNumTeamsForTargetDirective(CodeGenFunction &CGF,
651	const OMPExecutableDirective &D);
652
653	/// Check for a number of threads upper bound constant value (stored in \p
654	/// UpperBound), or expression (returned). If the value is conditional (via an
655	/// if-clause), store the condition in \p CondExpr. Similarly, a potential
656	/// thread limit expression is stored in \p ThreadLimitExpr. If \p
657	/// UpperBoundOnly is true, no expression evaluation is perfomed.
658	const Expr *getNumThreadsExprForTargetDirective(
659	CodeGenFunction &CGF, const OMPExecutableDirective &D,
660	int32_t &UpperBound, bool UpperBoundOnly,
661	llvm::Value **CondExpr = nullptr, const Expr **ThreadLimitExpr = nullptr);
662
663	/// Emit an expression that denotes the number of threads a target region
664	/// shall use. Will generate "i32 0" to allow the runtime to choose.
665	llvm::Value *
666	emitNumThreadsForTargetDirective(CodeGenFunction &CGF,
667	const OMPExecutableDirective &D);
668
669	/// Return the trip count of loops associated with constructs / 'target teams
670	/// distribute' and 'teams distribute parallel for'. \param SizeEmitter Emits
671	/// the int64 value for the number of iterations of the associated loop.
672	llvm::Value *emitTargetNumIterationsCall(
673	CodeGenFunction &CGF, const OMPExecutableDirective &D,
674	llvm::function_ref<llvm::Value *(CodeGenFunction &CGF,
675	const OMPLoopDirective &D)>
676	SizeEmitter);
677
678	/// Returns true if the current target is a GPU.
679	virtual bool isGPU() const { return false; }
680
681	/// Check if the variable length declaration is delayed:
682	virtual bool isDelayedVariableLengthDecl(CodeGenFunction &CGF,
683	const VarDecl *VD) const {
684	return false;
685	};
686
687	/// Get call to __kmpc_alloc_shared
688	virtual std::pair<llvm::Value *, llvm::Value *>
689	getKmpcAllocShared(CodeGenFunction &CGF, const VarDecl *VD) {
690	llvm_unreachable("not implemented");
691	}
692
693	/// Get call to __kmpc_free_shared
694	virtual void getKmpcFreeShared(
695	CodeGenFunction &CGF,
696	const std::pair<llvm::Value *, llvm::Value *> &AddrSizePair) {
697	llvm_unreachable("not implemented");
698	}
699
700	/// Emits code for OpenMP 'if' clause using specified \a CodeGen
701	/// function. Here is the logic:
702	/// if (Cond) {
703	/// ThenGen();
704	/// } else {
705	/// ElseGen();
706	/// }
707	void emitIfClause(CodeGenFunction &CGF, const Expr *Cond,
708	const RegionCodeGenTy &ThenGen,
709	const RegionCodeGenTy &ElseGen);
710
711	/// Checks if the \p Body is the \a CompoundStmt and returns its child
712	/// statement iff there is only one that is not evaluatable at the compile
713	/// time.
714	static const Stmt *getSingleCompoundChild(ASTContext &Ctx, const Stmt *Body);
715
716	/// Get the platform-specific name separator.
717	std::string getName(ArrayRef<StringRef> Parts) const;
718
719	/// Emit code for the specified user defined reduction construct.
720	virtual void emitUserDefinedReduction(CodeGenFunction *CGF,
721	const OMPDeclareReductionDecl *D);
722	/// Get combiner/initializer for the specified user-defined reduction, if any.
723	virtual std::pair<llvm::Function *, llvm::Function *>
724	getUserDefinedReduction(const OMPDeclareReductionDecl *D);
725
726	/// Emit the function for the user defined mapper construct.
727	void emitUserDefinedMapper(const OMPDeclareMapperDecl *D,
728	CodeGenFunction *CGF = nullptr);
729	/// Get the function for the specified user-defined mapper. If it does not
730	/// exist, create one.
731	llvm::Function *
732	getOrCreateUserDefinedMapperFunc(const OMPDeclareMapperDecl *D);
733
734	/// Emits outlined function for the specified OpenMP parallel directive
735	/// \a D. This outlined function has type void(*)(kmp_int32 *ThreadID,
736	/// kmp_int32 BoundID, struct context_vars*).
737	/// \param CGF Reference to current CodeGenFunction.
738	/// \param D OpenMP directive.
739	/// \param ThreadIDVar Variable for thread id in the current OpenMP region.
740	/// \param InnermostKind Kind of innermost directive (for simple directives it
741	/// is a directive itself, for combined - its innermost directive).
742	/// \param CodeGen Code generation sequence for the \a D directive.
743	virtual llvm::Function *emitParallelOutlinedFunction(
744	CodeGenFunction &CGF, const OMPExecutableDirective &D,
745	const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,
746	const RegionCodeGenTy &CodeGen);
747
748	/// Emits outlined function for the specified OpenMP teams directive
749	/// \a D. This outlined function has type void(*)(kmp_int32 *ThreadID,
750	/// kmp_int32 BoundID, struct context_vars*).
751	/// \param CGF Reference to current CodeGenFunction.
752	/// \param D OpenMP directive.
753	/// \param ThreadIDVar Variable for thread id in the current OpenMP region.
754	/// \param InnermostKind Kind of innermost directive (for simple directives it
755	/// is a directive itself, for combined - its innermost directive).
756	/// \param CodeGen Code generation sequence for the \a D directive.
757	virtual llvm::Function *emitTeamsOutlinedFunction(
758	CodeGenFunction &CGF, const OMPExecutableDirective &D,
759	const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,
760	const RegionCodeGenTy &CodeGen);
761
762	/// Emits outlined function for the OpenMP task directive \a D. This
763	/// outlined function has type void(*)(kmp_int32 ThreadID, struct task_t*
764	/// TaskT).
765	/// \param D OpenMP directive.
766	/// \param ThreadIDVar Variable for thread id in the current OpenMP region.
767	/// \param PartIDVar Variable for partition id in the current OpenMP untied
768	/// task region.
769	/// \param TaskTVar Variable for task_t argument.
770	/// \param InnermostKind Kind of innermost directive (for simple directives it
771	/// is a directive itself, for combined - its innermost directive).
772	/// \param CodeGen Code generation sequence for the \a D directive.
773	/// \param Tied true if task is generated for tied task, false otherwise.
774	/// \param NumberOfParts Number of parts in untied task. Ignored for tied
775	/// tasks.
776	///
777	virtual llvm::Function *emitTaskOutlinedFunction(
778	const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
779	const VarDecl *PartIDVar, const VarDecl *TaskTVar,
780	OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
781	bool Tied, unsigned &NumberOfParts);
782
783	/// Cleans up references to the objects in finished function.
784	///
785	virtual void functionFinished(CodeGenFunction &CGF);
786
787	/// Emits code for parallel or serial call of the \a OutlinedFn with
788	/// variables captured in a record which address is stored in \a
789	/// CapturedStruct.
790	/// \param OutlinedFn Outlined function to be run in parallel threads. Type of
791	/// this function is void(*)(kmp_int32 *, kmp_int32, struct context_vars*).
792	/// \param CapturedVars A pointer to the record with the references to
793	/// variables used in \a OutlinedFn function.
794	/// \param IfCond Condition in the associated 'if' clause, if it was
795	/// specified, nullptr otherwise.
796	/// \param NumThreads The value corresponding to the num_threads clause, if
797	/// any, or nullptr.
798	///
799	virtual void emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc,
800	llvm::Function *OutlinedFn,
801	ArrayRef<llvm::Value *> CapturedVars,
802	const Expr *IfCond, llvm::Value *NumThreads);
803
804	/// Emits a critical region.
805	/// \param CriticalName Name of the critical region.
806	/// \param CriticalOpGen Generator for the statement associated with the given
807	/// critical region.
808	/// \param Hint Value of the 'hint' clause (optional).
809	virtual void emitCriticalRegion(CodeGenFunction &CGF, StringRef CriticalName,
810	const RegionCodeGenTy &CriticalOpGen,
811	SourceLocation Loc,
812	const Expr *Hint = nullptr);
813
814	/// Emits a master region.
815	/// \param MasterOpGen Generator for the statement associated with the given
816	/// master region.
817	virtual void emitMasterRegion(CodeGenFunction &CGF,
818	const RegionCodeGenTy &MasterOpGen,
819	SourceLocation Loc);
820
821	/// Emits a masked region.
822	/// \param MaskedOpGen Generator for the statement associated with the given
823	/// masked region.
824	virtual void emitMaskedRegion(CodeGenFunction &CGF,
825	const RegionCodeGenTy &MaskedOpGen,
826	SourceLocation Loc,
827	const Expr *Filter = nullptr);
828
829	/// Emits code for a taskyield directive.
830	virtual void emitTaskyieldCall(CodeGenFunction &CGF, SourceLocation Loc);
831
832	/// Emit __kmpc_error call for error directive
833	/// extern void __kmpc_error(ident_t *loc, int severity, const char *message);
834	virtual void emitErrorCall(CodeGenFunction &CGF, SourceLocation Loc, Expr *ME,
835	bool IsFatal);
836
837	/// Emit a taskgroup region.
838	/// \param TaskgroupOpGen Generator for the statement associated with the
839	/// given taskgroup region.
840	virtual void emitTaskgroupRegion(CodeGenFunction &CGF,
841	const RegionCodeGenTy &TaskgroupOpGen,
842	SourceLocation Loc);
843
844	/// Emits a single region.
845	/// \param SingleOpGen Generator for the statement associated with the given
846	/// single region.
847	virtual void emitSingleRegion(CodeGenFunction &CGF,
848	const RegionCodeGenTy &SingleOpGen,
849	SourceLocation Loc,
850	ArrayRef<const Expr *> CopyprivateVars,
851	ArrayRef<const Expr *> DestExprs,
852	ArrayRef<const Expr *> SrcExprs,
853	ArrayRef<const Expr *> AssignmentOps);
854
855	/// Emit an ordered region.
856	/// \param OrderedOpGen Generator for the statement associated with the given
857	/// ordered region.
858	virtual void emitOrderedRegion(CodeGenFunction &CGF,
859	const RegionCodeGenTy &OrderedOpGen,
860	SourceLocation Loc, bool IsThreads);
861
862	/// Emit an implicit/explicit barrier for OpenMP threads.
863	/// \param Kind Directive for which this implicit barrier call must be
864	/// generated. Must be OMPD_barrier for explicit barrier generation.
865	/// \param EmitChecks true if need to emit checks for cancellation barriers.
866	/// \param ForceSimpleCall true simple barrier call must be emitted, false if
867	/// runtime class decides which one to emit (simple or with cancellation
868	/// checks).
869	///
870	virtual void emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc,
871	OpenMPDirectiveKind Kind,
872	bool EmitChecks = true,
873	bool ForceSimpleCall = false);
874
875	/// Check if the specified \a ScheduleKind is static non-chunked.
876	/// This kind of worksharing directive is emitted without outer loop.
877	/// \param ScheduleKind Schedule kind specified in the 'schedule' clause.
878	/// \param Chunked True if chunk is specified in the clause.
879	///
880	virtual bool isStaticNonchunked(OpenMPScheduleClauseKind ScheduleKind,
881	bool Chunked) const;
882
883	/// Check if the specified \a ScheduleKind is static non-chunked.
884	/// This kind of distribute directive is emitted without outer loop.
885	/// \param ScheduleKind Schedule kind specified in the 'dist_schedule' clause.
886	/// \param Chunked True if chunk is specified in the clause.
887	///
888	virtual bool isStaticNonchunked(OpenMPDistScheduleClauseKind ScheduleKind,
889	bool Chunked) const;
890
891	/// Check if the specified \a ScheduleKind is static chunked.
892	/// \param ScheduleKind Schedule kind specified in the 'schedule' clause.
893	/// \param Chunked True if chunk is specified in the clause.
894	///
895	virtual bool isStaticChunked(OpenMPScheduleClauseKind ScheduleKind,
896	bool Chunked) const;
897
898	/// Check if the specified \a ScheduleKind is static non-chunked.
899	/// \param ScheduleKind Schedule kind specified in the 'dist_schedule' clause.
900	/// \param Chunked True if chunk is specified in the clause.
901	///
902	virtual bool isStaticChunked(OpenMPDistScheduleClauseKind ScheduleKind,
903	bool Chunked) const;
904
905	/// Check if the specified \a ScheduleKind is dynamic.
906	/// This kind of worksharing directive is emitted without outer loop.
907	/// \param ScheduleKind Schedule Kind specified in the 'schedule' clause.
908	///
909	virtual bool isDynamic(OpenMPScheduleClauseKind ScheduleKind) const;
910
911	/// struct with the values to be passed to the dispatch runtime function
912	struct DispatchRTInput {
913	/// Loop lower bound
914	llvm::Value *LB = nullptr;
915	/// Loop upper bound
916	llvm::Value *UB = nullptr;
917	/// Chunk size specified using 'schedule' clause (nullptr if chunk
918	/// was not specified)
919	llvm::Value *Chunk = nullptr;
920	DispatchRTInput() = default;
921	DispatchRTInput(llvm::Value *LB, llvm::Value *UB, llvm::Value *Chunk)
922	: LB(LB), UB(UB), Chunk(Chunk) {}
923	};
924
925	/// Call the appropriate runtime routine to initialize it before start
926	/// of loop.
927
928	/// This is used for non static scheduled types and when the ordered
929	/// clause is present on the loop construct.
930	/// Depending on the loop schedule, it is necessary to call some runtime
931	/// routine before start of the OpenMP loop to get the loop upper / lower
932	/// bounds \a LB and \a UB and stride \a ST.
933	///
934	/// \param CGF Reference to current CodeGenFunction.
935	/// \param Loc Clang source location.
936	/// \param ScheduleKind Schedule kind, specified by the 'schedule' clause.
937	/// \param IVSize Size of the iteration variable in bits.
938	/// \param IVSigned Sign of the iteration variable.
939	/// \param Ordered true if loop is ordered, false otherwise.
940	/// \param DispatchValues struct containing llvm values for lower bound, upper
941	/// bound, and chunk expression.
942	/// For the default (nullptr) value, the chunk 1 will be used.
943	///
944	virtual void emitForDispatchInit(CodeGenFunction &CGF, SourceLocation Loc,
945	const OpenMPScheduleTy &ScheduleKind,
946	unsigned IVSize, bool IVSigned, bool Ordered,
947	const DispatchRTInput &DispatchValues);
948
949	/// Struct with the values to be passed to the static runtime function
950	struct StaticRTInput {
951	/// Size of the iteration variable in bits.
952	unsigned IVSize = 0;
953	/// Sign of the iteration variable.
954	bool IVSigned = false;
955	/// true if loop is ordered, false otherwise.
956	bool Ordered = false;
957	/// Address of the output variable in which the flag of the last iteration
958	/// is returned.
959	Address IL = Address::invalid();
960	/// Address of the output variable in which the lower iteration number is
961	/// returned.
962	Address LB = Address::invalid();
963	/// Address of the output variable in which the upper iteration number is
964	/// returned.
965	Address UB = Address::invalid();
966	/// Address of the output variable in which the stride value is returned
967	/// necessary to generated the static_chunked scheduled loop.
968	Address ST = Address::invalid();
969	/// Value of the chunk for the static_chunked scheduled loop. For the
970	/// default (nullptr) value, the chunk 1 will be used.
971	llvm::Value *Chunk = nullptr;
972	StaticRTInput(unsigned IVSize, bool IVSigned, bool Ordered, Address IL,
973	Address LB, Address UB, Address ST,
974	llvm::Value *Chunk = nullptr)
975	: IVSize(IVSize), IVSigned(IVSigned), Ordered(Ordered), IL(IL), LB(LB),
976	UB(UB), ST(ST), Chunk(Chunk) {}
977	};
978	/// Call the appropriate runtime routine to initialize it before start
979	/// of loop.
980	///
981	/// This is used only in case of static schedule, when the user did not
982	/// specify a ordered clause on the loop construct.
983	/// Depending on the loop schedule, it is necessary to call some runtime
984	/// routine before start of the OpenMP loop to get the loop upper / lower
985	/// bounds LB and UB and stride ST.
986	///
987	/// \param CGF Reference to current CodeGenFunction.
988	/// \param Loc Clang source location.
989	/// \param DKind Kind of the directive.
990	/// \param ScheduleKind Schedule kind, specified by the 'schedule' clause.
991	/// \param Values Input arguments for the construct.
992	///
993	virtual void emitForStaticInit(CodeGenFunction &CGF, SourceLocation Loc,
994	OpenMPDirectiveKind DKind,
995	const OpenMPScheduleTy &ScheduleKind,
996	const StaticRTInput &Values);
997
998	///
999	/// \param CGF Reference to current CodeGenFunction.
1000	/// \param Loc Clang source location.
1001	/// \param SchedKind Schedule kind, specified by the 'dist_schedule' clause.
1002	/// \param Values Input arguments for the construct.
1003	///
1004	virtual void emitDistributeStaticInit(CodeGenFunction &CGF,
1005	SourceLocation Loc,
1006	OpenMPDistScheduleClauseKind SchedKind,
1007	const StaticRTInput &Values);
1008
1009	/// Call the appropriate runtime routine to notify that we finished
1010	/// iteration of the ordered loop with the dynamic scheduling.
1011	///
1012	/// \param CGF Reference to current CodeGenFunction.
1013	/// \param Loc Clang source location.
1014	/// \param IVSize Size of the iteration variable in bits.
1015	/// \param IVSigned Sign of the iteration variable.
1016	///
1017	virtual void emitForOrderedIterationEnd(CodeGenFunction &CGF,
1018	SourceLocation Loc, unsigned IVSize,
1019	bool IVSigned);
1020
1021	/// Call the appropriate runtime routine to notify that we finished
1022	/// all the work with current loop.
1023	///
1024	/// \param CGF Reference to current CodeGenFunction.
1025	/// \param Loc Clang source location.
1026	/// \param DKind Kind of the directive for which the static finish is emitted.
1027	///
1028	virtual void emitForStaticFinish(CodeGenFunction &CGF, SourceLocation Loc,
1029	OpenMPDirectiveKind DKind);
1030
1031	/// Call __kmpc_dispatch_next(
1032	/// ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter,
1033	/// kmp_int[32|64] *p_lower, kmp_int[32|64] *p_upper,
1034	/// kmp_int[32|64] *p_stride);
1035	/// \param IVSize Size of the iteration variable in bits.
1036	/// \param IVSigned Sign of the iteration variable.
1037	/// \param IL Address of the output variable in which the flag of the
1038	/// last iteration is returned.
1039	/// \param LB Address of the output variable in which the lower iteration
1040	/// number is returned.
1041	/// \param UB Address of the output variable in which the upper iteration
1042	/// number is returned.
1043	/// \param ST Address of the output variable in which the stride value is
1044	/// returned.
1045	virtual llvm::Value *emitForNext(CodeGenFunction &CGF, SourceLocation Loc,
1046	unsigned IVSize, bool IVSigned,
1047	Address IL, Address LB,
1048	Address UB, Address ST);
1049
1050	/// Emits call to void __kmpc_push_num_threads(ident_t *loc, kmp_int32
1051	/// global_tid, kmp_int32 num_threads) to generate code for 'num_threads'
1052	/// clause.
1053	/// \param NumThreads An integer value of threads.
1054	virtual void emitNumThreadsClause(CodeGenFunction &CGF,
1055	llvm::Value *NumThreads,
1056	SourceLocation Loc);
1057
1058	/// Emit call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32
1059	/// global_tid, int proc_bind) to generate code for 'proc_bind' clause.
1060	virtual void emitProcBindClause(CodeGenFunction &CGF,
1061	llvm::omp::ProcBindKind ProcBind,
1062	SourceLocation Loc);
1063
1064	/// Returns address of the threadprivate variable for the current
1065	/// thread.
1066	/// \param VD Threadprivate variable.
1067	/// \param VDAddr Address of the global variable \a VD.
1068	/// \param Loc Location of the reference to threadprivate var.
1069	/// \return Address of the threadprivate variable for the current thread.
1070	virtual Address getAddrOfThreadPrivate(CodeGenFunction &CGF,
1071	const VarDecl *VD, Address VDAddr,
1072	SourceLocation Loc);
1073
1074	/// Returns the address of the variable marked as declare target with link
1075	/// clause OR as declare target with to clause and unified memory.
1076	virtual ConstantAddress getAddrOfDeclareTargetVar(const VarDecl *VD);
1077
1078	/// Emit a code for initialization of threadprivate variable. It emits
1079	/// a call to runtime library which adds initial value to the newly created
1080	/// threadprivate variable (if it is not constant) and registers destructor
1081	/// for the variable (if any).
1082	/// \param VD Threadprivate variable.
1083	/// \param VDAddr Address of the global variable \a VD.
1084	/// \param Loc Location of threadprivate declaration.
1085	/// \param PerformInit true if initialization expression is not constant.
1086	virtual llvm::Function *
1087	emitThreadPrivateVarDefinition(const VarDecl *VD, Address VDAddr,
1088	SourceLocation Loc, bool PerformInit,
1089	CodeGenFunction *CGF = nullptr);
1090
1091	/// Emit code for handling declare target functions in the runtime.
1092	/// \param FD Declare target function.
1093	/// \param Addr Address of the global \a FD.
1094	/// \param PerformInit true if initialization expression is not constant.
1095	virtual void emitDeclareTargetFunction(const FunctionDecl *FD,
1096	llvm::GlobalValue *GV);
1097
1098	/// Creates artificial threadprivate variable with name \p Name and type \p
1099	/// VarType.
1100	/// \param VarType Type of the artificial threadprivate variable.
1101	/// \param Name Name of the artificial threadprivate variable.
1102	virtual Address getAddrOfArtificialThreadPrivate(CodeGenFunction &CGF,
1103	QualType VarType,
1104	StringRef Name);
1105
1106	/// Emit flush of the variables specified in 'omp flush' directive.
1107	/// \param Vars List of variables to flush.
1108	virtual void emitFlush(CodeGenFunction &CGF, ArrayRef<const Expr *> Vars,
1109	SourceLocation Loc, llvm::AtomicOrdering AO);
1110
1111	/// Emit task region for the task directive. The task region is
1112	/// emitted in several steps:
1113	/// 1. Emit a call to kmp_task_t *__kmpc_omp_task_alloc(ident_t *, kmp_int32
1114	/// gtid, kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
1115	/// kmp_routine_entry_t *task_entry). Here task_entry is a pointer to the
1116	/// function:
1117	/// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) {
1118	/// TaskFunction(gtid, tt->part_id, tt->shareds);
1119	/// return 0;
1120	/// }
1121	/// 2. Copy a list of shared variables to field shareds of the resulting
1122	/// structure kmp_task_t returned by the previous call (if any).
1123	/// 3. Copy a pointer to destructions function to field destructions of the
1124	/// resulting structure kmp_task_t.
1125	/// 4. Emit a call to kmp_int32 __kmpc_omp_task(ident_t *, kmp_int32 gtid,
1126	/// kmp_task_t *new_task), where new_task is a resulting structure from
1127	/// previous items.
1128	/// \param D Current task directive.
1129	/// \param TaskFunction An LLVM function with type void (*)(i32 /*gtid*/, i32
1130	/// /*part_id*/, captured_struct */*__context*/);
1131	/// \param SharedsTy A type which contains references the shared variables.
1132	/// \param Shareds Context with the list of shared variables from the \p
1133	/// TaskFunction.
1134	/// \param IfCond Not a nullptr if 'if' clause was specified, nullptr
1135	/// otherwise.
1136	/// \param Data Additional data for task generation like tiednsee, final
1137	/// state, list of privates etc.
1138	virtual void emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc,
1139	const OMPExecutableDirective &D,
1140	llvm::Function *TaskFunction, QualType SharedsTy,
1141	Address Shareds, const Expr *IfCond,
1142	const OMPTaskDataTy &Data);
1143
1144	/// Emit task region for the taskloop directive. The taskloop region is
1145	/// emitted in several steps:
1146	/// 1. Emit a call to kmp_task_t *__kmpc_omp_task_alloc(ident_t *, kmp_int32
1147	/// gtid, kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
1148	/// kmp_routine_entry_t *task_entry). Here task_entry is a pointer to the
1149	/// function:
1150	/// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) {
1151	/// TaskFunction(gtid, tt->part_id, tt->shareds);
1152	/// return 0;
1153	/// }
1154	/// 2. Copy a list of shared variables to field shareds of the resulting
1155	/// structure kmp_task_t returned by the previous call (if any).
1156	/// 3. Copy a pointer to destructions function to field destructions of the
1157	/// resulting structure kmp_task_t.
1158	/// 4. Emit a call to void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t
1159	/// *task, int if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int
1160	/// nogroup, int sched, kmp_uint64 grainsize, void *task_dup ), where new_task
1161	/// is a resulting structure from
1162	/// previous items.
1163	/// \param D Current task directive.
1164	/// \param TaskFunction An LLVM function with type void (*)(i32 /*gtid*/, i32
1165	/// /*part_id*/, captured_struct */*__context*/);
1166	/// \param SharedsTy A type which contains references the shared variables.
1167	/// \param Shareds Context with the list of shared variables from the \p
1168	/// TaskFunction.
1169	/// \param IfCond Not a nullptr if 'if' clause was specified, nullptr
1170	/// otherwise.
1171	/// \param Data Additional data for task generation like tiednsee, final
1172	/// state, list of privates etc.
1173	virtual void emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc,
1174	const OMPLoopDirective &D,
1175	llvm::Function *TaskFunction,
1176	QualType SharedsTy, Address Shareds,
1177	const Expr *IfCond, const OMPTaskDataTy &Data);
1178
1179	/// Emit code for the directive that does not require outlining.
1180	///
1181	/// \param InnermostKind Kind of innermost directive (for simple directives it
1182	/// is a directive itself, for combined - its innermost directive).
1183	/// \param CodeGen Code generation sequence for the \a D directive.
1184	/// \param HasCancel true if region has inner cancel directive, false
1185	/// otherwise.
1186	virtual void emitInlinedDirective(CodeGenFunction &CGF,
1187	OpenMPDirectiveKind InnermostKind,
1188	const RegionCodeGenTy &CodeGen,
1189	bool HasCancel = false);
1190
1191	/// Emits reduction function.
1192	/// \param ReducerName Name of the function calling the reduction.
1193	/// \param ArgsElemType Array type containing pointers to reduction variables.
1194	/// \param Privates List of private copies for original reduction arguments.
1195	/// \param LHSExprs List of LHS in \a ReductionOps reduction operations.
1196	/// \param RHSExprs List of RHS in \a ReductionOps reduction operations.
1197	/// \param ReductionOps List of reduction operations in form 'LHS binop RHS'
1198	/// or 'operator binop(LHS, RHS)'.
1199	llvm::Function *emitReductionFunction(
1200	StringRef ReducerName, SourceLocation Loc, llvm::Type *ArgsElemType,
1201	ArrayRef<const Expr *> Privates, ArrayRef<const Expr *> LHSExprs,
1202	ArrayRef<const Expr *> RHSExprs, ArrayRef<const Expr *> ReductionOps);
1203
1204	/// Emits single reduction combiner
1205	void emitSingleReductionCombiner(CodeGenFunction &CGF,
1206	const Expr *ReductionOp,
1207	const Expr *PrivateRef,
1208	const DeclRefExpr *LHS,
1209	const DeclRefExpr *RHS);
1210
1211	struct ReductionOptionsTy {
1212	bool WithNowait;
1213	bool SimpleReduction;
1214	OpenMPDirectiveKind ReductionKind;
1215	};
1216	/// Emit a code for reduction clause. Next code should be emitted for
1217	/// reduction:
1218	/// \code
1219	///
1220	/// static kmp_critical_name lock = { 0 };
1221	///
1222	/// void reduce_func(void *lhs[<n>], void *rhs[<n>]) {
1223	/// ...
1224	/// *(Type<i>*)lhs[i] = RedOp<i>(*(Type<i>*)lhs[i], *(Type<i>*)rhs[i]);
1225	/// ...
1226	/// }
1227	///
1228	/// ...
1229	/// void *RedList[<n>] = {&<RHSExprs>[0], ..., &<RHSExprs>[<n>-1]};
1230	/// switch (__kmpc_reduce{_nowait}(<loc>, <gtid>, <n>, sizeof(RedList),
1231	/// RedList, reduce_func, &<lock>)) {
1232	/// case 1:
1233	/// ...
1234	/// <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]);
1235	/// ...
1236	/// __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>);
1237	/// break;
1238	/// case 2:
1239	/// ...
1240	/// Atomic(<LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]));
1241	/// ...
1242	/// break;
1243	/// default:;
1244	/// }
1245	/// \endcode
1246	///
1247	/// \param Privates List of private copies for original reduction arguments.
1248	/// \param LHSExprs List of LHS in \a ReductionOps reduction operations.
1249	/// \param RHSExprs List of RHS in \a ReductionOps reduction operations.
1250	/// \param ReductionOps List of reduction operations in form 'LHS binop RHS'
1251	/// or 'operator binop(LHS, RHS)'.
1252	/// \param Options List of options for reduction codegen:
1253	/// WithNowait true if parent directive has also nowait clause, false
1254	/// otherwise.
1255	/// SimpleReduction Emit reduction operation only. Used for omp simd
1256	/// directive on the host.
1257	/// ReductionKind The kind of reduction to perform.
1258	virtual void emitReduction(CodeGenFunction &CGF, SourceLocation Loc,
1259	ArrayRef<const Expr *> Privates,
1260	ArrayRef<const Expr *> LHSExprs,
1261	ArrayRef<const Expr *> RHSExprs,
1262	ArrayRef<const Expr *> ReductionOps,
1263	ReductionOptionsTy Options);
1264
1265	/// Emit a code for initialization of task reduction clause. Next code
1266	/// should be emitted for reduction:
1267	/// \code
1268	///
1269	/// _taskred_item_t red_data[n];
1270	/// ...
1271	/// red_data[i].shar = &shareds[i];
1272	/// red_data[i].orig = &origs[i];
1273	/// red_data[i].size = sizeof(origs[i]);
1274	/// red_data[i].f_init = (void*)RedInit<i>;
1275	/// red_data[i].f_fini = (void*)RedDest<i>;
1276	/// red_data[i].f_comb = (void*)RedOp<i>;
1277	/// red_data[i].flags = <Flag_i>;
1278	/// ...
1279	/// void* tg1 = __kmpc_taskred_init(gtid, n, red_data);
1280	/// \endcode
1281	/// For reduction clause with task modifier it emits the next call:
1282	/// \code
1283	///
1284	/// _taskred_item_t red_data[n];
1285	/// ...
1286	/// red_data[i].shar = &shareds[i];
1287	/// red_data[i].orig = &origs[i];
1288	/// red_data[i].size = sizeof(origs[i]);
1289	/// red_data[i].f_init = (void*)RedInit<i>;
1290	/// red_data[i].f_fini = (void*)RedDest<i>;
1291	/// red_data[i].f_comb = (void*)RedOp<i>;
1292	/// red_data[i].flags = <Flag_i>;
1293	/// ...
1294	/// void* tg1 = __kmpc_taskred_modifier_init(loc, gtid, is_worksharing, n,
1295	/// red_data);
1296	/// \endcode
1297	/// \param LHSExprs List of LHS in \a Data.ReductionOps reduction operations.
1298	/// \param RHSExprs List of RHS in \a Data.ReductionOps reduction operations.
1299	/// \param Data Additional data for task generation like tiedness, final
1300	/// state, list of privates, reductions etc.
1301	virtual llvm::Value *emitTaskReductionInit(CodeGenFunction &CGF,
1302	SourceLocation Loc,
1303	ArrayRef<const Expr *> LHSExprs,
1304	ArrayRef<const Expr *> RHSExprs,
1305	const OMPTaskDataTy &Data);
1306
1307	/// Emits the following code for reduction clause with task modifier:
1308	/// \code
1309	/// __kmpc_task_reduction_modifier_fini(loc, gtid, is_worksharing);
1310	/// \endcode
1311	virtual void emitTaskReductionFini(CodeGenFunction &CGF, SourceLocation Loc,
1312	bool IsWorksharingReduction);
1313
1314	/// Required to resolve existing problems in the runtime. Emits threadprivate
1315	/// variables to store the size of the VLAs/array sections for
1316	/// initializer/combiner/finalizer functions.
1317	/// \param RCG Allows to reuse an existing data for the reductions.
1318	/// \param N Reduction item for which fixups must be emitted.
1319	virtual void emitTaskReductionFixups(CodeGenFunction &CGF, SourceLocation Loc,
1320	ReductionCodeGen &RCG, unsigned N);
1321
1322	/// Get the address of `void *` type of the privatue copy of the reduction
1323	/// item specified by the \p SharedLVal.
1324	/// \param ReductionsPtr Pointer to the reduction data returned by the
1325	/// emitTaskReductionInit function.
1326	/// \param SharedLVal Address of the original reduction item.
1327	virtual Address getTaskReductionItem(CodeGenFunction &CGF, SourceLocation Loc,
1328	llvm::Value *ReductionsPtr,
1329	LValue SharedLVal);
1330
1331	/// Emit code for 'taskwait' directive.
1332	virtual void emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc,
1333	const OMPTaskDataTy &Data);
1334
1335	/// Emit code for 'cancellation point' construct.
1336	/// \param CancelRegion Region kind for which the cancellation point must be
1337	/// emitted.
1338	///
1339	virtual void emitCancellationPointCall(CodeGenFunction &CGF,
1340	SourceLocation Loc,
1341	OpenMPDirectiveKind CancelRegion);
1342
1343	/// Emit code for 'cancel' construct.
1344	/// \param IfCond Condition in the associated 'if' clause, if it was
1345	/// specified, nullptr otherwise.
1346	/// \param CancelRegion Region kind for which the cancel must be emitted.
1347	///
1348	virtual void emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc,
1349	const Expr *IfCond,
1350	OpenMPDirectiveKind CancelRegion);
1351
1352	/// Emit outilined function for 'target' directive.
1353	/// \param D Directive to emit.
1354	/// \param ParentName Name of the function that encloses the target region.
1355	/// \param OutlinedFn Outlined function value to be defined by this call.
1356	/// \param OutlinedFnID Outlined function ID value to be defined by this call.
1357	/// \param IsOffloadEntry True if the outlined function is an offload entry.
1358	/// \param CodeGen Code generation sequence for the \a D directive.
1359	/// An outlined function may not be an entry if, e.g. the if clause always
1360	/// evaluates to false.
1361	virtual void emitTargetOutlinedFunction(const OMPExecutableDirective &D,
1362	StringRef ParentName,
1363	llvm::Function *&OutlinedFn,
1364	llvm::Constant *&OutlinedFnID,
1365	bool IsOffloadEntry,
1366	const RegionCodeGenTy &CodeGen);
1367
1368	/// Emit the target offloading code associated with \a D. The emitted
1369	/// code attempts offloading the execution to the device, an the event of
1370	/// a failure it executes the host version outlined in \a OutlinedFn.
1371	/// \param D Directive to emit.
1372	/// \param OutlinedFn Host version of the code to be offloaded.
1373	/// \param OutlinedFnID ID of host version of the code to be offloaded.
1374	/// \param IfCond Expression evaluated in if clause associated with the target
1375	/// directive, or null if no if clause is used.
1376	/// \param Device Expression evaluated in device clause associated with the
1377	/// target directive, or null if no device clause is used and device modifier.
1378	/// \param SizeEmitter Callback to emit number of iterations for loop-based
1379	/// directives.
1380	virtual void emitTargetCall(
1381	CodeGenFunction &CGF, const OMPExecutableDirective &D,
1382	llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID, const Expr *IfCond,
1383	llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier> Device,
1384	llvm::function_ref<llvm::Value *(CodeGenFunction &CGF,
1385	const OMPLoopDirective &D)>
1386	SizeEmitter);
1387
1388	/// Emit the target regions enclosed in \a GD function definition or
1389	/// the function itself in case it is a valid device function. Returns true if
1390	/// \a GD was dealt with successfully.
1391	/// \param GD Function to scan.
1392	virtual bool emitTargetFunctions(GlobalDecl GD);
1393
1394	/// Emit the global variable if it is a valid device global variable.
1395	/// Returns true if \a GD was dealt with successfully.
1396	/// \param GD Variable declaration to emit.
1397	virtual bool emitTargetGlobalVariable(GlobalDecl GD);
1398
1399	/// Checks if the provided global decl \a GD is a declare target variable and
1400	/// registers it when emitting code for the host.
1401	virtual void registerTargetGlobalVariable(const VarDecl *VD,
1402	llvm::Constant *Addr);
1403
1404	/// Emit the global \a GD if it is meaningful for the target. Returns
1405	/// if it was emitted successfully.
1406	/// \param GD Global to scan.
1407	virtual bool emitTargetGlobal(GlobalDecl GD);
1408
1409	/// Creates all the offload entries in the current compilation unit
1410	/// along with the associated metadata.
1411	void createOffloadEntriesAndInfoMetadata();
1412
1413	/// Emits code for teams call of the \a OutlinedFn with
1414	/// variables captured in a record which address is stored in \a
1415	/// CapturedStruct.
1416	/// \param OutlinedFn Outlined function to be run by team masters. Type of
1417	/// this function is void(*)(kmp_int32 *, kmp_int32, struct context_vars*).
1418	/// \param CapturedVars A pointer to the record with the references to
1419	/// variables used in \a OutlinedFn function.
1420	///
1421	virtual void emitTeamsCall(CodeGenFunction &CGF,
1422	const OMPExecutableDirective &D,
1423	SourceLocation Loc, llvm::Function *OutlinedFn,
1424	ArrayRef<llvm::Value *> CapturedVars);
1425
1426	/// Emits call to void __kmpc_push_num_teams(ident_t *loc, kmp_int32
1427	/// global_tid, kmp_int32 num_teams, kmp_int32 thread_limit) to generate code
1428	/// for num_teams clause.
1429	/// \param NumTeams An integer expression of teams.
1430	/// \param ThreadLimit An integer expression of threads.
1431	virtual void emitNumTeamsClause(CodeGenFunction &CGF, const Expr *NumTeams,
1432	const Expr *ThreadLimit, SourceLocation Loc);
1433
1434	/// Emits call to void __kmpc_set_thread_limit(ident_t *loc, kmp_int32
1435	/// global_tid, kmp_int32 thread_limit) to generate code for
1436	/// thread_limit clause on target directive
1437	/// \param ThreadLimit An integer expression of threads.
1438	virtual void emitThreadLimitClause(CodeGenFunction &CGF,
1439	const Expr *ThreadLimit,
1440	SourceLocation Loc);
1441
1442	/// Struct that keeps all the relevant information that should be kept
1443	/// throughout a 'target data' region.
1444	class TargetDataInfo : public llvm::OpenMPIRBuilder::TargetDataInfo {
1445	public:
1446	explicit TargetDataInfo() : llvm::OpenMPIRBuilder::TargetDataInfo() {}
1447	explicit TargetDataInfo(bool RequiresDevicePointerInfo,
1448	bool SeparateBeginEndCalls)
1449	: llvm::OpenMPIRBuilder::TargetDataInfo(RequiresDevicePointerInfo,
1450	SeparateBeginEndCalls) {}
1451	/// Map between the a declaration of a capture and the corresponding new
1452	/// llvm address where the runtime returns the device pointers.
1453	llvm::DenseMap<const ValueDecl *, llvm::Value *> CaptureDeviceAddrMap;
1454	};
1455
1456	/// Emit the target data mapping code associated with \a D.
1457	/// \param D Directive to emit.
1458	/// \param IfCond Expression evaluated in if clause associated with the
1459	/// target directive, or null if no device clause is used.
1460	/// \param Device Expression evaluated in device clause associated with the
1461	/// target directive, or null if no device clause is used.
1462	/// \param Info A record used to store information that needs to be preserved
1463	/// until the region is closed.
1464	virtual void emitTargetDataCalls(CodeGenFunction &CGF,
1465	const OMPExecutableDirective &D,
1466	const Expr *IfCond, const Expr *Device,
1467	const RegionCodeGenTy &CodeGen,
1468	CGOpenMPRuntime::TargetDataInfo &Info);
1469
1470	/// Emit the data mapping/movement code associated with the directive
1471	/// \a D that should be of the form 'target [{enter|exit} data | update]'.
1472	/// \param D Directive to emit.
1473	/// \param IfCond Expression evaluated in if clause associated with the target
1474	/// directive, or null if no if clause is used.
1475	/// \param Device Expression evaluated in device clause associated with the
1476	/// target directive, or null if no device clause is used.
1477	virtual void emitTargetDataStandAloneCall(CodeGenFunction &CGF,
1478	const OMPExecutableDirective &D,
1479	const Expr *IfCond,
1480	const Expr *Device);
1481
1482	/// Marks function \a Fn with properly mangled versions of vector functions.
1483	/// \param FD Function marked as 'declare simd'.
1484	/// \param Fn LLVM function that must be marked with 'declare simd'
1485	/// attributes.
1486	virtual void emitDeclareSimdFunction(const FunctionDecl *FD,
1487	llvm::Function *Fn);
1488
1489	/// Emit initialization for doacross loop nesting support.
1490	/// \param D Loop-based construct used in doacross nesting construct.
1491	virtual void emitDoacrossInit(CodeGenFunction &CGF, const OMPLoopDirective &D,
1492	ArrayRef<Expr *> NumIterations);
1493
1494	/// Emit code for doacross ordered directive with 'depend' clause.
1495	/// \param C 'depend' clause with 'sink|source' dependency kind.
1496	virtual void emitDoacrossOrdered(CodeGenFunction &CGF,
1497	const OMPDependClause *C);
1498
1499	/// Emit code for doacross ordered directive with 'doacross' clause.
1500	/// \param C 'doacross' clause with 'sink|source' dependence type.
1501	virtual void emitDoacrossOrdered(CodeGenFunction &CGF,
1502	const OMPDoacrossClause *C);
1503
1504	/// Translates the native parameter of outlined function if this is required
1505	/// for target.
1506	/// \param FD Field decl from captured record for the parameter.
1507	/// \param NativeParam Parameter itself.
1508	virtual const VarDecl *translateParameter(const FieldDecl *FD,
1509	const VarDecl *NativeParam) const {
1510	return NativeParam;
1511	}
1512
1513	/// Gets the address of the native argument basing on the address of the
1514	/// target-specific parameter.
1515	/// \param NativeParam Parameter itself.
1516	/// \param TargetParam Corresponding target-specific parameter.
1517	virtual Address getParameterAddress(CodeGenFunction &CGF,
1518	const VarDecl *NativeParam,
1519	const VarDecl *TargetParam) const;
1520
1521	/// Choose default schedule type and chunk value for the
1522	/// dist_schedule clause.
1523	virtual void getDefaultDistScheduleAndChunk(CodeGenFunction &CGF,
1524	const OMPLoopDirective &S, OpenMPDistScheduleClauseKind &ScheduleKind,
1525	llvm::Value *&Chunk) const {}
1526
1527	/// Choose default schedule type and chunk value for the
1528	/// schedule clause.
1529	virtual void getDefaultScheduleAndChunk(CodeGenFunction &CGF,
1530	const OMPLoopDirective &S, OpenMPScheduleClauseKind &ScheduleKind,
1531	const Expr *&ChunkExpr) const;
1532
1533	/// Emits call of the outlined function with the provided arguments,
1534	/// translating these arguments to correct target-specific arguments.
1535	virtual void
1536	emitOutlinedFunctionCall(CodeGenFunction &CGF, SourceLocation Loc,
1537	llvm::FunctionCallee OutlinedFn,
1538	ArrayRef<llvm::Value *> Args = std::nullopt) const;
1539
1540	/// Emits OpenMP-specific function prolog.
1541	/// Required for device constructs.
1542	virtual void emitFunctionProlog(CodeGenFunction &CGF, const Decl *D);
1543
1544	/// Gets the OpenMP-specific address of the local variable.
1545	virtual Address getAddressOfLocalVariable(CodeGenFunction &CGF,
1546	const VarDecl *VD);
1547
1548	/// Marks the declaration as already emitted for the device code and returns
1549	/// true, if it was marked already, and false, otherwise.
1550	bool markAsGlobalTarget(GlobalDecl GD);
1551
1552	/// Emit deferred declare target variables marked for deferred emission.
1553	void emitDeferredTargetDecls() const;
1554
1555	/// Adjust some parameters for the target-based directives, like addresses of
1556	/// the variables captured by reference in lambdas.
1557	virtual void
1558	adjustTargetSpecificDataForLambdas(CodeGenFunction &CGF,
1559	const OMPExecutableDirective &D) const;
1560
1561	/// Perform check on requires decl to ensure that target architecture
1562	/// supports unified addressing
1563	virtual void processRequiresDirective(const OMPRequiresDecl *D);
1564
1565	/// Gets default memory ordering as specified in requires directive.
1566	llvm::AtomicOrdering getDefaultMemoryOrdering() const;
1567
1568	/// Checks if the variable has associated OMPAllocateDeclAttr attribute with
1569	/// the predefined allocator and translates it into the corresponding address
1570	/// space.
1571	virtual bool hasAllocateAttributeForGlobalVar(const VarDecl *VD, LangAS &AS);
1572
1573	/// Return whether the unified_shared_memory has been specified.
1574	bool hasRequiresUnifiedSharedMemory() const;
1575
1576	/// Checks if the \p VD variable is marked as nontemporal declaration in
1577	/// current context.
1578	bool isNontemporalDecl(const ValueDecl *VD) const;
1579
1580	/// Create specialized alloca to handle lastprivate conditionals.
1581	Address emitLastprivateConditionalInit(CodeGenFunction &CGF,
1582	const VarDecl *VD);
1583
1584	/// Checks if the provided \p LVal is lastprivate conditional and emits the
1585	/// code to update the value of the original variable.
1586	/// \code
1587	/// lastprivate(conditional: a)
1588	/// ...
1589	/// <type> a;
1590	/// lp_a = ...;
1591	/// #pragma omp critical(a)
1592	/// if (last_iv_a <= iv) {
1593	/// last_iv_a = iv;
1594	/// global_a = lp_a;
1595	/// }
1596	/// \endcode
1597	virtual void checkAndEmitLastprivateConditional(CodeGenFunction &CGF,
1598	const Expr *LHS);
1599
1600	/// Checks if the lastprivate conditional was updated in inner region and
1601	/// writes the value.
1602	/// \code
1603	/// lastprivate(conditional: a)
1604	/// ...
1605	/// <type> a;bool Fired = false;
1606	/// #pragma omp ... shared(a)
1607	/// {
1608	/// lp_a = ...;
1609	/// Fired = true;
1610	/// }
1611	/// if (Fired) {
1612	/// #pragma omp critical(a)
1613	/// if (last_iv_a <= iv) {
1614	/// last_iv_a = iv;
1615	/// global_a = lp_a;
1616	/// }
1617	/// Fired = false;
1618	/// }
1619	/// \endcode
1620	virtual void checkAndEmitSharedLastprivateConditional(
1621	CodeGenFunction &CGF, const OMPExecutableDirective &D,
1622	const llvm::DenseSet<CanonicalDeclPtr<const VarDecl>> &IgnoredDecls);
1623
1624	/// Gets the address of the global copy used for lastprivate conditional
1625	/// update, if any.
1626	/// \param PrivLVal LValue for the private copy.
1627	/// \param VD Original lastprivate declaration.
1628	virtual void emitLastprivateConditionalFinalUpdate(CodeGenFunction &CGF,
1629	LValue PrivLVal,
1630	const VarDecl *VD,
1631	SourceLocation Loc);
1632
1633	/// Emits list of dependecies based on the provided data (array of
1634	/// dependence/expression pairs).
1635	/// \returns Pointer to the first element of the array casted to VoidPtr type.
1636	std::pair<llvm::Value *, Address>
1637	emitDependClause(CodeGenFunction &CGF,
1638	ArrayRef<OMPTaskDataTy::DependData> Dependencies,
1639	SourceLocation Loc);
1640
1641	/// Emits list of dependecies based on the provided data (array of
1642	/// dependence/expression pairs) for depobj construct. In this case, the
1643	/// variable is allocated in dynamically. \returns Pointer to the first
1644	/// element of the array casted to VoidPtr type.
1645	Address emitDepobjDependClause(CodeGenFunction &CGF,
1646	const OMPTaskDataTy::DependData &Dependencies,
1647	SourceLocation Loc);
1648
1649	/// Emits the code to destroy the dependency object provided in depobj
1650	/// directive.
1651	void emitDestroyClause(CodeGenFunction &CGF, LValue DepobjLVal,
1652	SourceLocation Loc);
1653
1654	/// Updates the dependency kind in the specified depobj object.
1655	/// \param DepobjLVal LValue for the main depobj object.
1656	/// \param NewDepKind New dependency kind.
1657	void emitUpdateClause(CodeGenFunction &CGF, LValue DepobjLVal,
1658	OpenMPDependClauseKind NewDepKind, SourceLocation Loc);
1659
1660	/// Initializes user defined allocators specified in the uses_allocators
1661	/// clauses.
1662	void emitUsesAllocatorsInit(CodeGenFunction &CGF, const Expr *Allocator,
1663	const Expr *AllocatorTraits);
1664
1665	/// Destroys user defined allocators specified in the uses_allocators clause.
1666	void emitUsesAllocatorsFini(CodeGenFunction &CGF, const Expr *Allocator);
1667
1668	/// Returns true if the variable is a local variable in untied task.
1669	bool isLocalVarInUntiedTask(CodeGenFunction &CGF, const VarDecl *VD) const;
1670	};
1671
1672	/// Class supports emissionof SIMD-only code.
1673	class CGOpenMPSIMDRuntime final : public CGOpenMPRuntime {
1674	public:
1675	explicit CGOpenMPSIMDRuntime(CodeGenModule &CGM) : CGOpenMPRuntime(CGM) {}
1676	~CGOpenMPSIMDRuntime() override {}
1677
1678	/// Emits outlined function for the specified OpenMP parallel directive
1679	/// \a D. This outlined function has type void(*)(kmp_int32 *ThreadID,
1680	/// kmp_int32 BoundID, struct context_vars*).
1681	/// \param CGF Reference to current CodeGenFunction.
1682	/// \param D OpenMP directive.
1683	/// \param ThreadIDVar Variable for thread id in the current OpenMP region.
1684	/// \param InnermostKind Kind of innermost directive (for simple directives it
1685	/// is a directive itself, for combined - its innermost directive).
1686	/// \param CodeGen Code generation sequence for the \a D directive.
1687	llvm::Function *emitParallelOutlinedFunction(
1688	CodeGenFunction &CGF, const OMPExecutableDirective &D,
1689	const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,
1690	const RegionCodeGenTy &CodeGen) override;
1691
1692	/// Emits outlined function for the specified OpenMP teams directive
1693	/// \a D. This outlined function has type void(*)(kmp_int32 *ThreadID,
1694	/// kmp_int32 BoundID, struct context_vars*).
1695	/// \param CGF Reference to current CodeGenFunction.
1696	/// \param D OpenMP directive.
1697	/// \param ThreadIDVar Variable for thread id in the current OpenMP region.
1698	/// \param InnermostKind Kind of innermost directive (for simple directives it
1699	/// is a directive itself, for combined - its innermost directive).
1700	/// \param CodeGen Code generation sequence for the \a D directive.
1701	llvm::Function *emitTeamsOutlinedFunction(
1702	CodeGenFunction &CGF, const OMPExecutableDirective &D,
1703	const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,
1704	const RegionCodeGenTy &CodeGen) override;
1705
1706	/// Emits outlined function for the OpenMP task directive \a D. This
1707	/// outlined function has type void(*)(kmp_int32 ThreadID, struct task_t*
1708	/// TaskT).
1709	/// \param D OpenMP directive.
1710	/// \param ThreadIDVar Variable for thread id in the current OpenMP region.
1711	/// \param PartIDVar Variable for partition id in the current OpenMP untied
1712	/// task region.
1713	/// \param TaskTVar Variable for task_t argument.
1714	/// \param InnermostKind Kind of innermost directive (for simple directives it
1715	/// is a directive itself, for combined - its innermost directive).
1716	/// \param CodeGen Code generation sequence for the \a D directive.
1717	/// \param Tied true if task is generated for tied task, false otherwise.
1718	/// \param NumberOfParts Number of parts in untied task. Ignored for tied
1719	/// tasks.
1720	///
1721	llvm::Function *emitTaskOutlinedFunction(
1722	const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
1723	const VarDecl *PartIDVar, const VarDecl *TaskTVar,
1724	OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
1725	bool Tied, unsigned &NumberOfParts) override;
1726
1727	/// Emits code for parallel or serial call of the \a OutlinedFn with
1728	/// variables captured in a record which address is stored in \a
1729	/// CapturedStruct.
1730	/// \param OutlinedFn Outlined function to be run in parallel threads. Type of
1731	/// this function is void(*)(kmp_int32 *, kmp_int32, struct context_vars*).
1732	/// \param CapturedVars A pointer to the record with the references to
1733	/// variables used in \a OutlinedFn function.
1734	/// \param IfCond Condition in the associated 'if' clause, if it was
1735	/// specified, nullptr otherwise.
1736	/// \param NumThreads The value corresponding to the num_threads clause, if
1737	/// any, or nullptr.
1738	///
1739	void emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc,
1740	llvm::Function *OutlinedFn,
1741	ArrayRef<llvm::Value *> CapturedVars,
1742	const Expr *IfCond, llvm::Value *NumThreads) override;
1743
1744	/// Emits a critical region.
1745	/// \param CriticalName Name of the critical region.
1746	/// \param CriticalOpGen Generator for the statement associated with the given
1747	/// critical region.
1748	/// \param Hint Value of the 'hint' clause (optional).
1749	void emitCriticalRegion(CodeGenFunction &CGF, StringRef CriticalName,
1750	const RegionCodeGenTy &CriticalOpGen,
1751	SourceLocation Loc,
1752	const Expr *Hint = nullptr) override;
1753
1754	/// Emits a master region.
1755	/// \param MasterOpGen Generator for the statement associated with the given
1756	/// master region.
1757	void emitMasterRegion(CodeGenFunction &CGF,
1758	const RegionCodeGenTy &MasterOpGen,
1759	SourceLocation Loc) override;
1760
1761	/// Emits a masked region.
1762	/// \param MaskedOpGen Generator for the statement associated with the given
1763	/// masked region.
1764	void emitMaskedRegion(CodeGenFunction &CGF,
1765	const RegionCodeGenTy &MaskedOpGen, SourceLocation Loc,
1766	const Expr *Filter = nullptr) override;
1767
1768	/// Emits a masked region.
1769	/// \param MaskedOpGen Generator for the statement associated with the given
1770	/// masked region.
1771
1772	/// Emits code for a taskyield directive.
1773	void emitTaskyieldCall(CodeGenFunction &CGF, SourceLocation Loc) override;
1774
1775	/// Emit a taskgroup region.
1776	/// \param TaskgroupOpGen Generator for the statement associated with the
1777	/// given taskgroup region.
1778	void emitTaskgroupRegion(CodeGenFunction &CGF,
1779	const RegionCodeGenTy &TaskgroupOpGen,
1780	SourceLocation Loc) override;
1781
1782	/// Emits a single region.
1783	/// \param SingleOpGen Generator for the statement associated with the given
1784	/// single region.
1785	void emitSingleRegion(CodeGenFunction &CGF,
1786	const RegionCodeGenTy &SingleOpGen, SourceLocation Loc,
1787	ArrayRef<const Expr *> CopyprivateVars,
1788	ArrayRef<const Expr *> DestExprs,
1789	ArrayRef<const Expr *> SrcExprs,
1790	ArrayRef<const Expr *> AssignmentOps) override;
1791
1792	/// Emit an ordered region.
1793	/// \param OrderedOpGen Generator for the statement associated with the given
1794	/// ordered region.
1795	void emitOrderedRegion(CodeGenFunction &CGF,
1796	const RegionCodeGenTy &OrderedOpGen,
1797	SourceLocation Loc, bool IsThreads) override;
1798
1799	/// Emit an implicit/explicit barrier for OpenMP threads.
1800	/// \param Kind Directive for which this implicit barrier call must be
1801	/// generated. Must be OMPD_barrier for explicit barrier generation.
1802	/// \param EmitChecks true if need to emit checks for cancellation barriers.
1803	/// \param ForceSimpleCall true simple barrier call must be emitted, false if
1804	/// runtime class decides which one to emit (simple or with cancellation
1805	/// checks).
1806	///
1807	void emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc,
1808	OpenMPDirectiveKind Kind, bool EmitChecks = true,
1809	bool ForceSimpleCall = false) override;
1810
1811	/// This is used for non static scheduled types and when the ordered
1812	/// clause is present on the loop construct.
1813	/// Depending on the loop schedule, it is necessary to call some runtime
1814	/// routine before start of the OpenMP loop to get the loop upper / lower
1815	/// bounds \a LB and \a UB and stride \a ST.
1816	///
1817	/// \param CGF Reference to current CodeGenFunction.
1818	/// \param Loc Clang source location.
1819	/// \param ScheduleKind Schedule kind, specified by the 'schedule' clause.
1820	/// \param IVSize Size of the iteration variable in bits.
1821	/// \param IVSigned Sign of the iteration variable.
1822	/// \param Ordered true if loop is ordered, false otherwise.
1823	/// \param DispatchValues struct containing llvm values for lower bound, upper
1824	/// bound, and chunk expression.
1825	/// For the default (nullptr) value, the chunk 1 will be used.
1826	///
1827	void emitForDispatchInit(CodeGenFunction &CGF, SourceLocation Loc,
1828	const OpenMPScheduleTy &ScheduleKind,
1829	unsigned IVSize, bool IVSigned, bool Ordered,
1830	const DispatchRTInput &DispatchValues) override;
1831
1832	/// Call the appropriate runtime routine to initialize it before start
1833	/// of loop.
1834	///
1835	/// This is used only in case of static schedule, when the user did not
1836	/// specify a ordered clause on the loop construct.
1837	/// Depending on the loop schedule, it is necessary to call some runtime
1838	/// routine before start of the OpenMP loop to get the loop upper / lower
1839	/// bounds LB and UB and stride ST.
1840	///
1841	/// \param CGF Reference to current CodeGenFunction.
1842	/// \param Loc Clang source location.
1843	/// \param DKind Kind of the directive.
1844	/// \param ScheduleKind Schedule kind, specified by the 'schedule' clause.
1845	/// \param Values Input arguments for the construct.
1846	///
1847	void emitForStaticInit(CodeGenFunction &CGF, SourceLocation Loc,
1848	OpenMPDirectiveKind DKind,
1849	const OpenMPScheduleTy &ScheduleKind,
1850	const StaticRTInput &Values) override;
1851
1852	///
1853	/// \param CGF Reference to current CodeGenFunction.
1854	/// \param Loc Clang source location.
1855	/// \param SchedKind Schedule kind, specified by the 'dist_schedule' clause.
1856	/// \param Values Input arguments for the construct.
1857	///
1858	void emitDistributeStaticInit(CodeGenFunction &CGF, SourceLocation Loc,
1859	OpenMPDistScheduleClauseKind SchedKind,
1860	const StaticRTInput &Values) override;
1861
1862	/// Call the appropriate runtime routine to notify that we finished
1863	/// iteration of the ordered loop with the dynamic scheduling.
1864	///
1865	/// \param CGF Reference to current CodeGenFunction.
1866	/// \param Loc Clang source location.
1867	/// \param IVSize Size of the iteration variable in bits.
1868	/// \param IVSigned Sign of the iteration variable.
1869	///
1870	void emitForOrderedIterationEnd(CodeGenFunction &CGF, SourceLocation Loc,
1871	unsigned IVSize, bool IVSigned) override;
1872
1873	/// Call the appropriate runtime routine to notify that we finished
1874	/// all the work with current loop.
1875	///
1876	/// \param CGF Reference to current CodeGenFunction.
1877	/// \param Loc Clang source location.
1878	/// \param DKind Kind of the directive for which the static finish is emitted.
1879	///
1880	void emitForStaticFinish(CodeGenFunction &CGF, SourceLocation Loc,
1881	OpenMPDirectiveKind DKind) override;
1882
1883	/// Call __kmpc_dispatch_next(
1884	/// ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter,
1885	/// kmp_int[32|64] *p_lower, kmp_int[32|64] *p_upper,
1886	/// kmp_int[32|64] *p_stride);
1887	/// \param IVSize Size of the iteration variable in bits.
1888	/// \param IVSigned Sign of the iteration variable.
1889	/// \param IL Address of the output variable in which the flag of the
1890	/// last iteration is returned.
1891	/// \param LB Address of the output variable in which the lower iteration
1892	/// number is returned.
1893	/// \param UB Address of the output variable in which the upper iteration
1894	/// number is returned.
1895	/// \param ST Address of the output variable in which the stride value is
1896	/// returned.
1897	llvm::Value *emitForNext(CodeGenFunction &CGF, SourceLocation Loc,
1898	unsigned IVSize, bool IVSigned, Address IL,
1899	Address LB, Address UB, Address ST) override;
1900
1901	/// Emits call to void __kmpc_push_num_threads(ident_t *loc, kmp_int32
1902	/// global_tid, kmp_int32 num_threads) to generate code for 'num_threads'
1903	/// clause.
1904	/// \param NumThreads An integer value of threads.
1905	void emitNumThreadsClause(CodeGenFunction &CGF, llvm::Value *NumThreads,
1906	SourceLocation Loc) override;
1907
1908	/// Emit call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32
1909	/// global_tid, int proc_bind) to generate code for 'proc_bind' clause.
1910	void emitProcBindClause(CodeGenFunction &CGF,
1911	llvm::omp::ProcBindKind ProcBind,
1912	SourceLocation Loc) override;
1913
1914	/// Returns address of the threadprivate variable for the current
1915	/// thread.
1916	/// \param VD Threadprivate variable.
1917	/// \param VDAddr Address of the global variable \a VD.
1918	/// \param Loc Location of the reference to threadprivate var.
1919	/// \return Address of the threadprivate variable for the current thread.
1920	Address getAddrOfThreadPrivate(CodeGenFunction &CGF, const VarDecl *VD,
1921	Address VDAddr, SourceLocation Loc) override;
1922
1923	/// Emit a code for initialization of threadprivate variable. It emits
1924	/// a call to runtime library which adds initial value to the newly created
1925	/// threadprivate variable (if it is not constant) and registers destructor
1926	/// for the variable (if any).
1927	/// \param VD Threadprivate variable.
1928	/// \param VDAddr Address of the global variable \a VD.
1929	/// \param Loc Location of threadprivate declaration.
1930	/// \param PerformInit true if initialization expression is not constant.
1931	llvm::Function *
1932	emitThreadPrivateVarDefinition(const VarDecl *VD, Address VDAddr,
1933	SourceLocation Loc, bool PerformInit,
1934	CodeGenFunction *CGF = nullptr) override;
1935
1936	/// Creates artificial threadprivate variable with name \p Name and type \p
1937	/// VarType.
1938	/// \param VarType Type of the artificial threadprivate variable.
1939	/// \param Name Name of the artificial threadprivate variable.
1940	Address getAddrOfArtificialThreadPrivate(CodeGenFunction &CGF,
1941	QualType VarType,
1942	StringRef Name) override;
1943
1944	/// Emit flush of the variables specified in 'omp flush' directive.
1945	/// \param Vars List of variables to flush.
1946	void emitFlush(CodeGenFunction &CGF, ArrayRef<const Expr *> Vars,
1947	SourceLocation Loc, llvm::AtomicOrdering AO) override;
1948
1949	/// Emit task region for the task directive. The task region is
1950	/// emitted in several steps:
1951	/// 1. Emit a call to kmp_task_t *__kmpc_omp_task_alloc(ident_t *, kmp_int32
1952	/// gtid, kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
1953	/// kmp_routine_entry_t *task_entry). Here task_entry is a pointer to the
1954	/// function:
1955	/// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) {
1956	/// TaskFunction(gtid, tt->part_id, tt->shareds);
1957	/// return 0;
1958	/// }
1959	/// 2. Copy a list of shared variables to field shareds of the resulting
1960	/// structure kmp_task_t returned by the previous call (if any).
1961	/// 3. Copy a pointer to destructions function to field destructions of the
1962	/// resulting structure kmp_task_t.
1963	/// 4. Emit a call to kmp_int32 __kmpc_omp_task(ident_t *, kmp_int32 gtid,
1964	/// kmp_task_t *new_task), where new_task is a resulting structure from
1965	/// previous items.
1966	/// \param D Current task directive.
1967	/// \param TaskFunction An LLVM function with type void (*)(i32 /*gtid*/, i32
1968	/// /*part_id*/, captured_struct */*__context*/);
1969	/// \param SharedsTy A type which contains references the shared variables.
1970	/// \param Shareds Context with the list of shared variables from the \p
1971	/// TaskFunction.
1972	/// \param IfCond Not a nullptr if 'if' clause was specified, nullptr
1973	/// otherwise.
1974	/// \param Data Additional data for task generation like tiednsee, final
1975	/// state, list of privates etc.
1976	void emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc,
1977	const OMPExecutableDirective &D,
1978	llvm::Function *TaskFunction, QualType SharedsTy,
1979	Address Shareds, const Expr *IfCond,
1980	const OMPTaskDataTy &Data) override;
1981
1982	/// Emit task region for the taskloop directive. The taskloop region is
1983	/// emitted in several steps:
1984	/// 1. Emit a call to kmp_task_t *__kmpc_omp_task_alloc(ident_t *, kmp_int32
1985	/// gtid, kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
1986	/// kmp_routine_entry_t *task_entry). Here task_entry is a pointer to the
1987	/// function:
1988	/// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) {
1989	/// TaskFunction(gtid, tt->part_id, tt->shareds);
1990	/// return 0;
1991	/// }
1992	/// 2. Copy a list of shared variables to field shareds of the resulting
1993	/// structure kmp_task_t returned by the previous call (if any).
1994	/// 3. Copy a pointer to destructions function to field destructions of the
1995	/// resulting structure kmp_task_t.
1996	/// 4. Emit a call to void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t
1997	/// *task, int if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int
1998	/// nogroup, int sched, kmp_uint64 grainsize, void *task_dup ), where new_task
1999	/// is a resulting structure from
2000	/// previous items.
2001	/// \param D Current task directive.
2002	/// \param TaskFunction An LLVM function with type void (*)(i32 /*gtid*/, i32
2003	/// /*part_id*/, captured_struct */*__context*/);
2004	/// \param SharedsTy A type which contains references the shared variables.
2005	/// \param Shareds Context with the list of shared variables from the \p
2006	/// TaskFunction.
2007	/// \param IfCond Not a nullptr if 'if' clause was specified, nullptr
2008	/// otherwise.
2009	/// \param Data Additional data for task generation like tiednsee, final
2010	/// state, list of privates etc.
2011	void emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc,
2012	const OMPLoopDirective &D, llvm::Function *TaskFunction,
2013	QualType SharedsTy, Address Shareds, const Expr *IfCond,
2014	const OMPTaskDataTy &Data) override;
2015
2016	/// Emit a code for reduction clause. Next code should be emitted for
2017	/// reduction:
2018	/// \code
2019	///
2020	/// static kmp_critical_name lock = { 0 };
2021	///
2022	/// void reduce_func(void *lhs[<n>], void *rhs[<n>]) {
2023	/// ...
2024	/// *(Type<i>*)lhs[i] = RedOp<i>(*(Type<i>*)lhs[i], *(Type<i>*)rhs[i]);
2025	/// ...
2026	/// }
2027	///
2028	/// ...
2029	/// void *RedList[<n>] = {&<RHSExprs>[0], ..., &<RHSExprs>[<n>-1]};
2030	/// switch (__kmpc_reduce{_nowait}(<loc>, <gtid>, <n>, sizeof(RedList),
2031	/// RedList, reduce_func, &<lock>)) {
2032	/// case 1:
2033	/// ...
2034	/// <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]);
2035	/// ...
2036	/// __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>);
2037	/// break;
2038	/// case 2:
2039	/// ...
2040	/// Atomic(<LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]));
2041	/// ...
2042	/// break;
2043	/// default:;
2044	/// }
2045	/// \endcode
2046	///
2047	/// \param Privates List of private copies for original reduction arguments.
2048	/// \param LHSExprs List of LHS in \a ReductionOps reduction operations.
2049	/// \param RHSExprs List of RHS in \a ReductionOps reduction operations.
2050	/// \param ReductionOps List of reduction operations in form 'LHS binop RHS'
2051	/// or 'operator binop(LHS, RHS)'.
2052	/// \param Options List of options for reduction codegen:
2053	/// WithNowait true if parent directive has also nowait clause, false
2054	/// otherwise.
2055	/// SimpleReduction Emit reduction operation only. Used for omp simd
2056	/// directive on the host.
2057	/// ReductionKind The kind of reduction to perform.
2058	void emitReduction(CodeGenFunction &CGF, SourceLocation Loc,
2059	ArrayRef<const Expr *> Privates,
2060	ArrayRef<const Expr *> LHSExprs,
2061	ArrayRef<const Expr *> RHSExprs,
2062	ArrayRef<const Expr *> ReductionOps,
2063	ReductionOptionsTy Options) override;
2064
2065	/// Emit a code for initialization of task reduction clause. Next code
2066	/// should be emitted for reduction:
2067	/// \code
2068	///
2069	/// _taskred_item_t red_data[n];
2070	/// ...
2071	/// red_data[i].shar = &shareds[i];
2072	/// red_data[i].orig = &origs[i];
2073	/// red_data[i].size = sizeof(origs[i]);
2074	/// red_data[i].f_init = (void*)RedInit<i>;
2075	/// red_data[i].f_fini = (void*)RedDest<i>;
2076	/// red_data[i].f_comb = (void*)RedOp<i>;
2077	/// red_data[i].flags = <Flag_i>;
2078	/// ...
2079	/// void* tg1 = __kmpc_taskred_init(gtid, n, red_data);
2080	/// \endcode
2081	/// For reduction clause with task modifier it emits the next call:
2082	/// \code
2083	///
2084	/// _taskred_item_t red_data[n];
2085	/// ...
2086	/// red_data[i].shar = &shareds[i];
2087	/// red_data[i].orig = &origs[i];
2088	/// red_data[i].size = sizeof(origs[i]);
2089	/// red_data[i].f_init = (void*)RedInit<i>;
2090	/// red_data[i].f_fini = (void*)RedDest<i>;
2091	/// red_data[i].f_comb = (void*)RedOp<i>;
2092	/// red_data[i].flags = <Flag_i>;
2093	/// ...
2094	/// void* tg1 = __kmpc_taskred_modifier_init(loc, gtid, is_worksharing, n,
2095	/// red_data);
2096	/// \endcode
2097	/// \param LHSExprs List of LHS in \a Data.ReductionOps reduction operations.
2098	/// \param RHSExprs List of RHS in \a Data.ReductionOps reduction operations.
2099	/// \param Data Additional data for task generation like tiedness, final
2100	/// state, list of privates, reductions etc.
2101	llvm::Value *emitTaskReductionInit(CodeGenFunction &CGF, SourceLocation Loc,
2102	ArrayRef<const Expr *> LHSExprs,
2103	ArrayRef<const Expr *> RHSExprs,
2104	const OMPTaskDataTy &Data) override;
2105
2106	/// Emits the following code for reduction clause with task modifier:
2107	/// \code
2108	/// __kmpc_task_reduction_modifier_fini(loc, gtid, is_worksharing);
2109	/// \endcode
2110	void emitTaskReductionFini(CodeGenFunction &CGF, SourceLocation Loc,
2111	bool IsWorksharingReduction) override;
2112
2113	/// Required to resolve existing problems in the runtime. Emits threadprivate
2114	/// variables to store the size of the VLAs/array sections for
2115	/// initializer/combiner/finalizer functions + emits threadprivate variable to
2116	/// store the pointer to the original reduction item for the custom
2117	/// initializer defined by declare reduction construct.
2118	/// \param RCG Allows to reuse an existing data for the reductions.
2119	/// \param N Reduction item for which fixups must be emitted.
2120	void emitTaskReductionFixups(CodeGenFunction &CGF, SourceLocation Loc,
2121	ReductionCodeGen &RCG, unsigned N) override;
2122
2123	/// Get the address of `void *` type of the privatue copy of the reduction
2124	/// item specified by the \p SharedLVal.
2125	/// \param ReductionsPtr Pointer to the reduction data returned by the
2126	/// emitTaskReductionInit function.
2127	/// \param SharedLVal Address of the original reduction item.
2128	Address getTaskReductionItem(CodeGenFunction &CGF, SourceLocation Loc,
2129	llvm::Value *ReductionsPtr,
2130	LValue SharedLVal) override;
2131
2132	/// Emit code for 'taskwait' directive.
2133	void emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc,
2134	const OMPTaskDataTy &Data) override;
2135
2136	/// Emit code for 'cancellation point' construct.
2137	/// \param CancelRegion Region kind for which the cancellation point must be
2138	/// emitted.
2139	///
2140	void emitCancellationPointCall(CodeGenFunction &CGF, SourceLocation Loc,
2141	OpenMPDirectiveKind CancelRegion) override;
2142
2143	/// Emit code for 'cancel' construct.
2144	/// \param IfCond Condition in the associated 'if' clause, if it was
2145	/// specified, nullptr otherwise.
2146	/// \param CancelRegion Region kind for which the cancel must be emitted.
2147	///
2148	void emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc,
2149	const Expr *IfCond,
2150	OpenMPDirectiveKind CancelRegion) override;
2151
2152	/// Emit outilined function for 'target' directive.
2153	/// \param D Directive to emit.
2154	/// \param ParentName Name of the function that encloses the target region.
2155	/// \param OutlinedFn Outlined function value to be defined by this call.
2156	/// \param OutlinedFnID Outlined function ID value to be defined by this call.
2157	/// \param IsOffloadEntry True if the outlined function is an offload entry.
2158	/// \param CodeGen Code generation sequence for the \a D directive.
2159	/// An outlined function may not be an entry if, e.g. the if clause always
2160	/// evaluates to false.
2161	void emitTargetOutlinedFunction(const OMPExecutableDirective &D,
2162	StringRef ParentName,
2163	llvm::Function *&OutlinedFn,
2164	llvm::Constant *&OutlinedFnID,
2165	bool IsOffloadEntry,
2166	const RegionCodeGenTy &CodeGen) override;
2167
2168	/// Emit the target offloading code associated with \a D. The emitted
2169	/// code attempts offloading the execution to the device, an the event of
2170	/// a failure it executes the host version outlined in \a OutlinedFn.
2171	/// \param D Directive to emit.
2172	/// \param OutlinedFn Host version of the code to be offloaded.
2173	/// \param OutlinedFnID ID of host version of the code to be offloaded.
2174	/// \param IfCond Expression evaluated in if clause associated with the target
2175	/// directive, or null if no if clause is used.
2176	/// \param Device Expression evaluated in device clause associated with the
2177	/// target directive, or null if no device clause is used and device modifier.
2178	void emitTargetCall(
2179	CodeGenFunction &CGF, const OMPExecutableDirective &D,
2180	llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID, const Expr *IfCond,
2181	llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier> Device,
2182	llvm::function_ref<llvm::Value *(CodeGenFunction &CGF,
2183	const OMPLoopDirective &D)>
2184	SizeEmitter) override;
2185
2186	/// Emit the target regions enclosed in \a GD function definition or
2187	/// the function itself in case it is a valid device function. Returns true if
2188	/// \a GD was dealt with successfully.
2189	/// \param GD Function to scan.
2190	bool emitTargetFunctions(GlobalDecl GD) override;
2191
2192	/// Emit the global variable if it is a valid device global variable.
2193	/// Returns true if \a GD was dealt with successfully.
2194	/// \param GD Variable declaration to emit.
2195	bool emitTargetGlobalVariable(GlobalDecl GD) override;
2196
2197	/// Emit the global \a GD if it is meaningful for the target. Returns
2198	/// if it was emitted successfully.
2199	/// \param GD Global to scan.
2200	bool emitTargetGlobal(GlobalDecl GD) override;
2201
2202	/// Emits code for teams call of the \a OutlinedFn with
2203	/// variables captured in a record which address is stored in \a
2204	/// CapturedStruct.
2205	/// \param OutlinedFn Outlined function to be run by team masters. Type of
2206	/// this function is void(*)(kmp_int32 *, kmp_int32, struct context_vars*).
2207	/// \param CapturedVars A pointer to the record with the references to
2208	/// variables used in \a OutlinedFn function.
2209	///
2210	void emitTeamsCall(CodeGenFunction &CGF, const OMPExecutableDirective &D,
2211	SourceLocation Loc, llvm::Function *OutlinedFn,
2212	ArrayRef<llvm::Value *> CapturedVars) override;
2213
2214	/// Emits call to void __kmpc_push_num_teams(ident_t *loc, kmp_int32
2215	/// global_tid, kmp_int32 num_teams, kmp_int32 thread_limit) to generate code
2216	/// for num_teams clause.
2217	/// \param NumTeams An integer expression of teams.
2218	/// \param ThreadLimit An integer expression of threads.
2219	void emitNumTeamsClause(CodeGenFunction &CGF, const Expr *NumTeams,
2220	const Expr *ThreadLimit, SourceLocation Loc) override;
2221
2222	/// Emit the target data mapping code associated with \a D.
2223	/// \param D Directive to emit.
2224	/// \param IfCond Expression evaluated in if clause associated with the
2225	/// target directive, or null if no device clause is used.
2226	/// \param Device Expression evaluated in device clause associated with the
2227	/// target directive, or null if no device clause is used.
2228	/// \param Info A record used to store information that needs to be preserved
2229	/// until the region is closed.
2230	void emitTargetDataCalls(CodeGenFunction &CGF,
2231	const OMPExecutableDirective &D, const Expr *IfCond,
2232	const Expr *Device, const RegionCodeGenTy &CodeGen,
2233	CGOpenMPRuntime::TargetDataInfo &Info) override;
2234
2235	/// Emit the data mapping/movement code associated with the directive
2236	/// \a D that should be of the form 'target [{enter|exit} data | update]'.
2237	/// \param D Directive to emit.
2238	/// \param IfCond Expression evaluated in if clause associated with the target
2239	/// directive, or null if no if clause is used.
2240	/// \param Device Expression evaluated in device clause associated with the
2241	/// target directive, or null if no device clause is used.
2242	void emitTargetDataStandAloneCall(CodeGenFunction &CGF,
2243	const OMPExecutableDirective &D,
2244	const Expr *IfCond,
2245	const Expr *Device) override;
2246
2247	/// Emit initialization for doacross loop nesting support.
2248	/// \param D Loop-based construct used in doacross nesting construct.
2249	void emitDoacrossInit(CodeGenFunction &CGF, const OMPLoopDirective &D,
2250	ArrayRef<Expr *> NumIterations) override;
2251
2252	/// Emit code for doacross ordered directive with 'depend' clause.
2253	/// \param C 'depend' clause with 'sink|source' dependency kind.
2254	void emitDoacrossOrdered(CodeGenFunction &CGF,
2255	const OMPDependClause *C) override;
2256
2257	/// Emit code for doacross ordered directive with 'doacross' clause.
2258	/// \param C 'doacross' clause with 'sink|source' dependence type.
2259	void emitDoacrossOrdered(CodeGenFunction &CGF,
2260	const OMPDoacrossClause *C) override;
2261
2262	/// Translates the native parameter of outlined function if this is required
2263	/// for target.
2264	/// \param FD Field decl from captured record for the parameter.
2265	/// \param NativeParam Parameter itself.
2266	const VarDecl *translateParameter(const FieldDecl *FD,
2267	const VarDecl *NativeParam) const override;
2268
2269	/// Gets the address of the native argument basing on the address of the
2270	/// target-specific parameter.
2271	/// \param NativeParam Parameter itself.
2272	/// \param TargetParam Corresponding target-specific parameter.
2273	Address getParameterAddress(CodeGenFunction &CGF, const VarDecl *NativeParam,
2274	const VarDecl *TargetParam) const override;
2275
2276	/// Gets the OpenMP-specific address of the local variable.
2277	Address getAddressOfLocalVariable(CodeGenFunction &CGF,
2278	const VarDecl *VD) override {
2279	return Address::invalid();
2280	}
2281	};
2282
2283	} // namespace CodeGen
2284	// Utility for openmp doacross clause kind
2285	namespace {
2286	template <typename T> class OMPDoacrossKind {
2287	public:
2288	bool isSink(const T *) { return false; }
2289	bool isSource(const T *) { return false; }
2290	};
2291	template <> class OMPDoacrossKind<OMPDependClause> {
2292	public:
2293	bool isSink(const OMPDependClause *C) {
2294	return C->getDependencyKind() == OMPC_DEPEND_sink;
2295	}
2296	bool isSource(const OMPDependClause *C) {
2297	return C->getDependencyKind() == OMPC_DEPEND_source;
2298	}
2299	};
2300	template <> class OMPDoacrossKind<OMPDoacrossClause> {
2301	public:
2302	bool isSource(const OMPDoacrossClause *C) {
2303	return C->getDependenceType() == OMPC_DOACROSS_source ||
2304	C->getDependenceType() == OMPC_DOACROSS_source_omp_cur_iteration;
2305	}
2306	bool isSink(const OMPDoacrossClause *C) {
2307	return C->getDependenceType() == OMPC_DOACROSS_sink ||
2308	C->getDependenceType() == OMPC_DOACROSS_sink_omp_cur_iteration;
2309	}
2310	};
2311	} // namespace
2312	} // namespace clang
2313
2314	#endif
2315