1	//===-- OpenMP.cpp -- Open MP directive lowering --------------------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// Coding style: https://mlir.llvm.org/getting_started/DeveloperGuide/
10	//
11	//===----------------------------------------------------------------------===//
12
13	#include "flang/Lower/OpenMP.h"
14
15	#include "ClauseProcessor.h"
16	#include "Clauses.h"
17	#include "DataSharingProcessor.h"
18	#include "Decomposer.h"
19	#include "ReductionProcessor.h"
20	#include "Utils.h"
21	#include "flang/Common/idioms.h"
22	#include "flang/Lower/Bridge.h"
23	#include "flang/Lower/ConvertExpr.h"
24	#include "flang/Lower/ConvertVariable.h"
25	#include "flang/Lower/DirectivesCommon.h"
26	#include "flang/Lower/StatementContext.h"
27	#include "flang/Lower/SymbolMap.h"
28	#include "flang/Optimizer/Builder/BoxValue.h"
29	#include "flang/Optimizer/Builder/FIRBuilder.h"
30	#include "flang/Optimizer/Builder/Todo.h"
31	#include "flang/Optimizer/Dialect/FIRType.h"
32	#include "flang/Optimizer/HLFIR/HLFIROps.h"
33	#include "flang/Parser/characters.h"
34	#include "flang/Parser/parse-tree.h"
35	#include "flang/Semantics/openmp-directive-sets.h"
36	#include "flang/Semantics/tools.h"
37	#include "flang/Support/Flags.h"
38	#include "flang/Support/OpenMP-utils.h"
39	#include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h"
40	#include "mlir/Dialect/OpenMP/OpenMPDialect.h"
41	#include "mlir/Transforms/RegionUtils.h"
42	#include "llvm/ADT/STLExtras.h"
43	#include "llvm/Frontend/OpenMP/OMPConstants.h"
44	#include "llvm/Support/CommandLine.h"
45
46	using namespace Fortran::lower::omp;
47	using namespace Fortran::common::openmp;
48
49	static llvm::cl::opt<bool> DumpAtomicAnalysis("fdebug-dump-atomic-analysis");
50
51	//===----------------------------------------------------------------------===//
52	// Code generation helper functions
53	//===----------------------------------------------------------------------===//
54
55	static void genOMPDispatch(lower::AbstractConverter &converter,
56	lower::SymMap &symTable,
57	semantics::SemanticsContext &semaCtx,
58	lower::pft::Evaluation &eval, mlir::Location loc,
59	const ConstructQueue &queue,
60	ConstructQueue::const_iterator item);
61
62	static void processHostEvalClauses(lower::AbstractConverter &converter,
63	semantics::SemanticsContext &semaCtx,
64	lower::StatementContext &stmtCtx,
65	lower::pft::Evaluation &eval,
66	mlir::Location loc);
67
68	namespace {
69	/// Structure holding information that is needed to pass host-evaluated
70	/// information to later lowering stages.
71	class HostEvalInfo {
72	public:
73	// Allow this function access to private members in order to initialize them.
74	friend void ::processHostEvalClauses(lower::AbstractConverter &,
75	semantics::SemanticsContext &,
76	lower::StatementContext &,
77	lower::pft::Evaluation &,
78	mlir::Location);
79
80	/// Fill \c vars with values stored in \c ops.
81	///
82	/// The order in which values are stored matches the one expected by \see
83	/// bindOperands().
84	void collectValues(llvm::SmallVectorImpl<mlir::Value> &vars) const {
85	vars.append(ops.loopLowerBounds);
86	vars.append(ops.loopUpperBounds);
87	vars.append(ops.loopSteps);
88
89	if (ops.numTeamsLower)
90	vars.push_back(ops.numTeamsLower);
91
92	if (ops.numTeamsUpper)
93	vars.push_back(ops.numTeamsUpper);
94
95	if (ops.numThreads)
96	vars.push_back(ops.numThreads);
97
98	if (ops.threadLimit)
99	vars.push_back(ops.threadLimit);
100	}
101
102	/// Update \c ops, replacing all values with the corresponding block argument
103	/// in \c args.
104	///
105	/// The order in which values are stored in \c args is the same as the one
106	/// used by \see collectValues().
107	void bindOperands(llvm::ArrayRef<mlir::BlockArgument> args) {
108	assert(args.size() ==
109	ops.loopLowerBounds.size() + ops.loopUpperBounds.size() +
110	ops.loopSteps.size() + (ops.numTeamsLower ? 1 : 0) +
111	(ops.numTeamsUpper ? 1 : 0) + (ops.numThreads ? 1 : 0) +
112	(ops.threadLimit ? 1 : 0) &&
113	"invalid block argument list");
114	int argIndex = 0;
115	for (size_t i = 0; i < ops.loopLowerBounds.size(); ++i)
116	ops.loopLowerBounds[i] = args[argIndex++];
117
118	for (size_t i = 0; i < ops.loopUpperBounds.size(); ++i)
119	ops.loopUpperBounds[i] = args[argIndex++];
120
121	for (size_t i = 0; i < ops.loopSteps.size(); ++i)
122	ops.loopSteps[i] = args[argIndex++];
123
124	if (ops.numTeamsLower)
125	ops.numTeamsLower = args[argIndex++];
126
127	if (ops.numTeamsUpper)
128	ops.numTeamsUpper = args[argIndex++];
129
130	if (ops.numThreads)
131	ops.numThreads = args[argIndex++];
132
133	if (ops.threadLimit)
134	ops.threadLimit = args[argIndex++];
135	}
136
137	/// Update \p clauseOps and \p ivOut with the corresponding host-evaluated
138	/// values and Fortran symbols, respectively, if they have already been
139	/// initialized but not yet applied.
140	///
141	/// \returns whether an update was performed. If not, these clauses were not
142	/// evaluated in the host device.
143	bool apply(mlir::omp::LoopNestOperands &clauseOps,
144	llvm::SmallVectorImpl<const semantics::Symbol *> &ivOut) {
145	if (iv.empty() || loopNestApplied) {
146	loopNestApplied = true;
147	return false;
148	}
149
150	loopNestApplied = true;
151	clauseOps.loopLowerBounds = ops.loopLowerBounds;
152	clauseOps.loopUpperBounds = ops.loopUpperBounds;
153	clauseOps.loopSteps = ops.loopSteps;
154	ivOut.append(RHS: iv);
155	return true;
156	}
157
158	/// Update \p clauseOps with the corresponding host-evaluated values if they
159	/// have already been initialized but not yet applied.
160	///
161	/// \returns whether an update was performed. If not, these clauses were not
162	/// evaluated in the host device.
163	bool apply(mlir::omp::ParallelOperands &clauseOps) {
164	if (!ops.numThreads || parallelApplied) {
165	parallelApplied = true;
166	return false;
167	}
168
169	parallelApplied = true;
170	clauseOps.numThreads = ops.numThreads;
171	return true;
172	}
173
174	/// Update \p clauseOps with the corresponding host-evaluated values if they
175	/// have already been initialized.
176	///
177	/// \returns whether an update was performed. If not, these clauses were not
178	/// evaluated in the host device.
179	bool apply(mlir::omp::TeamsOperands &clauseOps) {
180	if (!ops.numTeamsLower && !ops.numTeamsUpper && !ops.threadLimit)
181	return false;
182
183	clauseOps.numTeamsLower = ops.numTeamsLower;
184	clauseOps.numTeamsUpper = ops.numTeamsUpper;
185	clauseOps.threadLimit = ops.threadLimit;
186	return true;
187	}
188
189	private:
190	mlir::omp::HostEvaluatedOperands ops;
191	llvm::SmallVector<const semantics::Symbol *> iv;
192	bool loopNestApplied = false, parallelApplied = false;
193	};
194	} // namespace
195
196	/// Stack of \see HostEvalInfo to represent the current nest of \c omp.target
197	/// operations being created.
198	///
199	/// The current implementation prevents nested 'target' regions from breaking
200	/// the handling of the outer region by keeping a stack of information
201	/// structures, but it will probably still require some further work to support
202	/// reverse offloading.
203	static llvm::SmallVector<HostEvalInfo, 0> hostEvalInfo;
204
205	/// Bind symbols to their corresponding entry block arguments.
206	///
207	/// The binding will be performed inside of the current block, which does not
208	/// necessarily have to be part of the operation for which the binding is done.
209	/// However, block arguments must be accessible. This enables controlling the
210	/// insertion point of any new MLIR operations related to the binding of
211	/// arguments of a loop wrapper operation.
212	///
213	/// \param [in] converter - PFT to MLIR conversion interface.
214	/// \param [in] op - owner operation of the block arguments to bind.
215	/// \param [in] args - entry block arguments information for the given
216	/// operation.
217	static void bindEntryBlockArgs(lower::AbstractConverter &converter,
218	mlir::omp::BlockArgOpenMPOpInterface op,
219	const EntryBlockArgs &args) {
220	assert(op != nullptr && "invalid block argument-defining operation");
221	assert(args.isValid() && "invalid args");
222	fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
223
224	auto bindSingleMapLike = [&converter](const semantics::Symbol &sym,
225	const mlir::BlockArgument &arg) {
226	fir::ExtendedValue extVal = converter.getSymbolExtendedValue(sym);
227	auto refType = mlir::dyn_cast<fir::ReferenceType>(arg.getType());
228	if (refType && fir::isa_builtin_cptr_type(refType.getElementType())) {
229	converter.bindSymbol(sym, arg);
230	} else {
231	extVal.match(
232	[&](const fir::BoxValue &v) {
233	converter.bindSymbol(sym, fir::BoxValue(arg, v.getLBounds(),
234	v.getExplicitParameters(),
235	v.getExplicitExtents()));
236	},
237	[&](const fir::MutableBoxValue &v) {
238	converter.bindSymbol(
239	sym, fir::MutableBoxValue(arg, v.getLBounds(),
240	v.getMutableProperties()));
241	},
242	[&](const fir::ArrayBoxValue &v) {
243	converter.bindSymbol(sym, fir::ArrayBoxValue(arg, v.getExtents(),
244	v.getLBounds(),
245	v.getSourceBox()));
246	},
247	[&](const fir::CharArrayBoxValue &v) {
248	converter.bindSymbol(sym, fir::CharArrayBoxValue(arg, v.getLen(),
249	v.getExtents(),
250	v.getLBounds()));
251	},
252	[&](const fir::CharBoxValue &v) {
253	converter.bindSymbol(sym, fir::CharBoxValue(arg, v.getLen()));
254	},
255	[&](const fir::UnboxedValue &v) { converter.bindSymbol(sym, arg); },
256	[&](const auto &) {
257	TODO(converter.getCurrentLocation(),
258	"target map clause operand unsupported type");
259	});
260	}
261	};
262
263	auto bindMapLike =
264	[&bindSingleMapLike](llvm::ArrayRef<const semantics::Symbol *> syms,
265	llvm::ArrayRef<mlir::BlockArgument> args) {
266	// Structure component symbols don't have bindings, and can only be
267	// explicitly mapped individually. If a member is captured implicitly
268	// we map the entirety of the derived type when we find its symbol.
269	llvm::SmallVector<const semantics::Symbol *> processedSyms;
270	llvm::copy_if(Range&: syms, Out: std::back_inserter(x&: processedSyms),
271	P: [](auto *sym) { return !sym->owner().IsDerivedType(); });
272
273	for (auto [sym, arg] : llvm::zip_equal(processedSyms, args))
274	bindSingleMapLike(*sym, arg);
275	};
276
277	auto bindPrivateLike = [&converter, &firOpBuilder](
278	llvm::ArrayRef<const semantics::Symbol *> syms,
279	llvm::ArrayRef<mlir::Value> vars,
280	llvm::ArrayRef<mlir::BlockArgument> args) {
281	llvm::SmallVector<const semantics::Symbol *> processedSyms;
282	for (auto *sym : syms) {
283	if (const auto *commonDet =
284	sym->detailsIf<semantics::CommonBlockDetails>()) {
285	llvm::transform(commonDet->objects(), std::back_inserter(x&: processedSyms),
286	[&](const auto &mem) { return &*mem; });
287	} else {
288	processedSyms.push_back(Elt: sym);
289	}
290	}
291
292	for (auto [sym, var, arg] : llvm::zip_equal(processedSyms, vars, args))
293	converter.bindSymbol(
294	*sym,
295	hlfir::translateToExtendedValue(
296	var.getLoc(), firOpBuilder, hlfir::Entity{arg},
297	/*contiguousHint=*/
298	evaluate::IsSimplyContiguous(*sym, converter.getFoldingContext()))
299	.first);
300	};
301
302	// Process in clause name alphabetical order to match block arguments order.
303	// Do not bind host_eval variables because they cannot be used inside of the
304	// corresponding region, except for very specific cases handled separately.
305	bindMapLike(args.hasDeviceAddr.syms, op.getHasDeviceAddrBlockArgs());
306	bindPrivateLike(args.inReduction.syms, args.inReduction.vars,
307	op.getInReductionBlockArgs());
308	bindMapLike(args.map.syms, op.getMapBlockArgs());
309	bindPrivateLike(args.priv.syms, args.priv.vars, op.getPrivateBlockArgs());
310	bindPrivateLike(args.reduction.syms, args.reduction.vars,
311	op.getReductionBlockArgs());
312	bindPrivateLike(args.taskReduction.syms, args.taskReduction.vars,
313	op.getTaskReductionBlockArgs());
314	bindMapLike(args.useDeviceAddr.syms, op.getUseDeviceAddrBlockArgs());
315	bindMapLike(args.useDevicePtr.syms, op.getUseDevicePtrBlockArgs());
316	}
317
318	/// Get the list of base values that the specified map-like variables point to.
319	///
320	/// This function must be kept in sync with changes to the `createMapInfoOp`
321	/// utility function, since it must take into account the potential introduction
322	/// of levels of indirection (i.e. intermediate ops).
323	///
324	/// \param [in] vars - list of values passed to map-like clauses, returned
325	/// by an `omp.map.info` operation.
326	/// \param [out] baseOps - populated with the `var_ptr` values of the
327	/// corresponding defining operations.
328	static void
329	extractMappedBaseValues(llvm::ArrayRef<mlir::Value> vars,
330	llvm::SmallVectorImpl<mlir::Value> &baseOps) {
331	llvm::transform(vars, std::back_inserter(baseOps), [](mlir::Value map) {
332	auto mapInfo = map.getDefiningOp<mlir::omp::MapInfoOp>();
333	assert(mapInfo && "expected all map vars to be defined by omp.map.info");
334
335	mlir::Value varPtr = mapInfo.getVarPtr();
336	if (auto boxAddr = varPtr.getDefiningOp<fir::BoxAddrOp>())
337	return boxAddr.getVal();
338
339	return varPtr;
340	});
341	}
342
343	/// Get the directive enumeration value corresponding to the given OpenMP
344	/// construct PFT node.
345	llvm::omp::Directive
346	extractOmpDirective(const parser::OpenMPConstruct &ompConstruct) {
347	return common::visit(
348	common::visitors{
349	[](const parser::OpenMPAllocatorsConstruct &c) {
350	return llvm::omp::OMPD_allocators;
351	},
352	[](const parser::OpenMPAssumeConstruct &c) {
353	return llvm::omp::OMPD_assume;
354	},
355	[](const parser::OpenMPAtomicConstruct &c) {
356	return llvm::omp::OMPD_atomic;
357	},
358	[](const parser::OpenMPBlockConstruct &c) {
359	return std::get<parser::OmpBlockDirective>(
360	std::get<parser::OmpBeginBlockDirective>(c.t).t)
361	.v;
362	},
363	[](const parser::OpenMPCriticalConstruct &c) {
364	return llvm::omp::OMPD_critical;
365	},
366	[](const parser::OpenMPDeclarativeAllocate &c) {
367	return llvm::omp::OMPD_allocate;
368	},
369	[](const parser::OpenMPDispatchConstruct &c) {
370	return llvm::omp::OMPD_dispatch;
371	},
372	[](const parser::OpenMPExecutableAllocate &c) {
373	return llvm::omp::OMPD_allocate;
374	},
375	[](const parser::OpenMPLoopConstruct &c) {
376	return std::get<parser::OmpLoopDirective>(
377	std::get<parser::OmpBeginLoopDirective>(c.t).t)
378	.v;
379	},
380	[](const parser::OpenMPSectionConstruct &c) {
381	return llvm::omp::OMPD_section;
382	},
383	[](const parser::OpenMPSectionsConstruct &c) {
384	return std::get<parser::OmpSectionsDirective>(
385	std::get<parser::OmpBeginSectionsDirective>(c.t).t)
386	.v;
387	},
388	[](const parser::OpenMPStandaloneConstruct &c) {
389	return common::visit(
390	common::visitors{
391	[](const parser::OpenMPSimpleStandaloneConstruct &c) {
392	return c.v.DirId();
393	},
394	[](const parser::OpenMPFlushConstruct &c) {
395	return llvm::omp::OMPD_flush;
396	},
397	[](const parser::OpenMPCancelConstruct &c) {
398	return llvm::omp::OMPD_cancel;
399	},
400	[](const parser::OpenMPCancellationPointConstruct &c) {
401	return llvm::omp::OMPD_cancellation_point;
402	},
403	[](const parser::OmpMetadirectiveDirective &c) {
404	return llvm::omp::OMPD_metadirective;
405	},
406	[](const parser::OpenMPDepobjConstruct &c) {
407	return llvm::omp::OMPD_depobj;
408	},
409	[](const parser::OpenMPInteropConstruct &c) {
410	return llvm::omp::OMPD_interop;
411	}},
412	c.u);
413	},
414	[](const parser::OpenMPUtilityConstruct &c) {
415	return common::visit(
416	common::visitors{[](const parser::OmpErrorDirective &c) {
417	return llvm::omp::OMPD_error;
418	},
419	[](const parser::OmpNothingDirective &c) {
420	return llvm::omp::OMPD_nothing;
421	}},
422	c.u);
423	}},
424	ompConstruct.u);
425	}
426
427	/// Populate the global \see hostEvalInfo after processing clauses for the given
428	/// \p eval OpenMP target construct, or nested constructs, if these must be
429	/// evaluated outside of the target region per the spec.
430	///
431	/// In particular, this will ensure that in 'target teams' and equivalent nested
432	/// constructs, the \c thread_limit and \c num_teams clauses will be evaluated
433	/// in the host. Additionally, loop bounds, steps and the \c num_threads clause
434	/// will also be evaluated in the host if a target SPMD construct is detected
435	/// (i.e. 'target teams distribute parallel do [simd]' or equivalent nesting).
436	///
437	/// The result, stored as a global, is intended to be used to populate the \c
438	/// host_eval operands of the associated \c omp.target operation, and also to be
439	/// checked and used by later lowering steps to populate the corresponding
440	/// operands of the \c omp.teams, \c omp.parallel or \c omp.loop_nest
441	/// operations.
442	static void processHostEvalClauses(lower::AbstractConverter &converter,
443	semantics::SemanticsContext &semaCtx,
444	lower::StatementContext &stmtCtx,
445	lower::pft::Evaluation &eval,
446	mlir::Location loc) {
447	// Obtain the list of clauses of the given OpenMP block or loop construct
448	// evaluation. Other evaluations passed to this lambda keep `clauses`
449	// unchanged.
450	auto extractClauses = [&semaCtx](lower::pft::Evaluation &eval,
451	List<Clause> &clauses) {
452	const auto *ompEval = eval.getIf<parser::OpenMPConstruct>();
453	if (!ompEval)
454	return;
455
456	const parser::OmpClauseList *beginClauseList = nullptr;
457	const parser::OmpClauseList *endClauseList = nullptr;
458	common::visit(
459	common::visitors{
460	[&](const parser::OpenMPBlockConstruct &ompConstruct) {
461	const auto &beginDirective =
462	std::get<parser::OmpBeginBlockDirective>(ompConstruct.t);
463	beginClauseList =
464	&std::get<parser::OmpClauseList>(beginDirective.t);
465	endClauseList = &std::get<parser::OmpClauseList>(
466	std::get<parser::OmpEndBlockDirective>(ompConstruct.t).t);
467	},
468	[&](const parser::OpenMPLoopConstruct &ompConstruct) {
469	const auto &beginDirective =
470	std::get<parser::OmpBeginLoopDirective>(ompConstruct.t);
471	beginClauseList =
472	&std::get<parser::OmpClauseList>(beginDirective.t);
473
474	if (auto &endDirective =
475	std::get<std::optional<parser::OmpEndLoopDirective>>(
476	ompConstruct.t))
477	endClauseList =
478	&std::get<parser::OmpClauseList>(endDirective->t);
479	},
480	[&](const auto &) {}},
481	ompEval->u);
482
483	assert(beginClauseList && "expected begin directive");
484	clauses.append(makeClauses(*beginClauseList, semaCtx));
485
486	if (endClauseList)
487	clauses.append(makeClauses(*endClauseList, semaCtx));
488	};
489
490	// Return the directive that is immediately nested inside of the given
491	// `parent` evaluation, if it is its only non-end-statement nested evaluation
492	// and it represents an OpenMP construct.
493	auto extractOnlyOmpNestedDir = [](lower::pft::Evaluation &parent)
494	-> std::optional<llvm::omp::Directive> {
495	if (!parent.hasNestedEvaluations())
496	return std::nullopt;
497
498	llvm::omp::Directive dir;
499	auto &nested = parent.getFirstNestedEvaluation();
500	if (const auto *ompEval = nested.getIf<parser::OpenMPConstruct>())
501	dir = extractOmpDirective(*ompEval);
502	else
503	return std::nullopt;
504
505	for (auto &sibling : parent.getNestedEvaluations())
506	if (&sibling != &nested && !sibling.isEndStmt())
507	return std::nullopt;
508
509	return dir;
510	};
511
512	// Process the given evaluation assuming it's part of a 'target' construct or
513	// captured by one, and store results in the global `hostEvalInfo`.
514	std::function<void(lower::pft::Evaluation &, const List<Clause> &)>
515	processEval;
516	processEval = [&](lower::pft::Evaluation &eval, const List<Clause> &clauses) {
517	using namespace llvm::omp;
518	ClauseProcessor cp(converter, semaCtx, clauses);
519
520	// Call `processEval` recursively with the immediately nested evaluation and
521	// its corresponding clauses if there is a single nested evaluation
522	// representing an OpenMP directive that passes the given test.
523	auto processSingleNestedIf = [&](llvm::function_ref<bool(Directive)> test) {
524	std::optional<Directive> nestedDir = extractOnlyOmpNestedDir(eval);
525	if (!nestedDir || !test(*nestedDir))
526	return;
527
528	lower::pft::Evaluation &nestedEval = eval.getFirstNestedEvaluation();
529	List<lower::omp::Clause> nestedClauses;
530	extractClauses(nestedEval, nestedClauses);
531	processEval(nestedEval, nestedClauses);
532	};
533
534	const auto *ompEval = eval.getIf<parser::OpenMPConstruct>();
535	if (!ompEval)
536	return;
537
538	HostEvalInfo &hostInfo = hostEvalInfo.back();
539
540	switch (extractOmpDirective(*ompEval)) {
541	case OMPD_teams_distribute_parallel_do:
542	case OMPD_teams_distribute_parallel_do_simd:
543	cp.processThreadLimit(stmtCtx, hostInfo.ops);
544	[[fallthrough]];
545	case OMPD_target_teams_distribute_parallel_do:
546	case OMPD_target_teams_distribute_parallel_do_simd:
547	cp.processNumTeams(stmtCtx, hostInfo.ops);
548	[[fallthrough]];
549	case OMPD_distribute_parallel_do:
550	case OMPD_distribute_parallel_do_simd:
551	cp.processNumThreads(stmtCtx, hostInfo.ops);
552	[[fallthrough]];
553	case OMPD_distribute:
554	case OMPD_distribute_simd:
555	cp.processCollapse(loc, eval, hostInfo.ops, hostInfo.iv);
556	break;
557
558	case OMPD_teams:
559	cp.processThreadLimit(stmtCtx, hostInfo.ops);
560	[[fallthrough]];
561	case OMPD_target_teams:
562	cp.processNumTeams(stmtCtx, hostInfo.ops);
563	processSingleNestedIf([](Directive nestedDir) {
564	return topDistributeSet.test(nestedDir) || topLoopSet.test(nestedDir);
565	});
566	break;
567
568	case OMPD_teams_distribute:
569	case OMPD_teams_distribute_simd:
570	cp.processThreadLimit(stmtCtx, hostInfo.ops);
571	[[fallthrough]];
572	case OMPD_target_teams_distribute:
573	case OMPD_target_teams_distribute_simd:
574	cp.processCollapse(loc, eval, hostInfo.ops, hostInfo.iv);
575	cp.processNumTeams(stmtCtx, hostInfo.ops);
576	break;
577
578	case OMPD_teams_loop:
579	cp.processThreadLimit(stmtCtx, hostInfo.ops);
580	[[fallthrough]];
581	case OMPD_target_teams_loop:
582	cp.processNumTeams(stmtCtx, hostInfo.ops);
583	[[fallthrough]];
584	case OMPD_loop:
585	cp.processCollapse(loc, eval, hostInfo.ops, hostInfo.iv);
586	break;
587
588	// Standalone 'target' case.
589	case OMPD_target: {
590	processSingleNestedIf(
591	[](Directive nestedDir) { return topTeamsSet.test(nestedDir); });
592	break;
593	}
594	default:
595	break;
596	}
597	};
598
599	assert(!hostEvalInfo.empty() && "expected HOST_EVAL info structure");
600
601	const auto *ompEval = eval.getIf<parser::OpenMPConstruct>();
602	assert(ompEval &&
603	llvm::omp::allTargetSet.test(extractOmpDirective(*ompEval)) &&
604	"expected TARGET construct evaluation");
605	(void)ompEval;
606
607	// Use the whole list of clauses passed to the construct here, rather than the
608	// ones only applied to omp.target.
609	List<lower::omp::Clause> clauses;
610	extractClauses(eval, clauses);
611	processEval(eval, clauses);
612	}
613
614	static lower::pft::Evaluation *
615	getCollapsedLoopEval(lower::pft::Evaluation &eval, int collapseValue) {
616	// Return the Evaluation of the innermost collapsed loop, or the current one
617	// if there was no COLLAPSE.
618	if (collapseValue == 0)
619	return &eval;
620
621	lower::pft::Evaluation *curEval = &eval.getFirstNestedEvaluation();
622	for (int i = 1; i < collapseValue; i++) {
623	// The nested evaluations should be DoConstructs (i.e. they should form
624	// a loop nest). Each DoConstruct is a tuple <NonLabelDoStmt, Block,
625	// EndDoStmt>.
626	assert(curEval->isA<parser::DoConstruct>());
627	curEval = &*std::next(curEval->getNestedEvaluations().begin());
628	}
629	return curEval;
630	}
631
632	static void genNestedEvaluations(lower::AbstractConverter &converter,
633	lower::pft::Evaluation &eval,
634	int collapseValue = 0) {
635	lower::pft::Evaluation *curEval = getCollapsedLoopEval(eval, collapseValue);
636
637	for (lower::pft::Evaluation &e : curEval->getNestedEvaluations())
638	converter.genEval(e);
639	}
640
641	static fir::GlobalOp globalInitialization(lower::AbstractConverter &converter,
642	fir::FirOpBuilder &firOpBuilder,
643	const semantics::Symbol &sym,
644	const lower::pft::Variable &var,
645	mlir::Location currentLocation) {
646	std::string globalName = converter.mangleName(sym);
647	mlir::StringAttr linkage = firOpBuilder.createInternalLinkage();
648	return Fortran::lower::defineGlobal(converter, var, globalName, linkage);
649	}
650
651	// Get the extended value for \p val by extracting additional variable
652	// information from \p base.
653	static fir::ExtendedValue getExtendedValue(fir::ExtendedValue base,
654	mlir::Value val) {
655	return base.match(
656	[&](const fir::MutableBoxValue &box) -> fir::ExtendedValue {
657	return fir::MutableBoxValue(val, box.nonDeferredLenParams(), {});
658	},
659	[&](const auto &) -> fir::ExtendedValue {
660	return fir::substBase(base, val);
661	});
662	}
663
664	#ifndef NDEBUG
665	static bool isThreadPrivate(lower::SymbolRef sym) {
666	if (const auto *details = sym->detailsIf<semantics::CommonBlockDetails>()) {
667	for (const auto &obj : details->objects())
668	if (!obj->test(semantics::Symbol::Flag::OmpThreadprivate))
669	return false;
670	return true;
671	}
672	return sym->test(semantics::Symbol::Flag::OmpThreadprivate);
673	}
674	#endif
675
676	static void threadPrivatizeVars(lower::AbstractConverter &converter,
677	lower::pft::Evaluation &eval) {
678	fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
679	mlir::Location currentLocation = converter.getCurrentLocation();
680	mlir::OpBuilder::InsertionGuard guard(firOpBuilder);
681	firOpBuilder.setInsertionPointToStart(firOpBuilder.getAllocaBlock());
682
683	// If the symbol corresponds to the original ThreadprivateOp, use the symbol
684	// value from that operation to create one ThreadprivateOp copy operation
685	// inside the parallel region.
686	// In some cases, however, the symbol will correspond to the original,
687	// non-threadprivate variable. This can happen, for instance, with a common
688	// block, declared in a separate module, used by a parent procedure and
689	// privatized in its child procedure.
690	auto genThreadprivateOp = [&](lower::SymbolRef sym) -> mlir::Value {
691	assert(isThreadPrivate(sym));
692	mlir::Value symValue = converter.getSymbolAddress(sym);
693	mlir::Operation *op = symValue.getDefiningOp();
694	if (auto declOp = mlir::dyn_cast<hlfir::DeclareOp>(op))
695	op = declOp.getMemref().getDefiningOp();
696	if (mlir::isa<mlir::omp::ThreadprivateOp>(op))
697	symValue = mlir::dyn_cast<mlir::omp::ThreadprivateOp>(op).getSymAddr();
698	return firOpBuilder.create<mlir::omp::ThreadprivateOp>(
699	currentLocation, symValue.getType(), symValue);
700	};
701
702	llvm::SetVector<const semantics::Symbol *> threadprivateSyms;
703	converter.collectSymbolSet(eval, threadprivateSyms,
704	semantics::Symbol::Flag::OmpThreadprivate,
705	/*collectSymbols=*/true,
706	/*collectHostAssociatedSymbols=*/true);
707	std::set<semantics::SourceName> threadprivateSymNames;
708
709	// For a COMMON block, the ThreadprivateOp is generated for itself instead of
710	// its members, so only bind the value of the new copied ThreadprivateOp
711	// inside the parallel region to the common block symbol only once for
712	// multiple members in one COMMON block.
713	llvm::SetVector<const semantics::Symbol *> commonSyms;
714	for (std::size_t i = 0; i < threadprivateSyms.size(); i++) {
715	const semantics::Symbol *sym = threadprivateSyms[i];
716	mlir::Value symThreadprivateValue;
717	// The variable may be used more than once, and each reference has one
718	// symbol with the same name. Only do once for references of one variable.
719	if (threadprivateSymNames.find(sym->name()) != threadprivateSymNames.end())
720	continue;
721	threadprivateSymNames.insert(sym->name());
722	if (const semantics::Symbol *common =
723	semantics::FindCommonBlockContaining(sym->GetUltimate())) {
724	mlir::Value commonThreadprivateValue;
725	if (commonSyms.contains(common)) {
726	commonThreadprivateValue = converter.getSymbolAddress(*common);
727	} else {
728	commonThreadprivateValue = genThreadprivateOp(*common);
729	converter.bindSymbol(*common, commonThreadprivateValue);
730	commonSyms.insert(common);
731	}
732	symThreadprivateValue = lower::genCommonBlockMember(
733	converter, currentLocation, sym->GetUltimate(),
734	commonThreadprivateValue);
735	} else {
736	symThreadprivateValue = genThreadprivateOp(*sym);
737	}
738
739	fir::ExtendedValue sexv = converter.getSymbolExtendedValue(*sym);
740	fir::ExtendedValue symThreadprivateExv =
741	getExtendedValue(sexv, symThreadprivateValue);
742	converter.bindSymbol(*sym, symThreadprivateExv);
743	}
744	}
745
746	static mlir::Operation *
747	createAndSetPrivatizedLoopVar(lower::AbstractConverter &converter,
748	mlir::Location loc, mlir::Value indexVal,
749	const semantics::Symbol *sym) {
750	fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
751	mlir::OpBuilder::InsertPoint insPt = firOpBuilder.saveInsertionPoint();
752	firOpBuilder.setInsertionPointToStart(firOpBuilder.getAllocaBlock());
753
754	mlir::Type tempTy = converter.genType(*sym);
755
756	assert(converter.isPresentShallowLookup(*sym) &&
757	"Expected symbol to be in symbol table.");
758
759	firOpBuilder.restoreInsertionPoint(insPt);
760	mlir::Value cvtVal = firOpBuilder.createConvert(loc, tempTy, indexVal);
761	hlfir::Entity lhs{converter.getSymbolAddress(*sym)};
762
763	lhs = hlfir::derefPointersAndAllocatables(loc, firOpBuilder, lhs);
764
765	mlir::Operation *storeOp =
766	firOpBuilder.create<hlfir::AssignOp>(loc, cvtVal, lhs);
767	return storeOp;
768	}
769
770	// This helper function implements the functionality of "promoting" non-CPTR
771	// arguments of use_device_ptr to use_device_addr arguments (automagic
772	// conversion of use_device_ptr -> use_device_addr in these cases). The way we
773	// do so currently is through the shuffling of operands from the
774	// devicePtrOperands to deviceAddrOperands, as well as the types, locations and
775	// symbols.
776	//
777	// This effectively implements some deprecated OpenMP functionality that some
778	// legacy applications unfortunately depend on (deprecated in specification
779	// version 5.2):
780	//
781	// "If a list item in a use_device_ptr clause is not of type C_PTR, the behavior
782	// is as if the list item appeared in a use_device_addr clause. Support for
783	// such list items in a use_device_ptr clause is deprecated."
784	static void promoteNonCPtrUseDevicePtrArgsToUseDeviceAddr(
785	llvm::SmallVectorImpl<mlir::Value> &useDeviceAddrVars,
786	llvm::SmallVectorImpl<const semantics::Symbol *> &useDeviceAddrSyms,
787	llvm::SmallVectorImpl<mlir::Value> &useDevicePtrVars,
788	llvm::SmallVectorImpl<const semantics::Symbol *> &useDevicePtrSyms) {
789	// Iterate over our use_device_ptr list and shift all non-cptr arguments into
790	// use_device_addr.
791	auto *varIt = useDevicePtrVars.begin();
792	auto *symIt = useDevicePtrSyms.begin();
793	while (varIt != useDevicePtrVars.end()) {
794	if (fir::isa_builtin_cptr_type(fir::unwrapRefType(varIt->getType()))) {
795	++varIt;
796	++symIt;
797	continue;
798	}
799
800	useDeviceAddrVars.push_back(*varIt);
801	useDeviceAddrSyms.push_back(Elt: *symIt);
802
803	varIt = useDevicePtrVars.erase(varIt);
804	symIt = useDevicePtrSyms.erase(CI: symIt);
805	}
806	}
807
808	/// Extract the list of function and variable symbols affected by the given
809	/// 'declare target' directive and return the intended device type for them.
810	static void getDeclareTargetInfo(
811	lower::AbstractConverter &converter, semantics::SemanticsContext &semaCtx,
812	lower::pft::Evaluation &eval,
813	const parser::OpenMPDeclareTargetConstruct &declareTargetConstruct,
814	mlir::omp::DeclareTargetOperands &clauseOps,
815	llvm::SmallVectorImpl<DeclareTargetCapturePair> &symbolAndClause) {
816	const auto &spec =
817	std::get<parser::OmpDeclareTargetSpecifier>(declareTargetConstruct.t);
818	if (const auto *objectList{parser::Unwrap<parser::OmpObjectList>(spec.u)}) {
819	ObjectList objects{makeObjects(*objectList, semaCtx)};
820	// Case: declare target(func, var1, var2)
821	gatherFuncAndVarSyms(objects, mlir::omp::DeclareTargetCaptureClause::to,
822	symbolAndClause);
823	} else if (const auto *clauseList{
824	parser::Unwrap<parser::OmpClauseList>(spec.u)}) {
825	List<Clause> clauses = makeClauses(*clauseList, semaCtx);
826	if (clauses.empty()) {
827	Fortran::lower::pft::FunctionLikeUnit *owningProc =
828	eval.getOwningProcedure();
829	if (owningProc && (!owningProc->isMainProgram() ||
830	owningProc->getMainProgramSymbol())) {
831	// Case: declare target, implicit capture of function
832	symbolAndClause.emplace_back(mlir::omp::DeclareTargetCaptureClause::to,
833	owningProc->getSubprogramSymbol());
834	}
835	}
836
837	ClauseProcessor cp(converter, semaCtx, clauses);
838	cp.processDeviceType(clauseOps);
839	cp.processEnter(symbolAndClause);
840	cp.processLink(symbolAndClause);
841	cp.processTo(symbolAndClause);
842
843	cp.processTODO<clause::Indirect>(converter.getCurrentLocation(),
844	llvm::omp::Directive::OMPD_declare_target);
845	}
846	}
847
848	static void collectDeferredDeclareTargets(
849	lower::AbstractConverter &converter, semantics::SemanticsContext &semaCtx,
850	lower::pft::Evaluation &eval,
851	const parser::OpenMPDeclareTargetConstruct &declareTargetConstruct,
852	llvm::SmallVectorImpl<lower::OMPDeferredDeclareTargetInfo>
853	&deferredDeclareTarget) {
854	mlir::omp::DeclareTargetOperands clauseOps;
855	llvm::SmallVector<DeclareTargetCapturePair> symbolAndClause;
856	getDeclareTargetInfo(converter, semaCtx, eval, declareTargetConstruct,
857	clauseOps, symbolAndClause);
858	// Return the device type only if at least one of the targets for the
859	// directive is a function or subroutine
860	mlir::ModuleOp mod = converter.getFirOpBuilder().getModule();
861
862	for (const DeclareTargetCapturePair &symClause : symbolAndClause) {
863	mlir::Operation *op = mod.lookupSymbol(
864	converter.mangleName(std::get<const semantics::Symbol &>(symClause)));
865
866	if (!op) {
867	deferredDeclareTarget.push_back({std::get<0>(symClause),
868	clauseOps.deviceType,
869	std::get<1>(symClause)});
870	}
871	}
872	}
873
874	static std::optional<mlir::omp::DeclareTargetDeviceType>
875	getDeclareTargetFunctionDevice(
876	lower::AbstractConverter &converter, semantics::SemanticsContext &semaCtx,
877	lower::pft::Evaluation &eval,
878	const parser::OpenMPDeclareTargetConstruct &declareTargetConstruct) {
879	mlir::omp::DeclareTargetOperands clauseOps;
880	llvm::SmallVector<DeclareTargetCapturePair> symbolAndClause;
881	getDeclareTargetInfo(converter, semaCtx, eval, declareTargetConstruct,
882	clauseOps, symbolAndClause);
883
884	// Return the device type only if at least one of the targets for the
885	// directive is a function or subroutine
886	mlir::ModuleOp mod = converter.getFirOpBuilder().getModule();
887	for (const DeclareTargetCapturePair &symClause : symbolAndClause) {
888	mlir::Operation *op = mod.lookupSymbol(
889	converter.mangleName(std::get<const semantics::Symbol &>(symClause)));
890
891	if (mlir::isa_and_nonnull<mlir::func::FuncOp>(op))
892	return clauseOps.deviceType;
893	}
894
895	return std::nullopt;
896	}
897
898	/// Set up the entry block of the given `omp.loop_nest` operation, adding a
899	/// block argument for each loop induction variable and allocating and
900	/// initializing a private value to hold each of them.
901	///
902	/// This function can also bind the symbols of any variables that should match
903	/// block arguments on parent loop wrapper operations attached to the same
904	/// loop. This allows the introduction of any necessary `hlfir.declare`
905	/// operations inside of the entry block of the `omp.loop_nest` operation and
906	/// not directly under any of the wrappers, which would invalidate them.
907	///
908	/// \param [in] op - the loop nest operation.
909	/// \param [in] converter - PFT to MLIR conversion interface.
910	/// \param [in] loc - location.
911	/// \param [in] args - symbols of induction variables.
912	/// \param [in] wrapperArgs - list of parent loop wrappers and their associated
913	/// entry block arguments.
914	static void genLoopVars(
915	mlir::Operation *op, lower::AbstractConverter &converter,
916	mlir::Location &loc, llvm::ArrayRef<const semantics::Symbol *> args,
917	llvm::ArrayRef<
918	std::pair<mlir::omp::BlockArgOpenMPOpInterface, const EntryBlockArgs &>>
919	wrapperArgs = {}) {
920	fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
921	auto &region = op->getRegion(0);
922
923	std::size_t loopVarTypeSize = 0;
924	for (const semantics::Symbol *arg : args)
925	loopVarTypeSize = std::max(loopVarTypeSize, arg->GetUltimate().size());
926	mlir::Type loopVarType = getLoopVarType(converter, loopVarTypeSize);
927	llvm::SmallVector<mlir::Type> tiv(args.size(), loopVarType);
928	llvm::SmallVector<mlir::Location> locs(args.size(), loc);
929	firOpBuilder.createBlock(&region, {}, tiv, locs);
930
931	// Update nested wrapper operands if parent wrappers have mapped these values
932	// to block arguments.
933	//
934	// Binding these values earlier would take care of this, but we cannot rely on
935	// that approach because binding in between the creation of a wrapper and the
936	// next one would result in 'hlfir.declare' operations being introduced inside
937	// of a wrapper, which is illegal.
938	mlir::IRMapping mapper;
939	for (auto [argGeneratingOp, blockArgs] : wrapperArgs) {
940	for (mlir::OpOperand &operand : argGeneratingOp->getOpOperands())
941	operand.set(mapper.lookupOrDefault(operand.get()));
942
943	for (const auto [arg, var] : llvm::zip_equal(
944	argGeneratingOp->getRegion(0).getArguments(), blockArgs.getVars()))
945	mapper.map(var, arg);
946	}
947
948	// Bind the entry block arguments of parent wrappers to the corresponding
949	// symbols.
950	for (auto [argGeneratingOp, blockArgs] : wrapperArgs)
951	bindEntryBlockArgs(converter, argGeneratingOp, blockArgs);
952
953	// The argument is not currently in memory, so make a temporary for the
954	// argument, and store it there, then bind that location to the argument.
955	mlir::Operation *storeOp = nullptr;
956	for (auto [argIndex, argSymbol] : llvm::enumerate(First&: args)) {
957	mlir::Value indexVal = fir::getBase(region.front().getArgument(argIndex));
958	storeOp =
959	createAndSetPrivatizedLoopVar(converter, loc, indexVal, argSymbol);
960	}
961	firOpBuilder.setInsertionPointAfter(storeOp);
962	}
963
964	static clause::Defaultmap::ImplicitBehavior
965	getDefaultmapIfPresent(const DefaultMapsTy &defaultMaps, mlir::Type varType) {
966	using DefMap = clause::Defaultmap;
967
968	if (defaultMaps.empty())
969	return DefMap::ImplicitBehavior::Default;
970
971	if (llvm::is_contained(defaultMaps, DefMap::VariableCategory::All))
972	return defaultMaps.at(DefMap::VariableCategory::All);
973
974	// NOTE: Unsure if complex and/or vector falls into a scalar type
975	// or aggregate, but the current default implicit behaviour is to
976	// treat them as such (c_ptr has its own behaviour, so perhaps
977	// being lumped in as a scalar isn't the right thing).
978	if ((fir::isa_trivial(varType) || fir::isa_char(varType) ||
979	fir::isa_builtin_cptr_type(varType)) &&
980	llvm::is_contained(defaultMaps, DefMap::VariableCategory::Scalar))
981	return defaultMaps.at(DefMap::VariableCategory::Scalar);
982
983	if (fir::isPointerType(varType) &&
984	llvm::is_contained(defaultMaps, DefMap::VariableCategory::Pointer))
985	return defaultMaps.at(DefMap::VariableCategory::Pointer);
986
987	if (fir::isAllocatableType(varType) &&
988	llvm::is_contained(defaultMaps, DefMap::VariableCategory::Allocatable))
989	return defaultMaps.at(DefMap::VariableCategory::Allocatable);
990
991	if (fir::isa_aggregate(varType) &&
992	llvm::is_contained(defaultMaps, DefMap::VariableCategory::Aggregate))
993	return defaultMaps.at(DefMap::VariableCategory::Aggregate);
994
995	return DefMap::ImplicitBehavior::Default;
996	}
997
998	static std::pair<llvm::omp::OpenMPOffloadMappingFlags,
999	mlir::omp::VariableCaptureKind>
1000	getImplicitMapTypeAndKind(fir::FirOpBuilder &firOpBuilder,
1001	lower::AbstractConverter &converter,
1002	const DefaultMapsTy &defaultMaps, mlir::Type varType,
1003	mlir::Location loc, const semantics::Symbol &sym) {
1004	using DefMap = clause::Defaultmap;
1005	// Check if a value of type `type` can be passed to the kernel by value.
1006	// All kernel parameters are of pointer type, so if the value can be
1007	// represented inside of a pointer, then it can be passed by value.
1008	auto isLiteralType = [&](mlir::Type type) {
1009	const mlir::DataLayout &dl = firOpBuilder.getDataLayout();
1010	mlir::Type ptrTy =
1011	mlir::LLVM::LLVMPointerType::get(&converter.getMLIRContext());
1012	uint64_t ptrSize = dl.getTypeSize(ptrTy);
1013	uint64_t ptrAlign = dl.getTypePreferredAlignment(ptrTy);
1014
1015	auto [size, align] = fir::getTypeSizeAndAlignmentOrCrash(
1016	loc, type, dl, converter.getKindMap());
1017	return size <= ptrSize && align <= ptrAlign;
1018	};
1019
1020	llvm::omp::OpenMPOffloadMappingFlags mapFlag =
1021	llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT;
1022
1023	auto implicitBehaviour = getDefaultmapIfPresent(defaultMaps, varType);
1024	if (implicitBehaviour == DefMap::ImplicitBehavior::Default) {
1025	mlir::omp::VariableCaptureKind captureKind =
1026	mlir::omp::VariableCaptureKind::ByRef;
1027
1028	// If a variable is specified in declare target link and if device
1029	// type is not specified as `nohost`, it needs to be mapped tofrom
1030	mlir::ModuleOp mod = firOpBuilder.getModule();
1031	mlir::Operation *op = mod.lookupSymbol(converter.mangleName(sym));
1032	auto declareTargetOp =
1033	llvm::dyn_cast_if_present<mlir::omp::DeclareTargetInterface>(op);
1034	if (declareTargetOp && declareTargetOp.isDeclareTarget()) {
1035	if (declareTargetOp.getDeclareTargetCaptureClause() ==
1036	mlir::omp::DeclareTargetCaptureClause::link &&
1037	declareTargetOp.getDeclareTargetDeviceType() !=
1038	mlir::omp::DeclareTargetDeviceType::nohost) {
1039	mapFlag |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TO;
1040	mapFlag |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_FROM;
1041	}
1042	} else if (fir::isa_trivial(varType) || fir::isa_char(varType)) {
1043	// Scalars behave as if they were "firstprivate".
1044	// TODO: Handle objects that are shared/lastprivate or were listed
1045	// in an in_reduction clause.
1046	if (isLiteralType(varType)) {
1047	captureKind = mlir::omp::VariableCaptureKind::ByCopy;
1048	} else {
1049	mapFlag |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TO;
1050	}
1051	} else if (!fir::isa_builtin_cptr_type(varType)) {
1052	mapFlag |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TO;
1053	mapFlag |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_FROM;
1054	}
1055	return std::make_pair(mapFlag, captureKind);
1056	}
1057
1058	switch (implicitBehaviour) {
1059	case DefMap::ImplicitBehavior::Alloc:
1060	return std::make_pair(llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_NONE,
1061	mlir::omp::VariableCaptureKind::ByRef);
1062	break;
1063	case DefMap::ImplicitBehavior::Firstprivate:
1064	case DefMap::ImplicitBehavior::None:
1065	TODO(loc, "Firstprivate and None are currently unsupported defaultmap "
1066	"behaviour");
1067	break;
1068	case DefMap::ImplicitBehavior::From:
1069	return std::make_pair(mapFlag |=
1070	llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_FROM,
1071	mlir::omp::VariableCaptureKind::ByRef);
1072	break;
1073	case DefMap::ImplicitBehavior::Present:
1074	return std::make_pair(mapFlag |=
1075	llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_PRESENT,
1076	mlir::omp::VariableCaptureKind::ByRef);
1077	break;
1078	case DefMap::ImplicitBehavior::To:
1079	return std::make_pair(mapFlag |=
1080	llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TO,
1081	(fir::isa_trivial(varType) || fir::isa_char(varType))
1082	? mlir::omp::VariableCaptureKind::ByCopy
1083	: mlir::omp::VariableCaptureKind::ByRef);
1084	break;
1085	case DefMap::ImplicitBehavior::Tofrom:
1086	return std::make_pair(mapFlag |=
1087	llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_FROM |
1088	llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TO,
1089	mlir::omp::VariableCaptureKind::ByRef);
1090	break;
1091	case DefMap::ImplicitBehavior::Default:
1092	llvm_unreachable(
1093	"Implicit None Behaviour Should Have Been Handled Earlier");
1094	break;
1095	}
1096
1097	return std::make_pair(mapFlag |=
1098	llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_FROM |
1099	llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TO,
1100	mlir::omp::VariableCaptureKind::ByRef);
1101	}
1102
1103	static void
1104	markDeclareTarget(mlir::Operation *op, lower::AbstractConverter &converter,
1105	mlir::omp::DeclareTargetCaptureClause captureClause,
1106	mlir::omp::DeclareTargetDeviceType deviceType) {
1107	// TODO: Add support for program local variables with declare target applied
1108	auto declareTargetOp = llvm::dyn_cast<mlir::omp::DeclareTargetInterface>(op);
1109	if (!declareTargetOp)
1110	fir::emitFatalError(
1111	converter.getCurrentLocation(),
1112	"Attempt to apply declare target on unsupported operation");
1113
1114	// The function or global already has a declare target applied to it, very
1115	// likely through implicit capture (usage in another declare target
1116	// function/subroutine). It should be marked as any if it has been assigned
1117	// both host and nohost, else we skip, as there is no change
1118	if (declareTargetOp.isDeclareTarget()) {
1119	if (declareTargetOp.getDeclareTargetDeviceType() != deviceType)
1120	declareTargetOp.setDeclareTarget(mlir::omp::DeclareTargetDeviceType::any,
1121	captureClause);
1122	return;
1123	}
1124
1125	declareTargetOp.setDeclareTarget(deviceType, captureClause);
1126	}
1127
1128	static bool isPointerAssignment(const evaluate::Assignment &assign) {
1129	return common::visit(
1130	common::visitors{
1131	[](const evaluate::Assignment::BoundsSpec &) { return true; },
1132	[](const evaluate::Assignment::BoundsRemapping &) { return true; },
1133	[](const auto &) { return false; },
1134	},
1135	assign.u);
1136	}
1137
1138	//===----------------------------------------------------------------------===//
1139	// Op body generation helper structures and functions
1140	//===----------------------------------------------------------------------===//
1141
1142	struct OpWithBodyGenInfo {
1143	/// A type for a code-gen callback function. This takes as argument the op for
1144	/// which the code is being generated and returns the arguments of the op's
1145	/// region.
1146	using GenOMPRegionEntryCBFn =
1147	std::function<llvm::SmallVector<const semantics::Symbol *>(
1148	mlir::Operation *)>;
1149
1150	OpWithBodyGenInfo(lower::AbstractConverter &converter,
1151	lower::SymMap &symTable,
1152	semantics::SemanticsContext &semaCtx, mlir::Location loc,
1153	lower::pft::Evaluation &eval, llvm::omp::Directive dir)
1154	: converter(converter), symTable(symTable), semaCtx(semaCtx), loc(loc),
1155	eval(eval), dir(dir) {}
1156
1157	OpWithBodyGenInfo &setClauses(const List<Clause> *value) {
1158	clauses = value;
1159	return *this;
1160	}
1161
1162	OpWithBodyGenInfo &setDataSharingProcessor(DataSharingProcessor *value) {
1163	dsp = value;
1164	return *this;
1165	}
1166
1167	OpWithBodyGenInfo &setEntryBlockArgs(const EntryBlockArgs *value) {
1168	blockArgs = value;
1169	return *this;
1170	}
1171
1172	OpWithBodyGenInfo &setGenRegionEntryCb(GenOMPRegionEntryCBFn value) {
1173	genRegionEntryCB = value;
1174	return *this;
1175	}
1176
1177	OpWithBodyGenInfo &setGenSkeletonOnly(bool value) {
1178	genSkeletonOnly = value;
1179	return *this;
1180	}
1181
1182	/// [inout] converter to use for the clauses.
1183	lower::AbstractConverter &converter;
1184	/// [in] Symbol table
1185	lower::SymMap &symTable;
1186	/// [in] Semantics context
1187	semantics::SemanticsContext &semaCtx;
1188	/// [in] location in source code.
1189	mlir::Location loc;
1190	/// [in] current PFT node/evaluation.
1191	lower::pft::Evaluation &eval;
1192	/// [in] leaf directive for which to generate the op body.
1193	llvm::omp::Directive dir;
1194	/// [in] list of clauses to process.
1195	const List<Clause> *clauses = nullptr;
1196	/// [in] if provided, processes the construct's data-sharing attributes.
1197	DataSharingProcessor *dsp = nullptr;
1198	/// [in] if provided, it is used to create the op's region entry block. It is
1199	/// overriden when a \see genRegionEntryCB is provided. This is only valid for
1200	/// operations implementing the \see mlir::omp::BlockArgOpenMPOpInterface.
1201	const EntryBlockArgs *blockArgs = nullptr;
1202	/// [in] if provided, it overrides the default op's region entry block
1203	/// creation.
1204	GenOMPRegionEntryCBFn genRegionEntryCB = nullptr;
1205	/// [in] if set to `true`, skip generating nested evaluations and dispatching
1206	/// any further leaf constructs.
1207	bool genSkeletonOnly = false;
1208	};
1209
1210	/// Create the body (block) for an OpenMP Operation.
1211	///
1212	/// \param [in] op - the operation the body belongs to.
1213	/// \param [in] info - options controlling code-gen for the construction.
1214	/// \param [in] queue - work queue with nested constructs.
1215	/// \param [in] item - item in the queue to generate body for.
1216	static void createBodyOfOp(mlir::Operation &op, const OpWithBodyGenInfo &info,
1217	const ConstructQueue &queue,
1218	ConstructQueue::const_iterator item) {
1219	fir::FirOpBuilder &firOpBuilder = info.converter.getFirOpBuilder();
1220
1221	auto insertMarker = [](fir::FirOpBuilder &builder) {
1222	mlir::Value undef = builder.create<fir::UndefOp>(builder.getUnknownLoc(),
1223	builder.getIndexType());
1224	return undef.getDefiningOp();
1225	};
1226
1227	// Create the entry block for the region and collect its arguments for use
1228	// within the region. The entry block will be created as follows:
1229	// - By default, it will be empty and have no arguments.
1230	// - Operations implementing the omp::BlockArgOpenMPOpInterface can set the
1231	// `info.blockArgs` pointer so that block arguments will be those
1232	// corresponding to entry block argument-generating clauses. Binding of
1233	// Fortran symbols to the new MLIR values is done automatically.
1234	// - If the `info.genRegionEntryCB` callback is set, it takes precedence and
1235	// allows callers to manually create the entry block with its intended
1236	// list of arguments and to bind these arguments to their corresponding
1237	// Fortran symbols. This is used for e.g. loop induction variables.
1238	auto regionArgs = [&]() -> llvm::SmallVector<const semantics::Symbol *> {
1239	if (info.genRegionEntryCB)
1240	return info.genRegionEntryCB(&op);
1241
1242	if (info.blockArgs) {
1243	genEntryBlock(firOpBuilder, *info.blockArgs, op.getRegion(0));
1244	bindEntryBlockArgs(info.converter,
1245	llvm::cast<mlir::omp::BlockArgOpenMPOpInterface>(op),
1246	*info.blockArgs);
1247	return llvm::to_vector(info.blockArgs->getSyms());
1248	}
1249
1250	firOpBuilder.createBlock(&op.getRegion(0));
1251	return {};
1252	}();
1253
1254	// Mark the earliest insertion point.
1255	mlir::Operation *marker = insertMarker(firOpBuilder);
1256
1257	// If it is an unstructured region, create empty blocks for all evaluations.
1258	if (lower::omp::isLastItemInQueue(item, queue) &&
1259	info.eval.lowerAsUnstructured()) {
1260	lower::createEmptyRegionBlocks<mlir::omp::TerminatorOp, mlir::omp::YieldOp>(
1261	firOpBuilder, info.eval.getNestedEvaluations());
1262	}
1263
1264	// Start with privatization, so that the lowering of the nested
1265	// code will use the right symbols.
1266	bool isLoop = llvm::omp::getDirectiveAssociation(info.dir) ==
1267	llvm::omp::Association::Loop;
1268	bool privatize = info.clauses;
1269
1270	firOpBuilder.setInsertionPoint(marker);
1271	std::optional<DataSharingProcessor> tempDsp;
1272	if (privatize && !info.dsp) {
1273	tempDsp.emplace(info.converter, info.semaCtx, *info.clauses, info.eval,
1274	Fortran::lower::omp::isLastItemInQueue(item, queue),
1275	/*useDelayedPrivatization=*/false, info.symTable);
1276	tempDsp->processStep1();
1277	}
1278
1279	if (info.dir == llvm::omp::Directive::OMPD_parallel) {
1280	threadPrivatizeVars(converter&: info.converter, eval&: info.eval);
1281	if (info.clauses) {
1282	firOpBuilder.setInsertionPoint(marker);
1283	ClauseProcessor(info.converter, info.semaCtx, *info.clauses)
1284	.processCopyin();
1285	}
1286	}
1287
1288	if (!info.genSkeletonOnly) {
1289	if (ConstructQueue::const_iterator next = std::next(x: item);
1290	next != queue.end()) {
1291	genOMPDispatch(info.converter, info.symTable, info.semaCtx, info.eval,
1292	info.loc, queue, next);
1293	} else {
1294	// genFIR(Evaluation&) tries to patch up unterminated blocks, causing
1295	// a lot of complications for our approach if the terminator generation
1296	// is delayed past this point. Insert a temporary terminator here, then
1297	// delete it.
1298	firOpBuilder.setInsertionPointToEnd(&op.getRegion(0).back());
1299	auto *temp = lower::genOpenMPTerminator(firOpBuilder, &op, info.loc);
1300	firOpBuilder.setInsertionPointAfter(marker);
1301	genNestedEvaluations(converter&: info.converter, eval&: info.eval);
1302	temp->erase();
1303	}
1304	}
1305
1306	// Get or create a unique exiting block from the given region, or
1307	// return nullptr if there is no exiting block.
1308	auto getUniqueExit = [&](mlir::Region &region) -> mlir::Block * {
1309	// Find the blocks where the OMP terminator should go. In simple cases
1310	// it is the single block in the operation's region. When the region
1311	// is more complicated, especially with unstructured control flow, there
1312	// may be multiple blocks, and some of them may have non-OMP terminators
1313	// resulting from lowering of the code contained within the operation.
1314	// All the remaining blocks are potential exit points from the op's region.
1315	//
1316	// Explicit control flow cannot exit any OpenMP region (other than via
1317	// STOP), and that is enforced by semantic checks prior to lowering. STOP
1318	// statements are lowered to a function call.
1319
1320	// Collect unterminated blocks.
1321	llvm::SmallVector<mlir::Block *> exits;
1322	for (mlir::Block &b : region) {
1323	if (b.empty() || !b.back().hasTrait<mlir::OpTrait::IsTerminator>())
1324	exits.push_back(&b);
1325	}
1326
1327	if (exits.empty())
1328	return nullptr;
1329	// If there already is a unique exiting block, do not create another one.
1330	// Additionally, some ops (e.g. omp.sections) require only 1 block in
1331	// its region.
1332	if (exits.size() == 1)
1333	return exits[0];
1334	mlir::Block *exit = firOpBuilder.createBlock(&region);
1335	for (mlir::Block *b : exits) {
1336	firOpBuilder.setInsertionPointToEnd(b);
1337	firOpBuilder.create<mlir::cf::BranchOp>(info.loc, exit);
1338	}
1339	return exit;
1340	};
1341
1342	if (auto *exitBlock = getUniqueExit(op.getRegion(0))) {
1343	firOpBuilder.setInsertionPointToEnd(exitBlock);
1344	auto *term = lower::genOpenMPTerminator(firOpBuilder, &op, info.loc);
1345	// Only insert lastprivate code when there actually is an exit block.
1346	// Such a block may not exist if the nested code produced an infinite
1347	// loop (this may not make sense in production code, but a user could
1348	// write that and we should handle it).
1349	firOpBuilder.setInsertionPoint(term);
1350	if (privatize) {
1351	// DataSharingProcessor::processStep2() may create operations before/after
1352	// the one passed as argument. We need to treat loop wrappers and their
1353	// nested loop as a unit, so we need to pass the bottom level wrapper (if
1354	// present). Otherwise, these operations will be inserted within a
1355	// wrapper region.
1356	mlir::Operation *privatizationBottomLevelOp = &op;
1357	if (auto loopNest = llvm::dyn_cast<mlir::omp::LoopNestOp>(op)) {
1358	llvm::SmallVector<mlir::omp::LoopWrapperInterface> wrappers;
1359	loopNest.gatherWrappers(wrappers);
1360	if (!wrappers.empty())
1361	privatizationBottomLevelOp = &*wrappers.front();
1362	}
1363
1364	if (!info.dsp) {
1365	assert(tempDsp.has_value());
1366	tempDsp->processStep2(privatizationBottomLevelOp, isLoop);
1367	} else {
1368	if (isLoop && regionArgs.size() > 0) {
1369	for (const auto &regionArg : regionArgs) {
1370	info.dsp->pushLoopIV(info.converter.getSymbolAddress(*regionArg));
1371	}
1372	}
1373	info.dsp->processStep2(privatizationBottomLevelOp, isLoop);
1374	}
1375	}
1376	}
1377
1378	firOpBuilder.setInsertionPointAfter(marker);
1379	marker->erase();
1380	}
1381
1382	static void genBodyOfTargetDataOp(
1383	lower::AbstractConverter &converter, lower::SymMap &symTable,
1384	semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
1385	mlir::omp::TargetDataOp &dataOp, const EntryBlockArgs &args,
1386	const mlir::Location &currentLocation, const ConstructQueue &queue,
1387	ConstructQueue::const_iterator item) {
1388	fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
1389
1390	genEntryBlock(firOpBuilder, args, dataOp.getRegion());
1391	bindEntryBlockArgs(converter, dataOp, args);
1392
1393	// Insert dummy instruction to remember the insertion position. The
1394	// marker will be deleted by clean up passes since there are no uses.
1395	// Remembering the position for further insertion is important since
1396	// there are hlfir.declares inserted above while setting block arguments
1397	// and new code from the body should be inserted after that.
1398	mlir::Value undefMarker = firOpBuilder.create<fir::UndefOp>(
1399	dataOp.getLoc(), firOpBuilder.getIndexType());
1400
1401	// Create blocks for unstructured regions. This has to be done since
1402	// blocks are initially allocated with the function as the parent region.
1403	if (eval.lowerAsUnstructured()) {
1404	lower::createEmptyRegionBlocks<mlir::omp::TerminatorOp, mlir::omp::YieldOp>(
1405	firOpBuilder, eval.getNestedEvaluations());
1406	}
1407
1408	firOpBuilder.create<mlir::omp::TerminatorOp>(currentLocation);
1409
1410	// Set the insertion point after the marker.
1411	firOpBuilder.setInsertionPointAfter(undefMarker.getDefiningOp());
1412
1413	if (ConstructQueue::const_iterator next = std::next(x: item);
1414	next != queue.end()) {
1415	genOMPDispatch(converter, symTable, semaCtx, eval, currentLocation, queue,
1416	next);
1417	} else {
1418	genNestedEvaluations(converter, eval);
1419	}
1420	}
1421
1422	// This generates intermediate common block member accesses within a region
1423	// and then rebinds the members symbol to the intermediate accessors we have
1424	// generated so that subsequent code generation will utilise these instead.
1425	//
1426	// When the scope changes, the bindings to the intermediate accessors should
1427	// be dropped in place of the original symbol bindings.
1428	//
1429	// This is for utilisation with TargetOp.
1430	static void genIntermediateCommonBlockAccessors(
1431	Fortran::lower::AbstractConverter &converter,
1432	const mlir::Location &currentLocation,
1433	llvm::ArrayRef<mlir::BlockArgument> mapBlockArgs,
1434	llvm::ArrayRef<const Fortran::semantics::Symbol *> mapSyms) {
1435	// Iterate over the symbol list, which will be shorter than the list of
1436	// arguments if new entry block arguments were introduced to implicitly map
1437	// outside values used by the bounds cloned into the target region. In that
1438	// case, the additional block arguments do not need processing here.
1439	for (auto [mapSym, mapArg] : llvm::zip_first(mapSyms, mapBlockArgs)) {
1440	auto *details = mapSym->detailsIf<Fortran::semantics::CommonBlockDetails>();
1441	if (!details)
1442	continue;
1443
1444	for (auto obj : details->objects()) {
1445	auto targetCBMemberBind = Fortran::lower::genCommonBlockMember(
1446	converter, currentLocation, *obj, mapArg);
1447	fir::ExtendedValue sexv = converter.getSymbolExtendedValue(*obj);
1448	fir::ExtendedValue targetCBExv =
1449	getExtendedValue(sexv, targetCBMemberBind);
1450	converter.bindSymbol(*obj, targetCBExv);
1451	}
1452	}
1453	}
1454
1455	// This functions creates a block for the body of the targetOp's region. It adds
1456	// all the symbols present in mapSymbols as block arguments to this block.
1457	static void genBodyOfTargetOp(
1458	lower::AbstractConverter &converter, lower::SymMap &symTable,
1459	semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
1460	mlir::omp::TargetOp &targetOp, const EntryBlockArgs &args,
1461	const mlir::Location &currentLocation, const ConstructQueue &queue,
1462	ConstructQueue::const_iterator item, DataSharingProcessor &dsp) {
1463	fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
1464	auto argIface = llvm::cast<mlir::omp::BlockArgOpenMPOpInterface>(*targetOp);
1465
1466	mlir::Region &region = targetOp.getRegion();
1467	mlir::Block *entryBlock = genEntryBlock(firOpBuilder, args, region);
1468	bindEntryBlockArgs(converter, targetOp, args);
1469	if (!hostEvalInfo.empty())
1470	hostEvalInfo.back().bindOperands(argIface.getHostEvalBlockArgs());
1471
1472	// Check if cloning the bounds introduced any dependency on the outer region.
1473	// If so, then either clone them as well if they are MemoryEffectFree, or else
1474	// copy them to a new temporary and add them to the map and block_argument
1475	// lists and replace their uses with the new temporary.
1476	llvm::SetVector<mlir::Value> valuesDefinedAbove;
1477	mlir::getUsedValuesDefinedAbove(region, valuesDefinedAbove);
1478	while (!valuesDefinedAbove.empty()) {
1479	for (mlir::Value val : valuesDefinedAbove) {
1480	mlir::Operation *valOp = val.getDefiningOp();
1481
1482	// NOTE: We skip BoxDimsOp's as the lesser of two evils is to map the
1483	// indices separately, as the alternative is to eventually map the Box,
1484	// which comes with a fairly large overhead comparatively. We could be
1485	// more robust about this and check using a BackwardsSlice to see if we
1486	// run the risk of mapping a box.
1487	if (valOp && mlir::isMemoryEffectFree(valOp) &&
1488	!mlir::isa<fir::BoxDimsOp>(valOp)) {
1489	mlir::Operation *clonedOp = valOp->clone();
1490	entryBlock->push_front(clonedOp);
1491
1492	auto replace = [entryBlock](mlir::OpOperand &use) {
1493	return use.getOwner()->getBlock() == entryBlock;
1494	};
1495
1496	valOp->getResults().replaceUsesWithIf(clonedOp->getResults(), replace);
1497	valOp->replaceUsesWithIf(clonedOp, replace);
1498	} else {
1499	auto savedIP = firOpBuilder.getInsertionPoint();
1500
1501	if (valOp)
1502	firOpBuilder.setInsertionPointAfter(valOp);
1503	else
1504	// This means val is a block argument
1505	firOpBuilder.setInsertionPoint(targetOp);
1506
1507	auto copyVal =
1508	firOpBuilder.createTemporary(val.getLoc(), val.getType());
1509	firOpBuilder.createStoreWithConvert(copyVal.getLoc(), val, copyVal);
1510
1511	fir::factory::AddrAndBoundsInfo info =
1512	fir::factory::getDataOperandBaseAddr(
1513	firOpBuilder, val, /*isOptional=*/false, val.getLoc());
1514	llvm::SmallVector<mlir::Value> bounds =
1515	fir::factory::genImplicitBoundsOps<mlir::omp::MapBoundsOp,
1516	mlir::omp::MapBoundsType>(
1517	firOpBuilder, info,
1518	hlfir::translateToExtendedValue(val.getLoc(), firOpBuilder,
1519	hlfir::Entity{val})
1520	.first,
1521	/*dataExvIsAssumedSize=*/false, val.getLoc());
1522
1523	std::stringstream name;
1524	firOpBuilder.setInsertionPoint(targetOp);
1525
1526	llvm::omp::OpenMPOffloadMappingFlags mapFlag =
1527	llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT;
1528	mlir::omp::VariableCaptureKind captureKind =
1529	mlir::omp::VariableCaptureKind::ByRef;
1530
1531	mlir::Type eleType = copyVal.getType();
1532	if (auto refType =
1533	mlir::dyn_cast<fir::ReferenceType>(copyVal.getType()))
1534	eleType = refType.getElementType();
1535
1536	if (fir::isa_trivial(eleType) || fir::isa_char(eleType)) {
1537	captureKind = mlir::omp::VariableCaptureKind::ByCopy;
1538	} else if (!fir::isa_builtin_cptr_type(eleType)) {
1539	mapFlag |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TO;
1540	}
1541
1542	mlir::Value mapOp = createMapInfoOp(
1543	firOpBuilder, copyVal.getLoc(), copyVal,
1544	/*varPtrPtr=*/mlir::Value{}, name.str(), bounds,
1545	/*members=*/llvm::SmallVector<mlir::Value>{},
1546	/*membersIndex=*/mlir::ArrayAttr{},
1547	static_cast<
1548	std::underlying_type_t<llvm::omp::OpenMPOffloadMappingFlags>>(
1549	mapFlag),
1550	captureKind, copyVal.getType());
1551
1552	// Get the index of the first non-map argument before modifying mapVars,
1553	// then append an element to mapVars and an associated entry block
1554	// argument at that index.
1555	unsigned insertIndex =
1556	argIface.getMapBlockArgsStart() + argIface.numMapBlockArgs();
1557	targetOp.getMapVarsMutable().append(mapOp);
1558	mlir::Value clonedValArg = region.insertArgument(
1559	insertIndex, copyVal.getType(), copyVal.getLoc());
1560
1561	firOpBuilder.setInsertionPointToStart(entryBlock);
1562	auto loadOp = firOpBuilder.create<fir::LoadOp>(clonedValArg.getLoc(),
1563	clonedValArg);
1564	val.replaceUsesWithIf(loadOp->getResult(0),
1565	[entryBlock](mlir::OpOperand &use) {
1566	return use.getOwner()->getBlock() == entryBlock;
1567	});
1568	firOpBuilder.setInsertionPoint(entryBlock, savedIP);
1569	}
1570	}
1571	valuesDefinedAbove.clear();
1572	mlir::getUsedValuesDefinedAbove(region, valuesDefinedAbove);
1573	}
1574
1575	// Insert dummy instruction to remember the insertion position. The
1576	// marker will be deleted since there are not uses.
1577	// In the HLFIR flow there are hlfir.declares inserted above while
1578	// setting block arguments.
1579	mlir::Value undefMarker = firOpBuilder.create<fir::UndefOp>(
1580	targetOp.getLoc(), firOpBuilder.getIndexType());
1581
1582	// Create blocks for unstructured regions. This has to be done since
1583	// blocks are initially allocated with the function as the parent region.
1584	if (lower::omp::isLastItemInQueue(item, queue) &&
1585	eval.lowerAsUnstructured()) {
1586	lower::createEmptyRegionBlocks<mlir::omp::TerminatorOp, mlir::omp::YieldOp>(
1587	firOpBuilder, eval.getNestedEvaluations());
1588	}
1589
1590	firOpBuilder.create<mlir::omp::TerminatorOp>(currentLocation);
1591
1592	// Create the insertion point after the marker.
1593	firOpBuilder.setInsertionPointAfter(undefMarker.getDefiningOp());
1594
1595	// If we map a common block using it's symbol e.g. map(tofrom: /common_block/)
1596	// and accessing its members within the target region, there is a large
1597	// chance we will end up with uses external to the region accessing the common
1598	// resolve these, we do so by generating new common block member accesses
1599	// within the region, binding them to the member symbol for the scope of the
1600	// region so that subsequent code generation within the region will utilise
1601	// our new member accesses we have created.
1602	genIntermediateCommonBlockAccessors(
1603	converter, currentLocation, argIface.getMapBlockArgs(), args.map.syms);
1604
1605	if (ConstructQueue::const_iterator next = std::next(x: item);
1606	next != queue.end()) {
1607	genOMPDispatch(converter, symTable, semaCtx, eval, currentLocation, queue,
1608	next);
1609	} else {
1610	genNestedEvaluations(converter, eval);
1611	}
1612
1613	dsp.processStep2(targetOp, /*isLoop=*/false);
1614	}
1615
1616	template <typename OpTy, typename... Args>
1617	static OpTy genOpWithBody(const OpWithBodyGenInfo &info,
1618	const ConstructQueue &queue,
1619	ConstructQueue::const_iterator item, Args &&...args) {
1620	auto op = info.converter.getFirOpBuilder().create<OpTy>(
1621	info.loc, std::forward<Args>(args)...);
1622	createBodyOfOp(*op, info, queue, item);
1623	return op;
1624	}
1625
1626	template <typename OpTy, typename ClauseOpsTy>
1627	static OpTy genWrapperOp(lower::AbstractConverter &converter,
1628	mlir::Location loc, const ClauseOpsTy &clauseOps,
1629	const EntryBlockArgs &args) {
1630	static_assert(
1631	OpTy::template hasTrait<mlir::omp::LoopWrapperInterface::Trait>(),
1632	"expected a loop wrapper");
1633	fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
1634
1635	// Create wrapper.
1636	auto op = firOpBuilder.create<OpTy>(loc, clauseOps);
1637
1638	// Create entry block with arguments.
1639	genEntryBlock(firOpBuilder, args, op.getRegion());
1640
1641	return op;
1642	}
1643
1644	//===----------------------------------------------------------------------===//
1645	// Code generation functions for clauses
1646	//===----------------------------------------------------------------------===//
1647
1648	static void genCancelClauses(lower::AbstractConverter &converter,
1649	semantics::SemanticsContext &semaCtx,
1650	const List<Clause> &clauses, mlir::Location loc,
1651	mlir::omp::CancelOperands &clauseOps) {
1652	ClauseProcessor cp(converter, semaCtx, clauses);
1653	cp.processCancelDirectiveName(clauseOps);
1654	cp.processIf(llvm::omp::Directive::OMPD_cancel, clauseOps);
1655	}
1656
1657	static void
1658	genCancellationPointClauses(lower::AbstractConverter &converter,
1659	semantics::SemanticsContext &semaCtx,
1660	const List<Clause> &clauses, mlir::Location loc,
1661	mlir::omp::CancellationPointOperands &clauseOps) {
1662	ClauseProcessor cp(converter, semaCtx, clauses);
1663	cp.processCancelDirectiveName(clauseOps);
1664	}
1665
1666	static void genCriticalDeclareClauses(
1667	lower::AbstractConverter &converter, semantics::SemanticsContext &semaCtx,
1668	const List<Clause> &clauses, mlir::Location loc,
1669	mlir::omp::CriticalDeclareOperands &clauseOps, llvm::StringRef name) {
1670	ClauseProcessor cp(converter, semaCtx, clauses);
1671	cp.processHint(clauseOps);
1672	clauseOps.symName =
1673	mlir::StringAttr::get(converter.getFirOpBuilder().getContext(), name);
1674	}
1675
1676	static void genDistributeClauses(lower::AbstractConverter &converter,
1677	semantics::SemanticsContext &semaCtx,
1678	lower::StatementContext &stmtCtx,
1679	const List<Clause> &clauses,
1680	mlir::Location loc,
1681	mlir::omp::DistributeOperands &clauseOps) {
1682	ClauseProcessor cp(converter, semaCtx, clauses);
1683	cp.processAllocate(clauseOps);
1684	cp.processDistSchedule(stmtCtx, clauseOps);
1685	cp.processOrder(clauseOps);
1686	}
1687
1688	static void genFlushClauses(lower::AbstractConverter &converter,
1689	semantics::SemanticsContext &semaCtx,
1690	const ObjectList &objects,
1691	const List<Clause> &clauses, mlir::Location loc,
1692	llvm::SmallVectorImpl<mlir::Value> &operandRange) {
1693	if (!objects.empty())
1694	genObjectList(objects, converter, operandRange);
1695
1696	ClauseProcessor cp(converter, semaCtx, clauses);
1697	cp.processTODO<clause::AcqRel, clause::Acquire, clause::Release,
1698	clause::SeqCst>(loc, llvm::omp::OMPD_flush);
1699	}
1700
1701	static void
1702	genLoopNestClauses(lower::AbstractConverter &converter,
1703	semantics::SemanticsContext &semaCtx,
1704	lower::pft::Evaluation &eval, const List<Clause> &clauses,
1705	mlir::Location loc, mlir::omp::LoopNestOperands &clauseOps,
1706	llvm::SmallVectorImpl<const semantics::Symbol *> &iv) {
1707	ClauseProcessor cp(converter, semaCtx, clauses);
1708
1709	if (hostEvalInfo.empty() || !hostEvalInfo.back().apply(clauseOps, iv))
1710	cp.processCollapse(loc, eval, clauseOps, iv);
1711
1712	clauseOps.loopInclusive = converter.getFirOpBuilder().getUnitAttr();
1713	}
1714
1715	static void genLoopClauses(
1716	lower::AbstractConverter &converter, semantics::SemanticsContext &semaCtx,
1717	const List<Clause> &clauses, mlir::Location loc,
1718	mlir::omp::LoopOperands &clauseOps,
1719	llvm::SmallVectorImpl<const semantics::Symbol *> &reductionSyms) {
1720	ClauseProcessor cp(converter, semaCtx, clauses);
1721	cp.processBind(clauseOps);
1722	cp.processOrder(clauseOps);
1723	cp.processReduction(loc, clauseOps, reductionSyms);
1724	cp.processTODO<clause::Lastprivate>(loc, llvm::omp::Directive::OMPD_loop);
1725	}
1726
1727	static void genMaskedClauses(lower::AbstractConverter &converter,
1728	semantics::SemanticsContext &semaCtx,
1729	lower::StatementContext &stmtCtx,
1730	const List<Clause> &clauses, mlir::Location loc,
1731	mlir::omp::MaskedOperands &clauseOps) {
1732	ClauseProcessor cp(converter, semaCtx, clauses);
1733	cp.processFilter(stmtCtx, clauseOps);
1734	}
1735
1736	static void
1737	genOrderedRegionClauses(lower::AbstractConverter &converter,
1738	semantics::SemanticsContext &semaCtx,
1739	const List<Clause> &clauses, mlir::Location loc,
1740	mlir::omp::OrderedRegionOperands &clauseOps) {
1741	ClauseProcessor cp(converter, semaCtx, clauses);
1742	cp.processTODO<clause::Simd>(loc, llvm::omp::Directive::OMPD_ordered);
1743	}
1744
1745	static void genParallelClauses(
1746	lower::AbstractConverter &converter, semantics::SemanticsContext &semaCtx,
1747	lower::StatementContext &stmtCtx, const List<Clause> &clauses,
1748	mlir::Location loc, mlir::omp::ParallelOperands &clauseOps,
1749	llvm::SmallVectorImpl<const semantics::Symbol *> &reductionSyms) {
1750	ClauseProcessor cp(converter, semaCtx, clauses);
1751	cp.processAllocate(clauseOps);
1752	cp.processIf(llvm::omp::Directive::OMPD_parallel, clauseOps);
1753
1754	if (hostEvalInfo.empty() || !hostEvalInfo.back().apply(clauseOps))
1755	cp.processNumThreads(stmtCtx, clauseOps);
1756
1757	cp.processProcBind(clauseOps);
1758	cp.processReduction(loc, clauseOps, reductionSyms);
1759	}
1760
1761	static void genScanClauses(lower::AbstractConverter &converter,
1762	semantics::SemanticsContext &semaCtx,
1763	const List<Clause> &clauses, mlir::Location loc,
1764	mlir::omp::ScanOperands &clauseOps) {
1765	ClauseProcessor cp(converter, semaCtx, clauses);
1766	cp.processInclusive(loc, clauseOps);
1767	cp.processExclusive(loc, clauseOps);
1768	}
1769
1770	static void genSectionsClauses(
1771	lower::AbstractConverter &converter, semantics::SemanticsContext &semaCtx,
1772	const List<Clause> &clauses, mlir::Location loc,
1773	mlir::omp::SectionsOperands &clauseOps,
1774	llvm::SmallVectorImpl<const semantics::Symbol *> &reductionSyms) {
1775	ClauseProcessor cp(converter, semaCtx, clauses);
1776	cp.processAllocate(clauseOps);
1777	cp.processNowait(clauseOps);
1778	cp.processReduction(loc, clauseOps, reductionSyms);
1779	// TODO Support delayed privatization.
1780	}
1781
1782	static void genSimdClauses(
1783	lower::AbstractConverter &converter, semantics::SemanticsContext &semaCtx,
1784	const List<Clause> &clauses, mlir::Location loc,
1785	mlir::omp::SimdOperands &clauseOps,
1786	llvm::SmallVectorImpl<const semantics::Symbol *> &reductionSyms) {
1787	ClauseProcessor cp(converter, semaCtx, clauses);
1788	cp.processAligned(clauseOps);
1789	cp.processIf(llvm::omp::Directive::OMPD_simd, clauseOps);
1790	cp.processNontemporal(clauseOps);
1791	cp.processOrder(clauseOps);
1792	cp.processReduction(loc, clauseOps, reductionSyms);
1793	cp.processSafelen(clauseOps);
1794	cp.processSimdlen(clauseOps);
1795
1796	cp.processTODO<clause::Linear>(loc, llvm::omp::Directive::OMPD_simd);
1797	}
1798
1799	static void genSingleClauses(lower::AbstractConverter &converter,
1800	semantics::SemanticsContext &semaCtx,
1801	const List<Clause> &clauses, mlir::Location loc,
1802	mlir::omp::SingleOperands &clauseOps) {
1803	ClauseProcessor cp(converter, semaCtx, clauses);
1804	cp.processAllocate(clauseOps);
1805	cp.processCopyprivate(loc, clauseOps);
1806	cp.processNowait(clauseOps);
1807	// TODO Support delayed privatization.
1808	}
1809
1810	static void genTargetClauses(
1811	lower::AbstractConverter &converter, semantics::SemanticsContext &semaCtx,
1812	lower::SymMap &symTable, lower::StatementContext &stmtCtx,
1813	lower::pft::Evaluation &eval, const List<Clause> &clauses,
1814	mlir::Location loc, mlir::omp::TargetOperands &clauseOps,
1815	DefaultMapsTy &defaultMaps,
1816	llvm::SmallVectorImpl<const semantics::Symbol *> &hasDeviceAddrSyms,
1817	llvm::SmallVectorImpl<const semantics::Symbol *> &isDevicePtrSyms,
1818	llvm::SmallVectorImpl<const semantics::Symbol *> &mapSyms) {
1819	ClauseProcessor cp(converter, semaCtx, clauses);
1820	cp.processBare(clauseOps);
1821	cp.processDefaultMap(stmtCtx, defaultMaps);
1822	cp.processDepend(symTable, stmtCtx, clauseOps);
1823	cp.processDevice(stmtCtx, clauseOps);
1824	cp.processHasDeviceAddr(stmtCtx, clauseOps, hasDeviceAddrSyms);
1825	if (!hostEvalInfo.empty()) {
1826	// Only process host_eval if compiling for the host device.
1827	processHostEvalClauses(converter, semaCtx, stmtCtx, eval, loc);
1828	hostEvalInfo.back().collectValues(clauseOps.hostEvalVars);
1829	}
1830	cp.processIf(llvm::omp::Directive::OMPD_target, clauseOps);
1831	cp.processIsDevicePtr(clauseOps, isDevicePtrSyms);
1832	cp.processMap(loc, stmtCtx, clauseOps, &mapSyms);
1833	cp.processNowait(clauseOps);
1834	cp.processThreadLimit(stmtCtx, clauseOps);
1835
1836	cp.processTODO<clause::Allocate, clause::InReduction, clause::UsesAllocators>(
1837	loc, llvm::omp::Directive::OMPD_target);
1838
1839	// `target private(..)` is only supported in delayed privatization mode.
1840	if (!enableDelayedPrivatizationStaging)
1841	cp.processTODO<clause::Firstprivate, clause::Private>(
1842	loc, llvm::omp::Directive::OMPD_target);
1843	}
1844
1845	static void genTargetDataClauses(
1846	lower::AbstractConverter &converter, semantics::SemanticsContext &semaCtx,
1847	lower::StatementContext &stmtCtx, const List<Clause> &clauses,
1848	mlir::Location loc, mlir::omp::TargetDataOperands &clauseOps,
1849	llvm::SmallVectorImpl<const semantics::Symbol *> &useDeviceAddrSyms,
1850	llvm::SmallVectorImpl<const semantics::Symbol *> &useDevicePtrSyms) {
1851	ClauseProcessor cp(converter, semaCtx, clauses);
1852	cp.processDevice(stmtCtx, clauseOps);
1853	cp.processIf(llvm::omp::Directive::OMPD_target_data, clauseOps);
1854	cp.processMap(loc, stmtCtx, clauseOps);
1855	cp.processUseDeviceAddr(stmtCtx, clauseOps, useDeviceAddrSyms);
1856	cp.processUseDevicePtr(stmtCtx, clauseOps, useDevicePtrSyms);
1857
1858	// This function implements the deprecated functionality of use_device_ptr
1859	// that allows users to provide non-CPTR arguments to it with the caveat
1860	// that the compiler will treat them as use_device_addr. A lot of legacy
1861	// code may still depend on this functionality, so we should support it
1862	// in some manner. We do so currently by simply shifting non-cptr operands
1863	// from the use_device_ptr lists into the use_device_addr lists.
1864	// TODO: Perhaps create a user provideable compiler option that will
1865	// re-introduce a hard-error rather than a warning in these cases.
1866	promoteNonCPtrUseDevicePtrArgsToUseDeviceAddr(
1867	clauseOps.useDeviceAddrVars, useDeviceAddrSyms,
1868	clauseOps.useDevicePtrVars, useDevicePtrSyms);
1869	}
1870
1871	static void genTargetEnterExitUpdateDataClauses(
1872	lower::AbstractConverter &converter, semantics::SemanticsContext &semaCtx,
1873	lower::SymMap &symTable, lower::StatementContext &stmtCtx,
1874	const List<Clause> &clauses, mlir::Location loc,
1875	llvm::omp::Directive directive,
1876	mlir::omp::TargetEnterExitUpdateDataOperands &clauseOps) {
1877	ClauseProcessor cp(converter, semaCtx, clauses);
1878	cp.processDepend(symTable, stmtCtx, clauseOps);
1879	cp.processDevice(stmtCtx, clauseOps);
1880	cp.processIf(directive, clauseOps);
1881
1882	if (directive == llvm::omp::Directive::OMPD_target_update)
1883	cp.processMotionClauses(stmtCtx, clauseOps);
1884	else
1885	cp.processMap(loc, stmtCtx, clauseOps);
1886
1887	cp.processNowait(clauseOps);
1888	}
1889
1890	static void genTaskClauses(
1891	lower::AbstractConverter &converter, semantics::SemanticsContext &semaCtx,
1892	lower::SymMap &symTable, lower::StatementContext &stmtCtx,
1893	const List<Clause> &clauses, mlir::Location loc,
1894	mlir::omp::TaskOperands &clauseOps,
1895	llvm::SmallVectorImpl<const semantics::Symbol *> &inReductionSyms) {
1896	ClauseProcessor cp(converter, semaCtx, clauses);
1897	cp.processAllocate(clauseOps);
1898	cp.processDepend(symTable, stmtCtx, clauseOps);
1899	cp.processFinal(stmtCtx, clauseOps);
1900	cp.processIf(llvm::omp::Directive::OMPD_task, clauseOps);
1901	cp.processInReduction(loc, clauseOps, inReductionSyms);
1902	cp.processMergeable(clauseOps);
1903	cp.processPriority(stmtCtx, clauseOps);
1904	cp.processUntied(clauseOps);
1905	cp.processDetach(clauseOps);
1906
1907	cp.processTODO<clause::Affinity>(loc, llvm::omp::Directive::OMPD_task);
1908	}
1909
1910	static void genTaskgroupClauses(
1911	lower::AbstractConverter &converter, semantics::SemanticsContext &semaCtx,
1912	const List<Clause> &clauses, mlir::Location loc,
1913	mlir::omp::TaskgroupOperands &clauseOps,
1914	llvm::SmallVectorImpl<const semantics::Symbol *> &taskReductionSyms) {
1915	ClauseProcessor cp(converter, semaCtx, clauses);
1916	cp.processAllocate(clauseOps);
1917	cp.processTaskReduction(loc, clauseOps, taskReductionSyms);
1918	}
1919
1920	static void genTaskloopClauses(lower::AbstractConverter &converter,
1921	semantics::SemanticsContext &semaCtx,
1922	lower::StatementContext &stmtCtx,
1923	const List<Clause> &clauses, mlir::Location loc,
1924	mlir::omp::TaskloopOperands &clauseOps) {
1925
1926	ClauseProcessor cp(converter, semaCtx, clauses);
1927	cp.processGrainsize(stmtCtx, clauseOps);
1928	cp.processNumTasks(stmtCtx, clauseOps);
1929
1930	cp.processTODO<clause::Allocate, clause::Collapse, clause::Default,
1931	clause::Final, clause::If, clause::InReduction,
1932	clause::Lastprivate, clause::Mergeable, clause::Nogroup,
1933	clause::Priority, clause::Reduction, clause::Shared,
1934	clause::Untied>(loc, llvm::omp::Directive::OMPD_taskloop);
1935	}
1936
1937	static void genTaskwaitClauses(lower::AbstractConverter &converter,
1938	semantics::SemanticsContext &semaCtx,
1939	const List<Clause> &clauses, mlir::Location loc,
1940	mlir::omp::TaskwaitOperands &clauseOps) {
1941	ClauseProcessor cp(converter, semaCtx, clauses);
1942	cp.processTODO<clause::Depend, clause::Nowait>(
1943	loc, llvm::omp::Directive::OMPD_taskwait);
1944	}
1945
1946	static void genWorkshareClauses(lower::AbstractConverter &converter,
1947	semantics::SemanticsContext &semaCtx,
1948	lower::StatementContext &stmtCtx,
1949	const List<Clause> &clauses, mlir::Location loc,
1950	mlir::omp::WorkshareOperands &clauseOps) {
1951	ClauseProcessor cp(converter, semaCtx, clauses);
1952	cp.processNowait(clauseOps);
1953	}
1954
1955	static void genTeamsClauses(
1956	lower::AbstractConverter &converter, semantics::SemanticsContext &semaCtx,
1957	lower::StatementContext &stmtCtx, const List<Clause> &clauses,
1958	mlir::Location loc, mlir::omp::TeamsOperands &clauseOps,
1959	llvm::SmallVectorImpl<const semantics::Symbol *> &reductionSyms) {
1960	ClauseProcessor cp(converter, semaCtx, clauses);
1961	cp.processAllocate(clauseOps);
1962	cp.processIf(llvm::omp::Directive::OMPD_teams, clauseOps);
1963
1964	if (hostEvalInfo.empty() || !hostEvalInfo.back().apply(clauseOps)) {
1965	cp.processNumTeams(stmtCtx, clauseOps);
1966	cp.processThreadLimit(stmtCtx, clauseOps);
1967	}
1968
1969	cp.processReduction(loc, clauseOps, reductionSyms);
1970	// TODO Support delayed privatization.
1971	}
1972
1973	static void genWsloopClauses(
1974	lower::AbstractConverter &converter, semantics::SemanticsContext &semaCtx,
1975	lower::StatementContext &stmtCtx, const List<Clause> &clauses,
1976	mlir::Location loc, mlir::omp::WsloopOperands &clauseOps,
1977	llvm::SmallVectorImpl<const semantics::Symbol *> &reductionSyms) {
1978	ClauseProcessor cp(converter, semaCtx, clauses);
1979	cp.processNowait(clauseOps);
1980	cp.processLinear(clauseOps);
1981	cp.processOrder(clauseOps);
1982	cp.processOrdered(clauseOps);
1983	cp.processReduction(loc, clauseOps, reductionSyms);
1984	cp.processSchedule(stmtCtx, clauseOps);
1985
1986	cp.processTODO<clause::Allocate>(loc, llvm::omp::Directive::OMPD_do);
1987	}
1988
1989	//===----------------------------------------------------------------------===//
1990	// Code generation functions for leaf constructs
1991	//===----------------------------------------------------------------------===//
1992
1993	static mlir::omp::BarrierOp
1994	genBarrierOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
1995	semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
1996	mlir::Location loc, const ConstructQueue &queue,
1997	ConstructQueue::const_iterator item) {
1998	return converter.getFirOpBuilder().create<mlir::omp::BarrierOp>(loc);
1999	}
2000
2001	static mlir::omp::CancelOp genCancelOp(lower::AbstractConverter &converter,
2002	semantics::SemanticsContext &semaCtx,
2003	lower::pft::Evaluation &eval,
2004	mlir::Location loc,
2005	const ConstructQueue &queue,
2006	ConstructQueue::const_iterator item) {
2007	mlir::omp::CancelOperands clauseOps;
2008	genCancelClauses(converter, semaCtx, item->clauses, loc, clauseOps);
2009
2010	return converter.getFirOpBuilder().create<mlir::omp::CancelOp>(loc,
2011	clauseOps);
2012	}
2013
2014	static mlir::omp::CancellationPointOp genCancellationPointOp(
2015	lower::AbstractConverter &converter, semantics::SemanticsContext &semaCtx,
2016	lower::pft::Evaluation &eval, mlir::Location loc,
2017	const ConstructQueue &queue, ConstructQueue::const_iterator item) {
2018	mlir::omp::CancellationPointOperands clauseOps;
2019	genCancellationPointClauses(converter, semaCtx, item->clauses, loc,
2020	clauseOps);
2021
2022	return converter.getFirOpBuilder().create<mlir::omp::CancellationPointOp>(
2023	loc, clauseOps);
2024	}
2025
2026	static mlir::omp::CriticalOp
2027	genCriticalOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
2028	semantics::SemanticsContext &semaCtx,
2029	lower::pft::Evaluation &eval, mlir::Location loc,
2030	const ConstructQueue &queue, ConstructQueue::const_iterator item,
2031	const std::optional<parser::Name> &name) {
2032	fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
2033	mlir::FlatSymbolRefAttr nameAttr;
2034
2035	if (name) {
2036	std::string nameStr = name->ToString();
2037	mlir::ModuleOp mod = firOpBuilder.getModule();
2038	auto global = mod.lookupSymbol<mlir::omp::CriticalDeclareOp>(nameStr);
2039	if (!global) {
2040	mlir::omp::CriticalDeclareOperands clauseOps;
2041	genCriticalDeclareClauses(converter, semaCtx, item->clauses, loc,
2042	clauseOps, nameStr);
2043
2044	mlir::OpBuilder modBuilder(mod.getBodyRegion());
2045	global = modBuilder.create<mlir::omp::CriticalDeclareOp>(loc, clauseOps);
2046	}
2047	nameAttr = mlir::FlatSymbolRefAttr::get(firOpBuilder.getContext(),
2048	global.getSymName());
2049	}
2050
2051	return genOpWithBody<mlir::omp::CriticalOp>(
2052	OpWithBodyGenInfo(converter, symTable, semaCtx, loc, eval,
2053	llvm::omp::Directive::OMPD_critical),
2054	queue, item, nameAttr);
2055	}
2056
2057	static mlir::omp::FlushOp
2058	genFlushOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
2059	semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
2060	mlir::Location loc, const ObjectList &objects,
2061	const ConstructQueue &queue, ConstructQueue::const_iterator item) {
2062	llvm::SmallVector<mlir::Value> operandRange;
2063	genFlushClauses(converter, semaCtx, objects, item->clauses, loc,
2064	operandRange);
2065
2066	return converter.getFirOpBuilder().create<mlir::omp::FlushOp>(
2067	converter.getCurrentLocation(), operandRange);
2068	}
2069
2070	static mlir::omp::LoopNestOp genLoopNestOp(
2071	lower::AbstractConverter &converter, lower::SymMap &symTable,
2072	semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
2073	mlir::Location loc, const ConstructQueue &queue,
2074	ConstructQueue::const_iterator item, mlir::omp::LoopNestOperands &clauseOps,
2075	llvm::ArrayRef<const semantics::Symbol *> iv,
2076	llvm::ArrayRef<
2077	std::pair<mlir::omp::BlockArgOpenMPOpInterface, const EntryBlockArgs &>>
2078	wrapperArgs,
2079	llvm::omp::Directive directive, DataSharingProcessor &dsp) {
2080	auto ivCallback = [&](mlir::Operation *op) {
2081	genLoopVars(op, converter, loc, iv, wrapperArgs);
2082	return llvm::SmallVector<const semantics::Symbol *>(iv);
2083	};
2084
2085	auto *nestedEval =
2086	getCollapsedLoopEval(eval, collapseValue: getCollapseValue(clauses: item->clauses));
2087
2088	return genOpWithBody<mlir::omp::LoopNestOp>(
2089	OpWithBodyGenInfo(converter, symTable, semaCtx, loc, *nestedEval,
2090	directive)
2091	.setClauses(&item->clauses)
2092	.setDataSharingProcessor(&dsp)
2093	.setGenRegionEntryCb(ivCallback),
2094	queue, item, clauseOps);
2095	}
2096
2097	static mlir::omp::LoopOp
2098	genLoopOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
2099	semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
2100	mlir::Location loc, const ConstructQueue &queue,
2101	ConstructQueue::const_iterator item) {
2102	mlir::omp::LoopOperands loopClauseOps;
2103	llvm::SmallVector<const semantics::Symbol *> loopReductionSyms;
2104	genLoopClauses(converter, semaCtx, item->clauses, loc, loopClauseOps,
2105	loopReductionSyms);
2106
2107	DataSharingProcessor dsp(converter, semaCtx, item->clauses, eval,
2108	/*shouldCollectPreDeterminedSymbols=*/true,
2109	/*useDelayedPrivatization=*/true, symTable);
2110	dsp.processStep1(clauseOps: &loopClauseOps);
2111
2112	mlir::omp::LoopNestOperands loopNestClauseOps;
2113	llvm::SmallVector<const semantics::Symbol *> iv;
2114	genLoopNestClauses(converter, semaCtx, eval, item->clauses, loc,
2115	loopNestClauseOps, iv);
2116
2117	EntryBlockArgs loopArgs;
2118	loopArgs.priv.syms = dsp.getDelayedPrivSymbols();
2119	loopArgs.priv.vars = loopClauseOps.privateVars;
2120	loopArgs.reduction.syms = loopReductionSyms;
2121	loopArgs.reduction.vars = loopClauseOps.reductionVars;
2122
2123	auto loopOp =
2124	genWrapperOp<mlir::omp::LoopOp>(converter, loc, loopClauseOps, loopArgs);
2125	genLoopNestOp(converter, symTable, semaCtx, eval, loc, queue, item,
2126	loopNestClauseOps, iv, {{loopOp, loopArgs}},
2127	llvm::omp::Directive::OMPD_loop, dsp);
2128	return loopOp;
2129	}
2130
2131	static mlir::omp::MaskedOp
2132	genMaskedOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
2133	lower::StatementContext &stmtCtx,
2134	semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
2135	mlir::Location loc, const ConstructQueue &queue,
2136	ConstructQueue::const_iterator item) {
2137	mlir::omp::MaskedOperands clauseOps;
2138	genMaskedClauses(converter, semaCtx, stmtCtx, item->clauses, loc, clauseOps);
2139
2140	return genOpWithBody<mlir::omp::MaskedOp>(
2141	OpWithBodyGenInfo(converter, symTable, semaCtx, loc, eval,
2142	llvm::omp::Directive::OMPD_masked),
2143	queue, item, clauseOps);
2144	}
2145
2146	static mlir::omp::MasterOp
2147	genMasterOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
2148	semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
2149	mlir::Location loc, const ConstructQueue &queue,
2150	ConstructQueue::const_iterator item) {
2151	return genOpWithBody<mlir::omp::MasterOp>(
2152	OpWithBodyGenInfo(converter, symTable, semaCtx, loc, eval,
2153	llvm::omp::Directive::OMPD_master),
2154	queue, item);
2155	}
2156
2157	static mlir::omp::OrderedOp
2158	genOrderedOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
2159	semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
2160	mlir::Location loc, const ConstructQueue &queue,
2161	ConstructQueue::const_iterator item) {
2162	TODO(loc, "OMPD_ordered");
2163	return nullptr;
2164	}
2165
2166	static mlir::omp::OrderedRegionOp
2167	genOrderedRegionOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
2168	semantics::SemanticsContext &semaCtx,
2169	lower::pft::Evaluation &eval, mlir::Location loc,
2170	const ConstructQueue &queue,
2171	ConstructQueue::const_iterator item) {
2172	mlir::omp::OrderedRegionOperands clauseOps;
2173	genOrderedRegionClauses(converter, semaCtx, item->clauses, loc, clauseOps);
2174
2175	return genOpWithBody<mlir::omp::OrderedRegionOp>(
2176	OpWithBodyGenInfo(converter, symTable, semaCtx, loc, eval,
2177	llvm::omp::Directive::OMPD_ordered),
2178	queue, item, clauseOps);
2179	}
2180
2181	static mlir::omp::ParallelOp
2182	genParallelOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
2183	semantics::SemanticsContext &semaCtx,
2184	lower::pft::Evaluation &eval, mlir::Location loc,
2185	const ConstructQueue &queue, ConstructQueue::const_iterator item,
2186	mlir::omp::ParallelOperands &clauseOps,
2187	const EntryBlockArgs &args, DataSharingProcessor *dsp,
2188	bool isComposite = false) {
2189	assert((!enableDelayedPrivatization || dsp) &&
2190	"expected valid DataSharingProcessor");
2191
2192	OpWithBodyGenInfo genInfo =
2193	OpWithBodyGenInfo(converter, symTable, semaCtx, loc, eval,
2194	llvm::omp::Directive::OMPD_parallel)
2195	.setClauses(&item->clauses)
2196	.setEntryBlockArgs(&args)
2197	.setGenSkeletonOnly(isComposite)
2198	.setDataSharingProcessor(dsp);
2199
2200	auto parallelOp =
2201	genOpWithBody<mlir::omp::ParallelOp>(genInfo, queue, item, clauseOps);
2202	parallelOp.setComposite(isComposite);
2203	return parallelOp;
2204	}
2205
2206	static mlir::omp::ScanOp
2207	genScanOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
2208	semantics::SemanticsContext &semaCtx, mlir::Location loc,
2209	const ConstructQueue &queue, ConstructQueue::const_iterator item) {
2210	mlir::omp::ScanOperands clauseOps;
2211	genScanClauses(converter, semaCtx, item->clauses, loc, clauseOps);
2212	return converter.getFirOpBuilder().create<mlir::omp::ScanOp>(
2213	converter.getCurrentLocation(), clauseOps);
2214	}
2215
2216	/// This breaks the normal prototype of the gen*Op functions: adding the
2217	/// sectionBlocks argument so that the enclosed section constructs can be
2218	/// lowered here with correct reduction symbol remapping.
2219	static mlir::omp::SectionsOp
2220	genSectionsOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
2221	semantics::SemanticsContext &semaCtx,
2222	lower::pft::Evaluation &eval, mlir::Location loc,
2223	const ConstructQueue &queue, ConstructQueue::const_iterator item,
2224	const parser::OmpSectionBlocks &sectionBlocks) {
2225	mlir::omp::SectionsOperands clauseOps;
2226	llvm::SmallVector<const semantics::Symbol *> reductionSyms;
2227	genSectionsClauses(converter, semaCtx, item->clauses, loc, clauseOps,
2228	reductionSyms);
2229
2230	auto &builder = converter.getFirOpBuilder();
2231
2232	// Insert privatizations before SECTIONS
2233	lower::SymMapScope scope(symTable);
2234	DataSharingProcessor dsp(converter, semaCtx, item->clauses, eval,
2235	lower::omp::isLastItemInQueue(item, queue),
2236	/*useDelayedPrivatization=*/false, symTable);
2237	dsp.processStep1();
2238
2239	List<Clause> nonDsaClauses;
2240	List<const clause::Lastprivate *> lastprivates;
2241
2242	for (const Clause &clause : item->clauses) {
2243	if (clause.id == llvm::omp::Clause::OMPC_lastprivate) {
2244	auto &lastp = std::get<clause::Lastprivate>(clause.u);
2245	lastprivateModifierNotSupported(lastp, converter.getCurrentLocation());
2246	lastprivates.push_back(&lastp);
2247	} else {
2248	switch (clause.id) {
2249	case llvm::omp::Clause::OMPC_firstprivate:
2250	case llvm::omp::Clause::OMPC_private:
2251	case llvm::omp::Clause::OMPC_shared:
2252	break;
2253	default:
2254	nonDsaClauses.push_back(Elt: clause);
2255	}
2256	}
2257	}
2258
2259	// SECTIONS construct.
2260	auto sectionsOp = builder.create<mlir::omp::SectionsOp>(loc, clauseOps);
2261
2262	// Create entry block with reduction variables as arguments.
2263	EntryBlockArgs args;
2264	// TODO: Add private syms and vars.
2265	args.reduction.syms = reductionSyms;
2266	args.reduction.vars = clauseOps.reductionVars;
2267
2268	genEntryBlock(builder, args, sectionsOp.getRegion());
2269	mlir::Operation *terminator =
2270	lower::genOpenMPTerminator(builder, sectionsOp, loc);
2271
2272	// Generate nested SECTION constructs.
2273	// This is done here rather than in genOMP([...], OpenMPSectionConstruct )
2274	// because we need to run genReductionVars on each omp.section so that the
2275	// reduction variable gets mapped to the private version
2276	for (auto [construct, nestedEval] :
2277	llvm::zip(sectionBlocks.v, eval.getNestedEvaluations())) {
2278	const auto *sectionConstruct =
2279	std::get_if<parser::OpenMPSectionConstruct>(&construct.u);
2280	if (!sectionConstruct) {
2281	assert(false &&
2282	"unexpected construct nested inside of SECTIONS construct");
2283	continue;
2284	}
2285
2286	ConstructQueue sectionQueue{buildConstructQueue(
2287	converter.getFirOpBuilder().getModule(), semaCtx, nestedEval,
2288	sectionConstruct->source, llvm::omp::Directive::OMPD_section, {})};
2289
2290	builder.setInsertionPoint(terminator);
2291	genOpWithBody<mlir::omp::SectionOp>(
2292	OpWithBodyGenInfo(converter, symTable, semaCtx, loc, nestedEval,
2293	llvm::omp::Directive::OMPD_section)
2294	.setClauses(&sectionQueue.begin()->clauses)
2295	.setDataSharingProcessor(&dsp)
2296	.setEntryBlockArgs(&args),
2297	sectionQueue, sectionQueue.begin());
2298	}
2299
2300	if (!lastprivates.empty()) {
2301	mlir::Region &sectionsBody = sectionsOp.getRegion();
2302	assert(sectionsBody.hasOneBlock());
2303	mlir::Block &body = sectionsBody.front();
2304
2305	auto lastSectionOp = llvm::find_if(
2306	llvm::reverse(body.getOperations()), [](const mlir::Operation &op) {
2307	return llvm::isa<mlir::omp::SectionOp>(op);
2308	});
2309	assert(lastSectionOp != body.rend());
2310
2311	for (const clause::Lastprivate *lastp : lastprivates) {
2312	builder.setInsertionPoint(
2313	lastSectionOp->getRegion(0).back().getTerminator());
2314	mlir::OpBuilder::InsertPoint insp = builder.saveInsertionPoint();
2315	const auto &objList = std::get<ObjectList>(lastp->t);
2316	for (const Object &object : objList) {
2317	semantics::Symbol *sym = object.sym();
2318	if (const auto *common =
2319	sym->detailsIf<semantics::CommonBlockDetails>()) {
2320	for (const auto &obj : common->objects())
2321	converter.copyHostAssociateVar(*obj, &insp, /*hostIsSource=*/false);
2322	} else {
2323	converter.copyHostAssociateVar(*sym, &insp, /*hostIsSource=*/false);
2324	}
2325	}
2326	}
2327	}
2328
2329	// Perform DataSharingProcessor's step2 out of SECTIONS
2330	builder.setInsertionPointAfter(sectionsOp.getOperation());
2331	dsp.processStep2(sectionsOp, false);
2332	// Emit implicit barrier to synchronize threads and avoid data
2333	// races on post-update of lastprivate variables when `nowait`
2334	// clause is present.
2335	if (clauseOps.nowait && !lastprivates.empty())
2336	builder.create<mlir::omp::BarrierOp>(loc);
2337
2338	return sectionsOp;
2339	}
2340
2341	static mlir::Operation *
2342	genScopeOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
2343	semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
2344	mlir::Location loc, const ConstructQueue &queue,
2345	ConstructQueue::const_iterator item) {
2346	TODO(loc, "Scope construct");
2347	return nullptr;
2348	}
2349
2350	static mlir::omp::SingleOp
2351	genSingleOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
2352	semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
2353	mlir::Location loc, const ConstructQueue &queue,
2354	ConstructQueue::const_iterator item) {
2355	mlir::omp::SingleOperands clauseOps;
2356	genSingleClauses(converter, semaCtx, item->clauses, loc, clauseOps);
2357
2358	return genOpWithBody<mlir::omp::SingleOp>(
2359	OpWithBodyGenInfo(converter, symTable, semaCtx, loc, eval,
2360	llvm::omp::Directive::OMPD_single)
2361	.setClauses(&item->clauses),
2362	queue, item, clauseOps);
2363	}
2364
2365	static mlir::omp::TargetOp
2366	genTargetOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
2367	lower::StatementContext &stmtCtx,
2368	semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
2369	mlir::Location loc, const ConstructQueue &queue,
2370	ConstructQueue::const_iterator item) {
2371	fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
2372	bool isTargetDevice =
2373	llvm::cast<mlir::omp::OffloadModuleInterface>(*converter.getModuleOp())
2374	.getIsTargetDevice();
2375
2376	// Introduce a new host_eval information structure for this target region.
2377	if (!isTargetDevice)
2378	hostEvalInfo.emplace_back();
2379
2380	mlir::omp::TargetOperands clauseOps;
2381	DefaultMapsTy defaultMaps;
2382	llvm::SmallVector<const semantics::Symbol *> mapSyms, isDevicePtrSyms,
2383	hasDeviceAddrSyms;
2384	genTargetClauses(converter, semaCtx, symTable, stmtCtx, eval, item->clauses,
2385	loc, clauseOps, defaultMaps, hasDeviceAddrSyms,
2386	isDevicePtrSyms, mapSyms);
2387
2388	DataSharingProcessor dsp(converter, semaCtx, item->clauses, eval,
2389	/*shouldCollectPreDeterminedSymbols=*/
2390	lower::omp::isLastItemInQueue(item, queue),
2391	/*useDelayedPrivatization=*/true, symTable);
2392	dsp.processStep1(clauseOps: &clauseOps);
2393
2394	// 5.8.1 Implicit Data-Mapping Attribute Rules
2395	// The following code follows the implicit data-mapping rules to map all the
2396	// symbols used inside the region that do not have explicit data-environment
2397	// attribute clauses (neither data-sharing; e.g. `private`, nor `map`
2398	// clauses).
2399	auto captureImplicitMap = [&](const semantics::Symbol &sym) {
2400	if (dsp.getAllSymbolsToPrivatize().contains(&sym))
2401	return;
2402
2403	// These symbols are mapped individually in processHasDeviceAddr.
2404	if (llvm::is_contained(Range&: hasDeviceAddrSyms, Element: &sym))
2405	return;
2406
2407	// Structure component symbols don't have bindings, and can only be
2408	// explicitly mapped individually. If a member is captured implicitly
2409	// we map the entirety of the derived type when we find its symbol.
2410	if (sym.owner().IsDerivedType())
2411	return;
2412
2413	// if the symbol is part of an already mapped common block, do not make a
2414	// map for it.
2415	if (const Fortran::semantics::Symbol *common =
2416	Fortran::semantics::FindCommonBlockContaining(sym.GetUltimate()))
2417	if (llvm::is_contained(Range&: mapSyms, Element: common))
2418	return;
2419
2420	// If we come across a symbol without a symbol address, we
2421	// return as we cannot process it, this is intended as a
2422	// catch all early exit for symbols that do not have a
2423	// corresponding extended value. Such as subroutines,
2424	// interfaces and named blocks.
2425	if (!converter.getSymbolAddress(sym))
2426	return;
2427
2428	if (!llvm::is_contained(Range&: mapSyms, Element: &sym)) {
2429	if (const auto *details =
2430	sym.template detailsIf<semantics::HostAssocDetails>())
2431	converter.copySymbolBinding(details->symbol(), sym);
2432	std::stringstream name;
2433	fir::ExtendedValue dataExv = converter.getSymbolExtendedValue(sym);
2434	name << sym.name().ToString();
2435
2436	mlir::FlatSymbolRefAttr mapperId;
2437	if (sym.GetType()->category() == semantics::DeclTypeSpec::TypeDerived) {
2438	auto &typeSpec = sym.GetType()->derivedTypeSpec();
2439	std::string mapperIdName =
2440	typeSpec.name().ToString() + llvm::omp::OmpDefaultMapperName;
2441	if (auto *sym = converter.getCurrentScope().FindSymbol(mapperIdName))
2442	mapperIdName = converter.mangleName(mapperIdName, sym->owner());
2443	if (converter.getModuleOp().lookupSymbol(mapperIdName))
2444	mapperId = mlir::FlatSymbolRefAttr::get(&converter.getMLIRContext(),
2445	mapperIdName);
2446	}
2447
2448	fir::factory::AddrAndBoundsInfo info =
2449	Fortran::lower::getDataOperandBaseAddr(
2450	converter, firOpBuilder, sym.GetUltimate(),
2451	converter.getCurrentLocation());
2452	llvm::SmallVector<mlir::Value> bounds =
2453	fir::factory::genImplicitBoundsOps<mlir::omp::MapBoundsOp,
2454	mlir::omp::MapBoundsType>(
2455	firOpBuilder, info, dataExv,
2456	semantics::IsAssumedSizeArray(sym.GetUltimate()),
2457	converter.getCurrentLocation());
2458	mlir::Value baseOp = info.rawInput;
2459	mlir::Type eleType = baseOp.getType();
2460	if (auto refType = mlir::dyn_cast<fir::ReferenceType>(baseOp.getType()))
2461	eleType = refType.getElementType();
2462
2463	std::pair<llvm::omp::OpenMPOffloadMappingFlags,
2464	mlir::omp::VariableCaptureKind>
2465	mapFlagAndKind = getImplicitMapTypeAndKind(
2466	firOpBuilder, converter, defaultMaps, eleType, loc, sym);
2467
2468	mlir::Value mapOp = createMapInfoOp(
2469	firOpBuilder, converter.getCurrentLocation(), baseOp,
2470	/*varPtrPtr=*/mlir::Value{}, name.str(), bounds, /*members=*/{},
2471	/*membersIndex=*/mlir::ArrayAttr{},
2472	static_cast<
2473	std::underlying_type_t<llvm::omp::OpenMPOffloadMappingFlags>>(
2474	std::get<0>(mapFlagAndKind)),
2475	std::get<1>(mapFlagAndKind), baseOp.getType(),
2476	/*partialMap=*/false, mapperId);
2477
2478	clauseOps.mapVars.push_back(mapOp);
2479	mapSyms.push_back(Elt: &sym);
2480	}
2481	};
2482	lower::pft::visitAllSymbols(eval, captureImplicitMap);
2483
2484	auto targetOp = firOpBuilder.create<mlir::omp::TargetOp>(loc, clauseOps);
2485
2486	llvm::SmallVector<mlir::Value> hasDeviceAddrBaseValues, mapBaseValues;
2487	extractMappedBaseValues(clauseOps.hasDeviceAddrVars, hasDeviceAddrBaseValues);
2488	extractMappedBaseValues(clauseOps.mapVars, mapBaseValues);
2489
2490	EntryBlockArgs args;
2491	args.hasDeviceAddr.syms = hasDeviceAddrSyms;
2492	args.hasDeviceAddr.vars = hasDeviceAddrBaseValues;
2493	args.hostEvalVars = clauseOps.hostEvalVars;
2494	// TODO: Add in_reduction syms and vars.
2495	args.map.syms = mapSyms;
2496	args.map.vars = mapBaseValues;
2497	args.priv.syms = dsp.getDelayedPrivSymbols();
2498	args.priv.vars = clauseOps.privateVars;
2499
2500	genBodyOfTargetOp(converter, symTable, semaCtx, eval, targetOp, args, loc,
2501	queue, item, dsp);
2502
2503	// Remove the host_eval information structure created for this target region.
2504	if (!isTargetDevice)
2505	hostEvalInfo.pop_back();
2506	return targetOp;
2507	}
2508
2509	static mlir::omp::TargetDataOp genTargetDataOp(
2510	lower::AbstractConverter &converter, lower::SymMap &symTable,
2511	lower::StatementContext &stmtCtx, semantics::SemanticsContext &semaCtx,
2512	lower::pft::Evaluation &eval, mlir::Location loc,
2513	const ConstructQueue &queue, ConstructQueue::const_iterator item) {
2514	mlir::omp::TargetDataOperands clauseOps;
2515	llvm::SmallVector<const semantics::Symbol *> useDeviceAddrSyms,
2516	useDevicePtrSyms;
2517	genTargetDataClauses(converter, semaCtx, stmtCtx, item->clauses, loc,
2518	clauseOps, useDeviceAddrSyms, useDevicePtrSyms);
2519
2520	auto targetDataOp =
2521	converter.getFirOpBuilder().create<mlir::omp::TargetDataOp>(loc,
2522	clauseOps);
2523
2524	llvm::SmallVector<mlir::Value> useDeviceAddrBaseValues,
2525	useDevicePtrBaseValues;
2526	extractMappedBaseValues(clauseOps.useDeviceAddrVars, useDeviceAddrBaseValues);
2527	extractMappedBaseValues(clauseOps.useDevicePtrVars, useDevicePtrBaseValues);
2528
2529	EntryBlockArgs args;
2530	args.useDeviceAddr.syms = useDeviceAddrSyms;
2531	args.useDeviceAddr.vars = useDeviceAddrBaseValues;
2532	args.useDevicePtr.syms = useDevicePtrSyms;
2533	args.useDevicePtr.vars = useDevicePtrBaseValues;
2534
2535	genBodyOfTargetDataOp(converter, symTable, semaCtx, eval, targetDataOp, args,
2536	loc, queue, item);
2537	return targetDataOp;
2538	}
2539
2540	template <typename OpTy>
2541	static OpTy genTargetEnterExitUpdateDataOp(
2542	lower::AbstractConverter &converter, lower::SymMap &symTable,
2543	lower::StatementContext &stmtCtx, semantics::SemanticsContext &semaCtx,
2544	mlir::Location loc, const ConstructQueue &queue,
2545	ConstructQueue::const_iterator item) {
2546	fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
2547
2548	// GCC 9.3.0 emits a (probably) bogus warning about an unused variable.
2549	[[maybe_unused]] llvm::omp::Directive directive;
2550	if constexpr (std::is_same_v<OpTy, mlir::omp::TargetEnterDataOp>) {
2551	directive = llvm::omp::Directive::OMPD_target_enter_data;
2552	} else if constexpr (std::is_same_v<OpTy, mlir::omp::TargetExitDataOp>) {
2553	directive = llvm::omp::Directive::OMPD_target_exit_data;
2554	} else if constexpr (std::is_same_v<OpTy, mlir::omp::TargetUpdateOp>) {
2555	directive = llvm::omp::Directive::OMPD_target_update;
2556	} else {
2557	llvm_unreachable("Unexpected TARGET DATA construct");
2558	}
2559
2560	mlir::omp::TargetEnterExitUpdateDataOperands clauseOps;
2561	genTargetEnterExitUpdateDataClauses(converter, semaCtx, symTable, stmtCtx,
2562	item->clauses, loc, directive, clauseOps);
2563
2564	return firOpBuilder.create<OpTy>(loc, clauseOps);
2565	}
2566
2567	static mlir::omp::TaskOp
2568	genTaskOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
2569	lower::StatementContext &stmtCtx,
2570	semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
2571	mlir::Location loc, const ConstructQueue &queue,
2572	ConstructQueue::const_iterator item) {
2573	mlir::omp::TaskOperands clauseOps;
2574	llvm::SmallVector<const semantics::Symbol *> inReductionSyms;
2575	genTaskClauses(converter, semaCtx, symTable, stmtCtx, item->clauses, loc,
2576	clauseOps, inReductionSyms);
2577
2578	if (!enableDelayedPrivatization)
2579	return genOpWithBody<mlir::omp::TaskOp>(
2580	OpWithBodyGenInfo(converter, symTable, semaCtx, loc, eval,
2581	llvm::omp::Directive::OMPD_task)
2582	.setClauses(&item->clauses),
2583	queue, item, clauseOps);
2584
2585	DataSharingProcessor dsp(converter, semaCtx, item->clauses, eval,
2586	lower::omp::isLastItemInQueue(item, queue),
2587	/*useDelayedPrivatization=*/true, symTable);
2588	dsp.processStep1(clauseOps: &clauseOps);
2589
2590	EntryBlockArgs taskArgs;
2591	taskArgs.priv.syms = dsp.getDelayedPrivSymbols();
2592	taskArgs.priv.vars = clauseOps.privateVars;
2593	taskArgs.inReduction.syms = inReductionSyms;
2594	taskArgs.inReduction.vars = clauseOps.inReductionVars;
2595
2596	return genOpWithBody<mlir::omp::TaskOp>(
2597	OpWithBodyGenInfo(converter, symTable, semaCtx, loc, eval,
2598	llvm::omp::Directive::OMPD_task)
2599	.setClauses(&item->clauses)
2600	.setDataSharingProcessor(&dsp)
2601	.setEntryBlockArgs(&taskArgs),
2602	queue, item, clauseOps);
2603	}
2604
2605	static mlir::omp::TaskgroupOp
2606	genTaskgroupOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
2607	semantics::SemanticsContext &semaCtx,
2608	lower::pft::Evaluation &eval, mlir::Location loc,
2609	const ConstructQueue &queue,
2610	ConstructQueue::const_iterator item) {
2611	mlir::omp::TaskgroupOperands clauseOps;
2612	llvm::SmallVector<const semantics::Symbol *> taskReductionSyms;
2613	genTaskgroupClauses(converter, semaCtx, item->clauses, loc, clauseOps,
2614	taskReductionSyms);
2615
2616	EntryBlockArgs taskgroupArgs;
2617	taskgroupArgs.taskReduction.syms = taskReductionSyms;
2618	taskgroupArgs.taskReduction.vars = clauseOps.taskReductionVars;
2619
2620	return genOpWithBody<mlir::omp::TaskgroupOp>(
2621	OpWithBodyGenInfo(converter, symTable, semaCtx, loc, eval,
2622	llvm::omp::Directive::OMPD_taskgroup)
2623	.setClauses(&item->clauses)
2624	.setEntryBlockArgs(&taskgroupArgs),
2625	queue, item, clauseOps);
2626	}
2627
2628	static mlir::omp::TaskwaitOp
2629	genTaskwaitOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
2630	semantics::SemanticsContext &semaCtx,
2631	lower::pft::Evaluation &eval, mlir::Location loc,
2632	const ConstructQueue &queue,
2633	ConstructQueue::const_iterator item) {
2634	mlir::omp::TaskwaitOperands clauseOps;
2635	genTaskwaitClauses(converter, semaCtx, item->clauses, loc, clauseOps);
2636	return converter.getFirOpBuilder().create<mlir::omp::TaskwaitOp>(loc,
2637	clauseOps);
2638	}
2639
2640	static mlir::omp::TaskyieldOp
2641	genTaskyieldOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
2642	semantics::SemanticsContext &semaCtx,
2643	lower::pft::Evaluation &eval, mlir::Location loc,
2644	const ConstructQueue &queue,
2645	ConstructQueue::const_iterator item) {
2646	return converter.getFirOpBuilder().create<mlir::omp::TaskyieldOp>(loc);
2647	}
2648
2649	static mlir::omp::WorkshareOp genWorkshareOp(
2650	lower::AbstractConverter &converter, lower::SymMap &symTable,
2651	lower::StatementContext &stmtCtx, semantics::SemanticsContext &semaCtx,
2652	lower::pft::Evaluation &eval, mlir::Location loc,
2653	const ConstructQueue &queue, ConstructQueue::const_iterator item) {
2654	mlir::omp::WorkshareOperands clauseOps;
2655	genWorkshareClauses(converter, semaCtx, stmtCtx, item->clauses, loc,
2656	clauseOps);
2657
2658	return genOpWithBody<mlir::omp::WorkshareOp>(
2659	OpWithBodyGenInfo(converter, symTable, semaCtx, loc, eval,
2660	llvm::omp::Directive::OMPD_workshare)
2661	.setClauses(&item->clauses),
2662	queue, item, clauseOps);
2663	}
2664
2665	static mlir::omp::TeamsOp
2666	genTeamsOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
2667	lower::StatementContext &stmtCtx,
2668	semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
2669	mlir::Location loc, const ConstructQueue &queue,
2670	ConstructQueue::const_iterator item) {
2671	mlir::omp::TeamsOperands clauseOps;
2672	llvm::SmallVector<const semantics::Symbol *> reductionSyms;
2673	genTeamsClauses(converter, semaCtx, stmtCtx, item->clauses, loc, clauseOps,
2674	reductionSyms);
2675
2676	EntryBlockArgs args;
2677	// TODO: Add private syms and vars.
2678	args.reduction.syms = reductionSyms;
2679	args.reduction.vars = clauseOps.reductionVars;
2680
2681	return genOpWithBody<mlir::omp::TeamsOp>(
2682	OpWithBodyGenInfo(converter, symTable, semaCtx, loc, eval,
2683	llvm::omp::Directive::OMPD_teams)
2684	.setClauses(&item->clauses)
2685	.setEntryBlockArgs(&args),
2686	queue, item, clauseOps);
2687	}
2688
2689	//===----------------------------------------------------------------------===//
2690	// Code generation for atomic operations
2691	//===----------------------------------------------------------------------===//
2692	static fir::FirOpBuilder::InsertPoint
2693	getInsertionPointBefore(mlir::Operation *op) {
2694	return fir::FirOpBuilder::InsertPoint(op->getBlock(),
2695	mlir::Block::iterator(op));
2696	}
2697
2698	static fir::FirOpBuilder::InsertPoint
2699	getInsertionPointAfter(mlir::Operation *op) {
2700	return fir::FirOpBuilder::InsertPoint(op->getBlock(),
2701	++mlir::Block::iterator(op));
2702	}
2703
2704	static mlir::IntegerAttr getAtomicHint(lower::AbstractConverter &converter,
2705	const List<Clause> &clauses) {
2706	fir::FirOpBuilder &builder = converter.getFirOpBuilder();
2707	for (const Clause &clause : clauses) {
2708	if (clause.id != llvm::omp::Clause::OMPC_hint)
2709	continue;
2710	auto &hint = std::get<clause::Hint>(clause.u);
2711	auto maybeVal = evaluate::ToInt64(hint.v);
2712	CHECK(maybeVal);
2713	return builder.getI64IntegerAttr(*maybeVal);
2714	}
2715	return nullptr;
2716	}
2717
2718	static mlir::omp::ClauseMemoryOrderKindAttr
2719	getAtomicMemoryOrder(lower::AbstractConverter &converter,
2720	semantics::SemanticsContext &semaCtx,
2721	const List<Clause> &clauses) {
2722	std::optional<mlir::omp::ClauseMemoryOrderKind> kind;
2723	unsigned version = semaCtx.langOptions().OpenMPVersion;
2724
2725	for (const Clause &clause : clauses) {
2726	switch (clause.id) {
2727	case llvm::omp::Clause::OMPC_acq_rel:
2728	kind = mlir::omp::ClauseMemoryOrderKind::Acq_rel;
2729	break;
2730	case llvm::omp::Clause::OMPC_acquire:
2731	kind = mlir::omp::ClauseMemoryOrderKind::Acquire;
2732	break;
2733	case llvm::omp::Clause::OMPC_relaxed:
2734	kind = mlir::omp::ClauseMemoryOrderKind::Relaxed;
2735	break;
2736	case llvm::omp::Clause::OMPC_release:
2737	kind = mlir::omp::ClauseMemoryOrderKind::Release;
2738	break;
2739	case llvm::omp::Clause::OMPC_seq_cst:
2740	kind = mlir::omp::ClauseMemoryOrderKind::Seq_cst;
2741	break;
2742	default:
2743	break;
2744	}
2745	}
2746
2747	// Starting with 5.1, if no memory-order clause is present, the effect
2748	// is as if "relaxed" was present.
2749	if (!kind) {
2750	if (version <= 50)
2751	return nullptr;
2752	kind = mlir::omp::ClauseMemoryOrderKind::Relaxed;
2753	}
2754	fir::FirOpBuilder &builder = converter.getFirOpBuilder();
2755	return mlir::omp::ClauseMemoryOrderKindAttr::get(builder.getContext(), *kind);
2756	}
2757
2758	static mlir::Operation * //
2759	genAtomicRead(lower::AbstractConverter &converter, mlir::Location loc,
2760	lower::StatementContext &stmtCtx, mlir::Value atomAddr,
2761	const semantics::SomeExpr &atom,
2762	const evaluate::Assignment &assign, mlir::IntegerAttr hint,
2763	mlir::omp::ClauseMemoryOrderKindAttr memOrder,
2764	fir::FirOpBuilder::InsertPoint preAt,
2765	fir::FirOpBuilder::InsertPoint atomicAt,
2766	fir::FirOpBuilder::InsertPoint postAt) {
2767	fir::FirOpBuilder &builder = converter.getFirOpBuilder();
2768	builder.restoreInsertionPoint(preAt);
2769
2770	mlir::Value storeAddr =
2771	fir::getBase(converter.genExprAddr(assign.lhs, stmtCtx, &loc));
2772	mlir::Type atomType = fir::unwrapRefType(atomAddr.getType());
2773	mlir::Type storeType = fir::unwrapRefType(storeAddr.getType());
2774
2775	mlir::Value toAddr = [&]() {
2776	if (atomType == storeType)
2777	return storeAddr;
2778	return builder.createTemporary(loc, atomType, ".tmp.atomval");
2779	}();
2780
2781	builder.restoreInsertionPoint(atomicAt);
2782	mlir::Operation *op = builder.create<mlir::omp::AtomicReadOp>(
2783	loc, atomAddr, toAddr, mlir::TypeAttr::get(atomType), hint, memOrder);
2784
2785	if (atomType != storeType) {
2786	lower::ExprToValueMap overrides;
2787	// The READ operation could be a part of UPDATE CAPTURE, so make sure
2788	// we don't emit extra code into the body of the atomic op.
2789	builder.restoreInsertionPoint(postAt);
2790	mlir::Value load = builder.create<fir::LoadOp>(loc, toAddr);
2791	overrides.try_emplace(&atom, load);
2792
2793	converter.overrideExprValues(&overrides);
2794	mlir::Value value =
2795	fir::getBase(converter.genExprValue(assign.rhs, stmtCtx, &loc));
2796	converter.resetExprOverrides();
2797
2798	builder.create<fir::StoreOp>(loc, value, storeAddr);
2799	}
2800	return op;
2801	}
2802
2803	static mlir::Operation * //
2804	genAtomicWrite(lower::AbstractConverter &converter, mlir::Location loc,
2805	lower::StatementContext &stmtCtx, mlir::Value atomAddr,
2806	const semantics::SomeExpr &atom,
2807	const evaluate::Assignment &assign, mlir::IntegerAttr hint,
2808	mlir::omp::ClauseMemoryOrderKindAttr memOrder,
2809	fir::FirOpBuilder::InsertPoint preAt,
2810	fir::FirOpBuilder::InsertPoint atomicAt,
2811	fir::FirOpBuilder::InsertPoint postAt) {
2812	fir::FirOpBuilder &builder = converter.getFirOpBuilder();
2813	builder.restoreInsertionPoint(preAt);
2814
2815	mlir::Value value =
2816	fir::getBase(converter.genExprValue(assign.rhs, stmtCtx, &loc));
2817	mlir::Type atomType = fir::unwrapRefType(atomAddr.getType());
2818	mlir::Value converted = builder.createConvert(loc, atomType, value);
2819
2820	builder.restoreInsertionPoint(atomicAt);
2821	mlir::Operation *op = builder.create<mlir::omp::AtomicWriteOp>(
2822	loc, atomAddr, converted, hint, memOrder);
2823	return op;
2824	}
2825
2826	static mlir::Operation *
2827	genAtomicUpdate(lower::AbstractConverter &converter, mlir::Location loc,
2828	lower::StatementContext &stmtCtx, mlir::Value atomAddr,
2829	const semantics::SomeExpr &atom,
2830	const evaluate::Assignment &assign, mlir::IntegerAttr hint,
2831	mlir::omp::ClauseMemoryOrderKindAttr memOrder,
2832	fir::FirOpBuilder::InsertPoint preAt,
2833	fir::FirOpBuilder::InsertPoint atomicAt,
2834	fir::FirOpBuilder::InsertPoint postAt) {
2835	lower::ExprToValueMap overrides;
2836	lower::StatementContext naCtx;
2837	fir::FirOpBuilder &builder = converter.getFirOpBuilder();
2838	builder.restoreInsertionPoint(preAt);
2839
2840	mlir::Type atomType = fir::unwrapRefType(atomAddr.getType());
2841
2842	// This must exist by now.
2843	SomeExpr input = *semantics::GetConvertInput(assign.rhs);
2844	std::vector<SomeExpr> args{semantics::GetTopLevelOperation(input).second};
2845	assert(!args.empty() && "Update operation without arguments");
2846	for (auto &arg : args) {
2847	if (!semantics::IsSameOrConvertOf(arg, atom)) {
2848	mlir::Value val = fir::getBase(converter.genExprValue(arg, naCtx, &loc));
2849	overrides.try_emplace(&arg, val);
2850	}
2851	}
2852
2853	builder.restoreInsertionPoint(atomicAt);
2854	auto updateOp =
2855	builder.create<mlir::omp::AtomicUpdateOp>(loc, atomAddr, hint, memOrder);
2856
2857	mlir::Region &region = updateOp->getRegion(0);
2858	mlir::Block *block = builder.createBlock(&region, {}, {atomType}, {loc});
2859	mlir::Value localAtom = fir::getBase(block->getArgument(0));
2860	overrides.try_emplace(&atom, localAtom);
2861
2862	converter.overrideExprValues(&overrides);
2863	mlir::Value updated =
2864	fir::getBase(converter.genExprValue(assign.rhs, stmtCtx, &loc));
2865	mlir::Value converted = builder.createConvert(loc, atomType, updated);
2866	builder.create<mlir::omp::YieldOp>(loc, converted);
2867	converter.resetExprOverrides();
2868
2869	builder.restoreInsertionPoint(postAt); // For naCtx cleanups
2870	return updateOp;
2871	}
2872
2873	static mlir::Operation *
2874	genAtomicOperation(lower::AbstractConverter &converter, mlir::Location loc,
2875	lower::StatementContext &stmtCtx, int action,
2876	mlir::Value atomAddr, const semantics::SomeExpr &atom,
2877	const evaluate::Assignment &assign, mlir::IntegerAttr hint,
2878	mlir::omp::ClauseMemoryOrderKindAttr memOrder,
2879	fir::FirOpBuilder::InsertPoint preAt,
2880	fir::FirOpBuilder::InsertPoint atomicAt,
2881	fir::FirOpBuilder::InsertPoint postAt) {
2882	if (isPointerAssignment(assign)) {
2883	TODO(loc, "Code generation for pointer assignment is not implemented yet");
2884	}
2885
2886	// This function and the functions called here do not preserve the
2887	// builder's insertion point, or set it to anything specific.
2888	switch (action) {
2889	case parser::OpenMPAtomicConstruct::Analysis::Read:
2890	return genAtomicRead(converter, loc, stmtCtx, atomAddr, atom, assign, hint,
2891	memOrder, preAt, atomicAt, postAt);
2892	case parser::OpenMPAtomicConstruct::Analysis::Write:
2893	return genAtomicWrite(converter, loc, stmtCtx, atomAddr, atom, assign, hint,
2894	memOrder, preAt, atomicAt, postAt);
2895	case parser::OpenMPAtomicConstruct::Analysis::Update:
2896	return genAtomicUpdate(converter, loc, stmtCtx, atomAddr, atom, assign,
2897	hint, memOrder, preAt, atomicAt, postAt);
2898	default:
2899	return nullptr;
2900	}
2901	}
2902
2903	//===----------------------------------------------------------------------===//
2904	// Code generation functions for the standalone version of constructs that can
2905	// also be a leaf of a composite construct
2906	//===----------------------------------------------------------------------===//
2907
2908	static mlir::omp::DistributeOp genStandaloneDistribute(
2909	lower::AbstractConverter &converter, lower::SymMap &symTable,
2910	lower::StatementContext &stmtCtx, semantics::SemanticsContext &semaCtx,
2911	lower::pft::Evaluation &eval, mlir::Location loc,
2912	const ConstructQueue &queue, ConstructQueue::const_iterator item) {
2913	mlir::omp::DistributeOperands distributeClauseOps;
2914	genDistributeClauses(converter, semaCtx, stmtCtx, item->clauses, loc,
2915	distributeClauseOps);
2916
2917	DataSharingProcessor dsp(converter, semaCtx, item->clauses, eval,
2918	/*shouldCollectPreDeterminedSymbols=*/true,
2919	enableDelayedPrivatization, symTable);
2920	dsp.processStep1(clauseOps: &distributeClauseOps);
2921
2922	mlir::omp::LoopNestOperands loopNestClauseOps;
2923	llvm::SmallVector<const semantics::Symbol *> iv;
2924	genLoopNestClauses(converter, semaCtx, eval, item->clauses, loc,
2925	loopNestClauseOps, iv);
2926
2927	EntryBlockArgs distributeArgs;
2928	distributeArgs.priv.syms = dsp.getDelayedPrivSymbols();
2929	distributeArgs.priv.vars = distributeClauseOps.privateVars;
2930	auto distributeOp = genWrapperOp<mlir::omp::DistributeOp>(
2931	converter, loc, distributeClauseOps, distributeArgs);
2932
2933	genLoopNestOp(converter, symTable, semaCtx, eval, loc, queue, item,
2934	loopNestClauseOps, iv, {{distributeOp, distributeArgs}},
2935	llvm::omp::Directive::OMPD_distribute, dsp);
2936	return distributeOp;
2937	}
2938
2939	static mlir::omp::WsloopOp genStandaloneDo(
2940	lower::AbstractConverter &converter, lower::SymMap &symTable,
2941	lower::StatementContext &stmtCtx, semantics::SemanticsContext &semaCtx,
2942	lower::pft::Evaluation &eval, mlir::Location loc,
2943	const ConstructQueue &queue, ConstructQueue::const_iterator item) {
2944	mlir::omp::WsloopOperands wsloopClauseOps;
2945	llvm::SmallVector<const semantics::Symbol *> wsloopReductionSyms;
2946	genWsloopClauses(converter, semaCtx, stmtCtx, item->clauses, loc,
2947	wsloopClauseOps, wsloopReductionSyms);
2948
2949	DataSharingProcessor dsp(converter, semaCtx, item->clauses, eval,
2950	/*shouldCollectPreDeterminedSymbols=*/true,
2951	enableDelayedPrivatization, symTable);
2952	dsp.processStep1(clauseOps: &wsloopClauseOps);
2953
2954	mlir::omp::LoopNestOperands loopNestClauseOps;
2955	llvm::SmallVector<const semantics::Symbol *> iv;
2956	genLoopNestClauses(converter, semaCtx, eval, item->clauses, loc,
2957	loopNestClauseOps, iv);
2958
2959	EntryBlockArgs wsloopArgs;
2960	wsloopArgs.priv.syms = dsp.getDelayedPrivSymbols();
2961	wsloopArgs.priv.vars = wsloopClauseOps.privateVars;
2962	wsloopArgs.reduction.syms = wsloopReductionSyms;
2963	wsloopArgs.reduction.vars = wsloopClauseOps.reductionVars;
2964	auto wsloopOp = genWrapperOp<mlir::omp::WsloopOp>(
2965	converter, loc, wsloopClauseOps, wsloopArgs);
2966
2967	genLoopNestOp(converter, symTable, semaCtx, eval, loc, queue, item,
2968	loopNestClauseOps, iv, {{wsloopOp, wsloopArgs}},
2969	llvm::omp::Directive::OMPD_do, dsp);
2970	return wsloopOp;
2971	}
2972
2973	static mlir::omp::ParallelOp genStandaloneParallel(
2974	lower::AbstractConverter &converter, lower::SymMap &symTable,
2975	lower::StatementContext &stmtCtx, semantics::SemanticsContext &semaCtx,
2976	lower::pft::Evaluation &eval, mlir::Location loc,
2977	const ConstructQueue &queue, ConstructQueue::const_iterator item) {
2978	mlir::omp::ParallelOperands parallelClauseOps;
2979	llvm::SmallVector<const semantics::Symbol *> parallelReductionSyms;
2980	genParallelClauses(converter, semaCtx, stmtCtx, item->clauses, loc,
2981	parallelClauseOps, parallelReductionSyms);
2982
2983	std::optional<DataSharingProcessor> dsp;
2984	if (enableDelayedPrivatization) {
2985	dsp.emplace(converter, semaCtx, item->clauses, eval,
2986	lower::omp::isLastItemInQueue(item, queue),
2987	/*useDelayedPrivatization=*/true, symTable);
2988	dsp->processStep1(clauseOps: &parallelClauseOps);
2989	}
2990
2991	EntryBlockArgs parallelArgs;
2992	if (dsp)
2993	parallelArgs.priv.syms = dsp->getDelayedPrivSymbols();
2994	parallelArgs.priv.vars = parallelClauseOps.privateVars;
2995	parallelArgs.reduction.syms = parallelReductionSyms;
2996	parallelArgs.reduction.vars = parallelClauseOps.reductionVars;
2997	return genParallelOp(converter, symTable, semaCtx, eval, loc, queue, item,
2998	parallelClauseOps, parallelArgs,
2999	enableDelayedPrivatization ? &dsp.value() : nullptr);
3000	}
3001
3002	static mlir::omp::SimdOp
3003	genStandaloneSimd(lower::AbstractConverter &converter, lower::SymMap &symTable,
3004	semantics::SemanticsContext &semaCtx,
3005	lower::pft::Evaluation &eval, mlir::Location loc,
3006	const ConstructQueue &queue,
3007	ConstructQueue::const_iterator item) {
3008	mlir::omp::SimdOperands simdClauseOps;
3009	llvm::SmallVector<const semantics::Symbol *> simdReductionSyms;
3010	genSimdClauses(converter, semaCtx, item->clauses, loc, simdClauseOps,
3011	simdReductionSyms);
3012
3013	DataSharingProcessor dsp(converter, semaCtx, item->clauses, eval,
3014	/*shouldCollectPreDeterminedSymbols=*/true,
3015	enableDelayedPrivatization, symTable);
3016	dsp.processStep1(clauseOps: &simdClauseOps);
3017
3018	mlir::omp::LoopNestOperands loopNestClauseOps;
3019	llvm::SmallVector<const semantics::Symbol *> iv;
3020	genLoopNestClauses(converter, semaCtx, eval, item->clauses, loc,
3021	loopNestClauseOps, iv);
3022
3023	EntryBlockArgs simdArgs;
3024	simdArgs.priv.syms = dsp.getDelayedPrivSymbols();
3025	simdArgs.priv.vars = simdClauseOps.privateVars;
3026	simdArgs.reduction.syms = simdReductionSyms;
3027	simdArgs.reduction.vars = simdClauseOps.reductionVars;
3028	auto simdOp =
3029	genWrapperOp<mlir::omp::SimdOp>(converter, loc, simdClauseOps, simdArgs);
3030
3031	genLoopNestOp(converter, symTable, semaCtx, eval, loc, queue, item,
3032	loopNestClauseOps, iv, {{simdOp, simdArgs}},
3033	llvm::omp::Directive::OMPD_simd, dsp);
3034	return simdOp;
3035	}
3036
3037	static mlir::omp::TaskloopOp genStandaloneTaskloop(
3038	lower::AbstractConverter &converter, lower::SymMap &symTable,
3039	lower::StatementContext &stmtCtx, semantics::SemanticsContext &semaCtx,
3040	lower::pft::Evaluation &eval, mlir::Location loc,
3041	const ConstructQueue &queue, ConstructQueue::const_iterator item) {
3042	mlir::omp::TaskloopOperands taskloopClauseOps;
3043	genTaskloopClauses(converter, semaCtx, stmtCtx, item->clauses, loc,
3044	taskloopClauseOps);
3045	DataSharingProcessor dsp(converter, semaCtx, item->clauses, eval,
3046	/*shouldCollectPreDeterminedSymbols=*/true,
3047	enableDelayedPrivatization, symTable);
3048	dsp.processStep1(clauseOps: &taskloopClauseOps);
3049
3050	mlir::omp::LoopNestOperands loopNestClauseOps;
3051	llvm::SmallVector<const semantics::Symbol *> iv;
3052	genLoopNestClauses(converter, semaCtx, eval, item->clauses, loc,
3053	loopNestClauseOps, iv);
3054
3055	EntryBlockArgs taskloopArgs;
3056	taskloopArgs.priv.syms = dsp.getDelayedPrivSymbols();
3057	taskloopArgs.priv.vars = taskloopClauseOps.privateVars;
3058
3059	auto taskLoopOp = genWrapperOp<mlir::omp::TaskloopOp>(
3060	converter, loc, taskloopClauseOps, taskloopArgs);
3061
3062	genLoopNestOp(converter, symTable, semaCtx, eval, loc, queue, item,
3063	loopNestClauseOps, iv, {{taskLoopOp, taskloopArgs}},
3064	llvm::omp::Directive::OMPD_taskloop, dsp);
3065	return taskLoopOp;
3066	}
3067
3068	//===----------------------------------------------------------------------===//
3069	// Code generation functions for composite constructs
3070	//===----------------------------------------------------------------------===//
3071
3072	static mlir::omp::DistributeOp genCompositeDistributeParallelDo(
3073	lower::AbstractConverter &converter, lower::SymMap &symTable,
3074	lower::StatementContext &stmtCtx, semantics::SemanticsContext &semaCtx,
3075	lower::pft::Evaluation &eval, mlir::Location loc,
3076	const ConstructQueue &queue, ConstructQueue::const_iterator item) {
3077	assert(std::distance(item, queue.end()) == 3 && "Invalid leaf constructs");
3078	ConstructQueue::const_iterator distributeItem = item;
3079	ConstructQueue::const_iterator parallelItem = std::next(x: distributeItem);
3080	ConstructQueue::const_iterator doItem = std::next(x: parallelItem);
3081
3082	// Create parent omp.parallel first.
3083	mlir::omp::ParallelOperands parallelClauseOps;
3084	llvm::SmallVector<const semantics::Symbol *> parallelReductionSyms;
3085	genParallelClauses(converter, semaCtx, stmtCtx, parallelItem->clauses, loc,
3086	parallelClauseOps, parallelReductionSyms);
3087
3088	DataSharingProcessor dsp(converter, semaCtx, doItem->clauses, eval,
3089	/*shouldCollectPreDeterminedSymbols=*/true,
3090	/*useDelayedPrivatization=*/true, symTable);
3091	dsp.processStep1(clauseOps: &parallelClauseOps);
3092
3093	EntryBlockArgs parallelArgs;
3094	parallelArgs.priv.syms = dsp.getDelayedPrivSymbols();
3095	parallelArgs.priv.vars = parallelClauseOps.privateVars;
3096	parallelArgs.reduction.syms = parallelReductionSyms;
3097	parallelArgs.reduction.vars = parallelClauseOps.reductionVars;
3098	genParallelOp(converter, symTable, semaCtx, eval, loc, queue, parallelItem,
3099	parallelClauseOps, parallelArgs, &dsp, /*isComposite=*/true);
3100
3101	// Clause processing.
3102	mlir::omp::DistributeOperands distributeClauseOps;
3103	genDistributeClauses(converter, semaCtx, stmtCtx, distributeItem->clauses,
3104	loc, distributeClauseOps);
3105
3106	mlir::omp::WsloopOperands wsloopClauseOps;
3107	llvm::SmallVector<const semantics::Symbol *> wsloopReductionSyms;
3108	genWsloopClauses(converter, semaCtx, stmtCtx, doItem->clauses, loc,
3109	wsloopClauseOps, wsloopReductionSyms);
3110
3111	mlir::omp::LoopNestOperands loopNestClauseOps;
3112	llvm::SmallVector<const semantics::Symbol *> iv;
3113	genLoopNestClauses(converter, semaCtx, eval, doItem->clauses, loc,
3114	loopNestClauseOps, iv);
3115
3116	// Operation creation.
3117	EntryBlockArgs distributeArgs;
3118	// TODO: Add private syms and vars.
3119	auto distributeOp = genWrapperOp<mlir::omp::DistributeOp>(
3120	converter, loc, distributeClauseOps, distributeArgs);
3121	distributeOp.setComposite(/*val=*/true);
3122
3123	EntryBlockArgs wsloopArgs;
3124	// TODO: Add private syms and vars.
3125	wsloopArgs.reduction.syms = wsloopReductionSyms;
3126	wsloopArgs.reduction.vars = wsloopClauseOps.reductionVars;
3127	auto wsloopOp = genWrapperOp<mlir::omp::WsloopOp>(
3128	converter, loc, wsloopClauseOps, wsloopArgs);
3129	wsloopOp.setComposite(/*val=*/true);
3130
3131	genLoopNestOp(converter, symTable, semaCtx, eval, loc, queue, doItem,
3132	loopNestClauseOps, iv,
3133	{{distributeOp, distributeArgs}, {wsloopOp, wsloopArgs}},
3134	llvm::omp::Directive::OMPD_distribute_parallel_do, dsp);
3135	return distributeOp;
3136	}
3137
3138	static mlir::omp::DistributeOp genCompositeDistributeParallelDoSimd(
3139	lower::AbstractConverter &converter, lower::SymMap &symTable,
3140	lower::StatementContext &stmtCtx, semantics::SemanticsContext &semaCtx,
3141	lower::pft::Evaluation &eval, mlir::Location loc,
3142	const ConstructQueue &queue, ConstructQueue::const_iterator item) {
3143	assert(std::distance(item, queue.end()) == 4 && "Invalid leaf constructs");
3144	ConstructQueue::const_iterator distributeItem = item;
3145	ConstructQueue::const_iterator parallelItem = std::next(x: distributeItem);
3146	ConstructQueue::const_iterator doItem = std::next(x: parallelItem);
3147	ConstructQueue::const_iterator simdItem = std::next(x: doItem);
3148
3149	// Create parent omp.parallel first.
3150	mlir::omp::ParallelOperands parallelClauseOps;
3151	llvm::SmallVector<const semantics::Symbol *> parallelReductionSyms;
3152	genParallelClauses(converter, semaCtx, stmtCtx, parallelItem->clauses, loc,
3153	parallelClauseOps, parallelReductionSyms);
3154
3155	DataSharingProcessor parallelItemDSP(
3156	converter, semaCtx, parallelItem->clauses, eval,
3157	/*shouldCollectPreDeterminedSymbols=*/false,
3158	/*useDelayedPrivatization=*/true, symTable);
3159	parallelItemDSP.processStep1(clauseOps: &parallelClauseOps);
3160
3161	EntryBlockArgs parallelArgs;
3162	parallelArgs.priv.syms = parallelItemDSP.getDelayedPrivSymbols();
3163	parallelArgs.priv.vars = parallelClauseOps.privateVars;
3164	parallelArgs.reduction.syms = parallelReductionSyms;
3165	parallelArgs.reduction.vars = parallelClauseOps.reductionVars;
3166	genParallelOp(converter, symTable, semaCtx, eval, loc, queue, parallelItem,
3167	parallelClauseOps, parallelArgs, &parallelItemDSP,
3168	/*isComposite=*/true);
3169
3170	// Clause processing.
3171	mlir::omp::DistributeOperands distributeClauseOps;
3172	genDistributeClauses(converter, semaCtx, stmtCtx, distributeItem->clauses,
3173	loc, distributeClauseOps);
3174
3175	mlir::omp::WsloopOperands wsloopClauseOps;
3176	llvm::SmallVector<const semantics::Symbol *> wsloopReductionSyms;
3177	genWsloopClauses(converter, semaCtx, stmtCtx, doItem->clauses, loc,
3178	wsloopClauseOps, wsloopReductionSyms);
3179
3180	mlir::omp::SimdOperands simdClauseOps;
3181	llvm::SmallVector<const semantics::Symbol *> simdReductionSyms;
3182	genSimdClauses(converter, semaCtx, simdItem->clauses, loc, simdClauseOps,
3183	simdReductionSyms);
3184
3185	DataSharingProcessor simdItemDSP(converter, semaCtx, simdItem->clauses, eval,
3186	/*shouldCollectPreDeterminedSymbols=*/true,
3187	/*useDelayedPrivatization=*/true, symTable);
3188	simdItemDSP.processStep1(clauseOps: &simdClauseOps);
3189
3190	mlir::omp::LoopNestOperands loopNestClauseOps;
3191	llvm::SmallVector<const semantics::Symbol *> iv;
3192	genLoopNestClauses(converter, semaCtx, eval, simdItem->clauses, loc,
3193	loopNestClauseOps, iv);
3194
3195	// Operation creation.
3196	EntryBlockArgs distributeArgs;
3197	// TODO: Add private syms and vars.
3198	auto distributeOp = genWrapperOp<mlir::omp::DistributeOp>(
3199	converter, loc, distributeClauseOps, distributeArgs);
3200	distributeOp.setComposite(/*val=*/true);
3201
3202	EntryBlockArgs wsloopArgs;
3203	// TODO: Add private syms and vars.
3204	wsloopArgs.reduction.syms = wsloopReductionSyms;
3205	wsloopArgs.reduction.vars = wsloopClauseOps.reductionVars;
3206	auto wsloopOp = genWrapperOp<mlir::omp::WsloopOp>(
3207	converter, loc, wsloopClauseOps, wsloopArgs);
3208	wsloopOp.setComposite(/*val=*/true);
3209
3210	EntryBlockArgs simdArgs;
3211	simdArgs.priv.syms = simdItemDSP.getDelayedPrivSymbols();
3212	simdArgs.priv.vars = simdClauseOps.privateVars;
3213	simdArgs.reduction.syms = simdReductionSyms;
3214	simdArgs.reduction.vars = simdClauseOps.reductionVars;
3215	auto simdOp =
3216	genWrapperOp<mlir::omp::SimdOp>(converter, loc, simdClauseOps, simdArgs);
3217	simdOp.setComposite(/*val=*/true);
3218
3219	genLoopNestOp(converter, symTable, semaCtx, eval, loc, queue, simdItem,
3220	loopNestClauseOps, iv,
3221	{{distributeOp, distributeArgs},
3222	{wsloopOp, wsloopArgs},
3223	{simdOp, simdArgs}},
3224	llvm::omp::Directive::OMPD_distribute_parallel_do_simd,
3225	simdItemDSP);
3226	return distributeOp;
3227	}
3228
3229	static mlir::omp::DistributeOp genCompositeDistributeSimd(
3230	lower::AbstractConverter &converter, lower::SymMap &symTable,
3231	lower::StatementContext &stmtCtx, semantics::SemanticsContext &semaCtx,
3232	lower::pft::Evaluation &eval, mlir::Location loc,
3233	const ConstructQueue &queue, ConstructQueue::const_iterator item) {
3234	assert(std::distance(item, queue.end()) == 2 && "Invalid leaf constructs");
3235	ConstructQueue::const_iterator distributeItem = item;
3236	ConstructQueue::const_iterator simdItem = std::next(x: distributeItem);
3237
3238	// Clause processing.
3239	mlir::omp::DistributeOperands distributeClauseOps;
3240	genDistributeClauses(converter, semaCtx, stmtCtx, distributeItem->clauses,
3241	loc, distributeClauseOps);
3242
3243	mlir::omp::SimdOperands simdClauseOps;
3244	llvm::SmallVector<const semantics::Symbol *> simdReductionSyms;
3245	genSimdClauses(converter, semaCtx, simdItem->clauses, loc, simdClauseOps,
3246	simdReductionSyms);
3247
3248	// TODO: Support delayed privatization.
3249	DataSharingProcessor dsp(converter, semaCtx, simdItem->clauses, eval,
3250	/*shouldCollectPreDeterminedSymbols=*/true,
3251	/*useDelayedPrivatization=*/false, symTable);
3252	dsp.processStep1();
3253
3254	// Pass the innermost leaf construct's clauses because that's where COLLAPSE
3255	// is placed by construct decomposition.
3256	mlir::omp::LoopNestOperands loopNestClauseOps;
3257	llvm::SmallVector<const semantics::Symbol *> iv;
3258	genLoopNestClauses(converter, semaCtx, eval, simdItem->clauses, loc,
3259	loopNestClauseOps, iv);
3260
3261	// Operation creation.
3262	EntryBlockArgs distributeArgs;
3263	// TODO: Add private syms and vars.
3264	auto distributeOp = genWrapperOp<mlir::omp::DistributeOp>(
3265	converter, loc, distributeClauseOps, distributeArgs);
3266	distributeOp.setComposite(/*val=*/true);
3267
3268	EntryBlockArgs simdArgs;
3269	// TODO: Add private syms and vars.
3270	simdArgs.reduction.syms = simdReductionSyms;
3271	simdArgs.reduction.vars = simdClauseOps.reductionVars;
3272	auto simdOp =
3273	genWrapperOp<mlir::omp::SimdOp>(converter, loc, simdClauseOps, simdArgs);
3274	simdOp.setComposite(/*val=*/true);
3275
3276	genLoopNestOp(converter, symTable, semaCtx, eval, loc, queue, simdItem,
3277	loopNestClauseOps, iv,
3278	{{distributeOp, distributeArgs}, {simdOp, simdArgs}},
3279	llvm::omp::Directive::OMPD_distribute_simd, dsp);
3280	return distributeOp;
3281	}
3282
3283	static mlir::omp::WsloopOp genCompositeDoSimd(
3284	lower::AbstractConverter &converter, lower::SymMap &symTable,
3285	lower::StatementContext &stmtCtx, semantics::SemanticsContext &semaCtx,
3286	lower::pft::Evaluation &eval, mlir::Location loc,
3287	const ConstructQueue &queue, ConstructQueue::const_iterator item) {
3288	assert(std::distance(item, queue.end()) == 2 && "Invalid leaf constructs");
3289	ConstructQueue::const_iterator doItem = item;
3290	ConstructQueue::const_iterator simdItem = std::next(x: doItem);
3291
3292	// Clause processing.
3293	mlir::omp::WsloopOperands wsloopClauseOps;
3294	llvm::SmallVector<const semantics::Symbol *> wsloopReductionSyms;
3295	genWsloopClauses(converter, semaCtx, stmtCtx, doItem->clauses, loc,
3296	wsloopClauseOps, wsloopReductionSyms);
3297
3298	mlir::omp::SimdOperands simdClauseOps;
3299	llvm::SmallVector<const semantics::Symbol *> simdReductionSyms;
3300	genSimdClauses(converter, semaCtx, simdItem->clauses, loc, simdClauseOps,
3301	simdReductionSyms);
3302
3303	// TODO: Support delayed privatization.
3304	DataSharingProcessor dsp(converter, semaCtx, simdItem->clauses, eval,
3305	/*shouldCollectPreDeterminedSymbols=*/true,
3306	/*useDelayedPrivatization=*/false, symTable);
3307	dsp.processStep1();
3308
3309	// Pass the innermost leaf construct's clauses because that's where COLLAPSE
3310	// is placed by construct decomposition.
3311	mlir::omp::LoopNestOperands loopNestClauseOps;
3312	llvm::SmallVector<const semantics::Symbol *> iv;
3313	genLoopNestClauses(converter, semaCtx, eval, simdItem->clauses, loc,
3314	loopNestClauseOps, iv);
3315
3316	// Operation creation.
3317	EntryBlockArgs wsloopArgs;
3318	// TODO: Add private syms and vars.
3319	wsloopArgs.reduction.syms = wsloopReductionSyms;
3320	wsloopArgs.reduction.vars = wsloopClauseOps.reductionVars;
3321	auto wsloopOp = genWrapperOp<mlir::omp::WsloopOp>(
3322	converter, loc, wsloopClauseOps, wsloopArgs);
3323	wsloopOp.setComposite(/*val=*/true);
3324
3325	EntryBlockArgs simdArgs;
3326	// TODO: Add private syms and vars.
3327	simdArgs.reduction.syms = simdReductionSyms;
3328	simdArgs.reduction.vars = simdClauseOps.reductionVars;
3329	auto simdOp =
3330	genWrapperOp<mlir::omp::SimdOp>(converter, loc, simdClauseOps, simdArgs);
3331	simdOp.setComposite(/*val=*/true);
3332
3333	genLoopNestOp(converter, symTable, semaCtx, eval, loc, queue, simdItem,
3334	loopNestClauseOps, iv,
3335	{{wsloopOp, wsloopArgs}, {simdOp, simdArgs}},
3336	llvm::omp::Directive::OMPD_do_simd, dsp);
3337	return wsloopOp;
3338	}
3339
3340	static mlir::omp::TaskloopOp genCompositeTaskloopSimd(
3341	lower::AbstractConverter &converter, lower::SymMap &symTable,
3342	lower::StatementContext &stmtCtx, semantics::SemanticsContext &semaCtx,
3343	lower::pft::Evaluation &eval, mlir::Location loc,
3344	const ConstructQueue &queue, ConstructQueue::const_iterator item) {
3345	assert(std::distance(item, queue.end()) == 2 && "Invalid leaf constructs");
3346	TODO(loc, "Composite TASKLOOP SIMD");
3347	return nullptr;
3348	}
3349
3350	//===----------------------------------------------------------------------===//
3351	// Dispatch
3352	//===----------------------------------------------------------------------===//
3353
3354	static bool genOMPCompositeDispatch(
3355	lower::AbstractConverter &converter, lower::SymMap &symTable,
3356	lower::StatementContext &stmtCtx, semantics::SemanticsContext &semaCtx,
3357	lower::pft::Evaluation &eval, mlir::Location loc,
3358	const ConstructQueue &queue, ConstructQueue::const_iterator item,
3359	mlir::Operation *&newOp) {
3360	using llvm::omp::Directive;
3361	using lower::omp::matchLeafSequence;
3362
3363	// TODO: Privatization for composite constructs is currently only done based
3364	// on the clauses for their last leaf construct, which may not always be
3365	// correct. Consider per-leaf privatization of composite constructs once
3366	// delayed privatization is supported by all participating ops.
3367	if (matchLeafSequence(item, queue, Directive::OMPD_distribute_parallel_do))
3368	newOp = genCompositeDistributeParallelDo(converter, symTable, stmtCtx,
3369	semaCtx, eval, loc, queue, item);
3370	else if (matchLeafSequence(item, queue,
3371	Directive::OMPD_distribute_parallel_do_simd))
3372	newOp = genCompositeDistributeParallelDoSimd(
3373	converter, symTable, stmtCtx, semaCtx, eval, loc, queue, item);
3374	else if (matchLeafSequence(item, queue, Directive::OMPD_distribute_simd))
3375	newOp = genCompositeDistributeSimd(converter, symTable, stmtCtx, semaCtx,
3376	eval, loc, queue, item);
3377	else if (matchLeafSequence(item, queue, Directive::OMPD_do_simd))
3378	newOp = genCompositeDoSimd(converter, symTable, stmtCtx, semaCtx, eval, loc,
3379	queue, item);
3380	else if (matchLeafSequence(item, queue, Directive::OMPD_taskloop_simd))
3381	newOp = genCompositeTaskloopSimd(converter, symTable, stmtCtx, semaCtx,
3382	eval, loc, queue, item);
3383	else
3384	return false;
3385
3386	return true;
3387	}
3388
3389	static void genOMPDispatch(lower::AbstractConverter &converter,
3390	lower::SymMap &symTable,
3391	semantics::SemanticsContext &semaCtx,
3392	lower::pft::Evaluation &eval, mlir::Location loc,
3393	const ConstructQueue &queue,
3394	ConstructQueue::const_iterator item) {
3395	assert(item != queue.end());
3396
3397	lower::StatementContext stmtCtx;
3398	mlir::Operation *newOp = nullptr;
3399
3400	// Generate cleanup code for the stmtCtx after newOp
3401	auto finalizeStmtCtx = [&]() {
3402	if (newOp) {
3403	fir::FirOpBuilder &builder = converter.getFirOpBuilder();
3404	fir::FirOpBuilder::InsertionGuard guard(builder);
3405	builder.setInsertionPointAfter(newOp);
3406	stmtCtx.finalizeAndPop();
3407	}
3408	};
3409
3410	bool loopLeaf = llvm::omp::getDirectiveAssociation(item->id) ==
3411	llvm::omp::Association::Loop;
3412	if (loopLeaf) {
3413	symTable.pushScope();
3414	if (genOMPCompositeDispatch(converter, symTable, stmtCtx, semaCtx, eval,
3415	loc, queue, item, newOp)) {
3416	symTable.popScope();
3417	finalizeStmtCtx();
3418	return;
3419	}
3420	}
3421
3422	switch (llvm::omp::Directive dir = item->id) {
3423	case llvm::omp::Directive::OMPD_barrier:
3424	newOp = genBarrierOp(converter, symTable, semaCtx, eval, loc, queue, item);
3425	break;
3426	case llvm::omp::Directive::OMPD_distribute:
3427	newOp = genStandaloneDistribute(converter, symTable, stmtCtx, semaCtx, eval,
3428	loc, queue, item);
3429	break;
3430	case llvm::omp::Directive::OMPD_do:
3431	newOp = genStandaloneDo(converter, symTable, stmtCtx, semaCtx, eval, loc,
3432	queue, item);
3433	break;
3434	case llvm::omp::Directive::OMPD_loop:
3435	newOp = genLoopOp(converter, symTable, semaCtx, eval, loc, queue, item);
3436	break;
3437	case llvm::omp::Directive::OMPD_masked:
3438	newOp = genMaskedOp(converter, symTable, stmtCtx, semaCtx, eval, loc, queue,
3439	item);
3440	break;
3441	case llvm::omp::Directive::OMPD_master:
3442	newOp = genMasterOp(converter, symTable, semaCtx, eval, loc, queue, item);
3443	break;
3444	case llvm::omp::Directive::OMPD_ordered:
3445	// Block-associated "ordered" construct.
3446	newOp = genOrderedRegionOp(converter, symTable, semaCtx, eval, loc, queue,
3447	item);
3448	break;
3449	case llvm::omp::Directive::OMPD_parallel:
3450	newOp = genStandaloneParallel(converter, symTable, stmtCtx, semaCtx, eval,
3451	loc, queue, item);
3452	break;
3453	case llvm::omp::Directive::OMPD_scan:
3454	newOp = genScanOp(converter, symTable, semaCtx, loc, queue, item);
3455	break;
3456	case llvm::omp::Directive::OMPD_section:
3457	llvm_unreachable("genOMPDispatch: OMPD_section");
3458	// Lowered in the enclosing genSectionsOp.
3459	break;
3460	case llvm::omp::Directive::OMPD_sections:
3461	// Called directly from genOMP([...], OpenMPSectionsConstruct) because it
3462	// has a different prototype.
3463	// This code path is still taken when iterating through the construct queue
3464	// in genBodyOfOp
3465	break;
3466	case llvm::omp::Directive::OMPD_simd:
3467	newOp =
3468	genStandaloneSimd(converter, symTable, semaCtx, eval, loc, queue, item);
3469	break;
3470	case llvm::omp::Directive::OMPD_scope:
3471	newOp = genScopeOp(converter, symTable, semaCtx, eval, loc, queue, item);
3472	break;
3473	case llvm::omp::Directive::OMPD_single:
3474	newOp = genSingleOp(converter, symTable, semaCtx, eval, loc, queue, item);
3475	break;
3476	case llvm::omp::Directive::OMPD_target:
3477	newOp = genTargetOp(converter, symTable, stmtCtx, semaCtx, eval, loc, queue,
3478	item);
3479	break;
3480	case llvm::omp::Directive::OMPD_target_data:
3481	newOp = genTargetDataOp(converter, symTable, stmtCtx, semaCtx, eval, loc,
3482	queue, item);
3483	break;
3484	case llvm::omp::Directive::OMPD_target_enter_data:
3485	newOp = genTargetEnterExitUpdateDataOp<mlir::omp::TargetEnterDataOp>(
3486	converter, symTable, stmtCtx, semaCtx, loc, queue, item);
3487	break;
3488	case llvm::omp::Directive::OMPD_target_exit_data:
3489	newOp = genTargetEnterExitUpdateDataOp<mlir::omp::TargetExitDataOp>(
3490	converter, symTable, stmtCtx, semaCtx, loc, queue, item);
3491	break;
3492	case llvm::omp::Directive::OMPD_target_update:
3493	newOp = genTargetEnterExitUpdateDataOp<mlir::omp::TargetUpdateOp>(
3494	converter, symTable, stmtCtx, semaCtx, loc, queue, item);
3495	break;
3496	case llvm::omp::Directive::OMPD_task:
3497	newOp = genTaskOp(converter, symTable, stmtCtx, semaCtx, eval, loc, queue,
3498	item);
3499	break;
3500	case llvm::omp::Directive::OMPD_taskgroup:
3501	newOp =
3502	genTaskgroupOp(converter, symTable, semaCtx, eval, loc, queue, item);
3503	break;
3504	case llvm::omp::Directive::OMPD_taskloop:
3505	newOp = genStandaloneTaskloop(converter, symTable, stmtCtx, semaCtx, eval,
3506	loc, queue, item);
3507	break;
3508	case llvm::omp::Directive::OMPD_taskwait:
3509	newOp = genTaskwaitOp(converter, symTable, semaCtx, eval, loc, queue, item);
3510	break;
3511	case llvm::omp::Directive::OMPD_taskyield:
3512	newOp =
3513	genTaskyieldOp(converter, symTable, semaCtx, eval, loc, queue, item);
3514	break;
3515	case llvm::omp::Directive::OMPD_teams:
3516	newOp = genTeamsOp(converter, symTable, stmtCtx, semaCtx, eval, loc, queue,
3517	item);
3518	break;
3519	case llvm::omp::Directive::OMPD_tile:
3520	case llvm::omp::Directive::OMPD_unroll: {
3521	unsigned version = semaCtx.langOptions().OpenMPVersion;
3522	TODO(loc, "Unhandled loop directive (" +
3523	llvm::omp::getOpenMPDirectiveName(dir, version) + ")");
3524	}
3525	// case llvm::omp::Directive::OMPD_workdistribute:
3526	case llvm::omp::Directive::OMPD_workshare:
3527	newOp = genWorkshareOp(converter, symTable, stmtCtx, semaCtx, eval, loc,
3528	queue, item);
3529	break;
3530	default:
3531	// Combined and composite constructs should have been split into a sequence
3532	// of leaf constructs when building the construct queue.
3533	assert(!llvm::omp::isLeafConstruct(dir) &&
3534	"Unexpected compound construct.");
3535	break;
3536	}
3537
3538	finalizeStmtCtx();
3539	if (loopLeaf)
3540	symTable.popScope();
3541	}
3542
3543	//===----------------------------------------------------------------------===//
3544	// OpenMPDeclarativeConstruct visitors
3545	//===----------------------------------------------------------------------===//
3546	static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
3547	semantics::SemanticsContext &semaCtx,
3548	lower::pft::Evaluation &eval,
3549	const parser::OpenMPUtilityConstruct &);
3550
3551	static void
3552	genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
3553	semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
3554	const parser::OpenMPDeclarativeAllocate &declarativeAllocate) {
3555	TODO(converter.getCurrentLocation(), "OpenMPDeclarativeAllocate");
3556	}
3557
3558	static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
3559	semantics::SemanticsContext &semaCtx,
3560	lower::pft::Evaluation &eval,
3561	const parser::OpenMPDeclarativeAssumes &assumesConstruct) {
3562	TODO(converter.getCurrentLocation(), "OpenMP ASSUMES declaration");
3563	}
3564
3565	static void
3566	genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
3567	semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
3568	const parser::OmpDeclareVariantDirective &declareVariantDirective) {
3569	TODO(converter.getCurrentLocation(), "OmpDeclareVariantDirective");
3570	}
3571
3572	static void genOMP(
3573	lower::AbstractConverter &converter, lower::SymMap &symTable,
3574	semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
3575	const parser::OpenMPDeclareReductionConstruct &declareReductionConstruct) {
3576	TODO(converter.getCurrentLocation(), "OpenMPDeclareReductionConstruct");
3577	}
3578
3579	static void
3580	genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
3581	semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
3582	const parser::OpenMPDeclareSimdConstruct &declareSimdConstruct) {
3583	TODO(converter.getCurrentLocation(), "OpenMPDeclareSimdConstruct");
3584	}
3585
3586	static void
3587	genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
3588	semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
3589	const parser::OpenMPDeclareMapperConstruct &declareMapperConstruct) {
3590	mlir::Location loc = converter.genLocation(declareMapperConstruct.source);
3591	fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
3592	lower::StatementContext stmtCtx;
3593	const auto &spec =
3594	std::get<parser::OmpMapperSpecifier>(declareMapperConstruct.t);
3595	const auto &mapperName{std::get<std::string>(spec.t)};
3596	const auto &varType{std::get<parser::TypeSpec>(spec.t)};
3597	const auto &varName{std::get<parser::Name>(spec.t)};
3598	assert(varType.declTypeSpec->category() ==
3599	semantics::DeclTypeSpec::Category::TypeDerived &&
3600	"Expected derived type");
3601
3602	std::string mapperNameStr = mapperName;
3603	if (auto *sym = converter.getCurrentScope().FindSymbol(mapperNameStr))
3604	mapperNameStr = converter.mangleName(mapperNameStr, sym->owner());
3605
3606	// Save current insertion point before moving to the module scope to create
3607	// the DeclareMapperOp
3608	mlir::OpBuilder::InsertionGuard guard(firOpBuilder);
3609
3610	firOpBuilder.setInsertionPointToStart(converter.getModuleOp().getBody());
3611	auto mlirType = converter.genType(varType.declTypeSpec->derivedTypeSpec());
3612	auto declMapperOp = firOpBuilder.create<mlir::omp::DeclareMapperOp>(
3613	loc, mapperNameStr, mlirType);
3614	auto &region = declMapperOp.getRegion();
3615	firOpBuilder.createBlock(&region);
3616	auto varVal = region.addArgument(firOpBuilder.getRefType(mlirType), loc);
3617	converter.bindSymbol(*varName.symbol, varVal);
3618
3619	// Populate the declareMapper region with the map information.
3620	mlir::omp::DeclareMapperInfoOperands clauseOps;
3621	const auto *clauseList{
3622	parser::Unwrap<parser::OmpClauseList>(declareMapperConstruct.t)};
3623	List<Clause> clauses = makeClauses(*clauseList, semaCtx);
3624	ClauseProcessor cp(converter, semaCtx, clauses);
3625	cp.processMap(loc, stmtCtx, clauseOps);
3626	firOpBuilder.create<mlir::omp::DeclareMapperInfoOp>(loc, clauseOps.mapVars);
3627	}
3628
3629	static void
3630	genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
3631	semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
3632	const parser::OpenMPDeclareTargetConstruct &declareTargetConstruct) {
3633	mlir::omp::DeclareTargetOperands clauseOps;
3634	llvm::SmallVector<DeclareTargetCapturePair> symbolAndClause;
3635	mlir::ModuleOp mod = converter.getFirOpBuilder().getModule();
3636	getDeclareTargetInfo(converter, semaCtx, eval, declareTargetConstruct,
3637	clauseOps, symbolAndClause);
3638
3639	for (const DeclareTargetCapturePair &symClause : symbolAndClause) {
3640	mlir::Operation *op = mod.lookupSymbol(
3641	converter.mangleName(std::get<const semantics::Symbol &>(symClause)));
3642
3643	// Some symbols are deferred until later in the module, these are handled
3644	// upon finalization of the module for OpenMP inside of Bridge, so we simply
3645	// skip for now.
3646	if (!op)
3647	continue;
3648
3649	markDeclareTarget(
3650	op, converter,
3651	std::get<mlir::omp::DeclareTargetCaptureClause>(symClause),
3652	clauseOps.deviceType);
3653	}
3654	}
3655
3656	static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
3657	semantics::SemanticsContext &semaCtx,
3658	lower::pft::Evaluation &eval,
3659	const parser::OpenMPRequiresConstruct &requiresConstruct) {
3660	// Requires directives are gathered and processed in semantics and
3661	// then combined in the lowering bridge before triggering codegen
3662	// just once. Hence, there is no need to lower each individual
3663	// occurrence here.
3664	}
3665
3666	static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
3667	semantics::SemanticsContext &semaCtx,
3668	lower::pft::Evaluation &eval,
3669	const parser::OpenMPThreadprivate &threadprivate) {
3670	// The directive is lowered when instantiating the variable to
3671	// support the case of threadprivate variable declared in module.
3672	}
3673
3674	static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
3675	semantics::SemanticsContext &semaCtx,
3676	lower::pft::Evaluation &eval,
3677	const parser::OmpMetadirectiveDirective &meta) {
3678	TODO(converter.getCurrentLocation(), "METADIRECTIVE");
3679	}
3680
3681	static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
3682	semantics::SemanticsContext &semaCtx,
3683	lower::pft::Evaluation &eval,
3684	const parser::OpenMPDeclarativeConstruct &ompDeclConstruct) {
3685	Fortran::common::visit(
3686	[&](auto &&s) { return genOMP(converter, symTable, semaCtx, eval, s); },
3687	ompDeclConstruct.u);
3688	}
3689
3690	//===----------------------------------------------------------------------===//
3691	// OpenMPStandaloneConstruct visitors
3692	//===----------------------------------------------------------------------===//
3693
3694	static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
3695	semantics::SemanticsContext &semaCtx,
3696	lower::pft::Evaluation &eval,
3697	const parser::OpenMPSimpleStandaloneConstruct &construct) {
3698	const auto &directive = std::get<parser::OmpDirectiveName>(construct.v.t);
3699	List<Clause> clauses = makeClauses(construct.v.Clauses(), semaCtx);
3700	mlir::Location currentLocation = converter.genLocation(directive.source);
3701
3702	ConstructQueue queue{
3703	buildConstructQueue(converter.getFirOpBuilder().getModule(), semaCtx,
3704	eval, directive.source, directive.v, clauses)};
3705	if (directive.v == llvm::omp::Directive::OMPD_ordered) {
3706	// Standalone "ordered" directive.
3707	genOrderedOp(converter, symTable, semaCtx, eval, currentLocation, queue,
3708	queue.begin());
3709	} else {
3710	// Dispatch handles the "block-associated" variant of "ordered".
3711	genOMPDispatch(converter, symTable, semaCtx, eval, currentLocation, queue,
3712	queue.begin());
3713	}
3714	}
3715
3716	static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
3717	semantics::SemanticsContext &semaCtx,
3718	lower::pft::Evaluation &eval,
3719	const parser::OpenMPFlushConstruct &construct) {
3720	const auto &argumentList = construct.v.Arguments();
3721	const auto &clauseList = construct.v.Clauses();
3722	ObjectList objects = makeObjects(argumentList, semaCtx);
3723	List<Clause> clauses =
3724	makeList(clauseList.v, [&](auto &&s) { return makeClause(s, semaCtx); });
3725	mlir::Location currentLocation = converter.genLocation(construct.source);
3726
3727	ConstructQueue queue{buildConstructQueue(
3728	converter.getFirOpBuilder().getModule(), semaCtx, eval, construct.source,
3729	llvm::omp::Directive::OMPD_flush, clauses)};
3730	genFlushOp(converter, symTable, semaCtx, eval, currentLocation, objects,
3731	queue, queue.begin());
3732	}
3733
3734	static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
3735	semantics::SemanticsContext &semaCtx,
3736	lower::pft::Evaluation &eval,
3737	const parser::OpenMPCancelConstruct &cancelConstruct) {
3738	List<Clause> clauses = makeList(cancelConstruct.v.Clauses().v, [&](auto &&s) {
3739	return makeClause(s, semaCtx);
3740	});
3741	mlir::Location loc = converter.genLocation(cancelConstruct.source);
3742
3743	ConstructQueue queue{buildConstructQueue(
3744	converter.getFirOpBuilder().getModule(), semaCtx, eval,
3745	cancelConstruct.source, llvm::omp::Directive::OMPD_cancel, clauses)};
3746	genCancelOp(converter, semaCtx, eval, loc, queue, queue.begin());
3747	}
3748
3749	static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
3750	semantics::SemanticsContext &semaCtx,
3751	lower::pft::Evaluation &eval,
3752	const parser::OpenMPCancellationPointConstruct
3753	&cancellationPointConstruct) {
3754	List<Clause> clauses =
3755	makeList(cancellationPointConstruct.v.Clauses().v,
3756	[&](auto &&s) { return makeClause(s, semaCtx); });
3757	mlir::Location loc = converter.genLocation(cancellationPointConstruct.source);
3758
3759	ConstructQueue queue{
3760	buildConstructQueue(converter.getFirOpBuilder().getModule(), semaCtx,
3761	eval, cancellationPointConstruct.source,
3762	llvm::omp::Directive::OMPD_cancel, clauses)};
3763	genCancellationPointOp(converter, semaCtx, eval, loc, queue, queue.begin());
3764	}
3765
3766	static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
3767	semantics::SemanticsContext &semaCtx,
3768	lower::pft::Evaluation &eval,
3769	const parser::OpenMPDepobjConstruct &construct) {
3770	// These values will be ignored until the construct itself is implemented,
3771	// but run them anyway for the sake of testing (via a Todo test).
3772	ObjectList objects = makeObjects(construct.v.Arguments(), semaCtx);
3773	assert(objects.size() == 1);
3774	List<Clause> clauses = makeClauses(construct.v.Clauses(), semaCtx);
3775	assert(clauses.size() == 1);
3776	(void)objects;
3777	(void)clauses;
3778
3779	TODO(converter.getCurrentLocation(), "OpenMPDepobjConstruct");
3780	}
3781
3782	static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
3783	semantics::SemanticsContext &semaCtx,
3784	lower::pft::Evaluation &eval,
3785	const parser::OpenMPInteropConstruct &interopConstruct) {
3786	TODO(converter.getCurrentLocation(), "OpenMPInteropConstruct");
3787	}
3788
3789	static void
3790	genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
3791	semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
3792	const parser::OpenMPStandaloneConstruct &standaloneConstruct) {
3793	Fortran::common::visit(
3794	[&](auto &&s) { return genOMP(converter, symTable, semaCtx, eval, s); },
3795	standaloneConstruct.u);
3796	}
3797
3798	static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
3799	semantics::SemanticsContext &semaCtx,
3800	lower::pft::Evaluation &eval,
3801	const parser::OpenMPAllocatorsConstruct &allocsConstruct) {
3802	TODO(converter.getCurrentLocation(), "OpenMPAllocatorsConstruct");
3803	}
3804
3805	//===----------------------------------------------------------------------===//
3806	// OpenMPConstruct visitors
3807	//===----------------------------------------------------------------------===//
3808
3809	[[maybe_unused]] static void
3810	dumpAtomicAnalysis(const parser::OpenMPAtomicConstruct::Analysis &analysis) {
3811	auto whatStr = [](int k) {
3812	std::string txt = "?";
3813	switch (k & parser::OpenMPAtomicConstruct::Analysis::Action) {
3814	case parser::OpenMPAtomicConstruct::Analysis::None:
3815	txt = "None";
3816	break;
3817	case parser::OpenMPAtomicConstruct::Analysis::Read:
3818	txt = "Read";
3819	break;
3820	case parser::OpenMPAtomicConstruct::Analysis::Write:
3821	txt = "Write";
3822	break;
3823	case parser::OpenMPAtomicConstruct::Analysis::Update:
3824	txt = "Update";
3825	break;
3826	}
3827	switch (k & parser::OpenMPAtomicConstruct::Analysis::Condition) {
3828	case parser::OpenMPAtomicConstruct::Analysis::IfTrue:
3829	txt += " | IfTrue";
3830	break;
3831	case parser::OpenMPAtomicConstruct::Analysis::IfFalse:
3832	txt += " | IfFalse";
3833	break;
3834	}
3835	return txt;
3836	};
3837
3838	auto exprStr = [&](const parser::TypedExpr &expr) {
3839	if (auto *maybe = expr.get()) {
3840	if (maybe->v)
3841	return maybe->v->AsFortran();
3842	}
3843	return "<null>"s;
3844	};
3845	auto assignStr = [&](const parser::AssignmentStmt::TypedAssignment &assign) {
3846	if (auto *maybe = assign.get(); maybe && maybe->v) {
3847	std::string str;
3848	llvm::raw_string_ostream os(str);
3849	maybe->v->AsFortran(os);
3850	return str;
3851	}
3852	return "<null>"s;
3853	};
3854
3855	const SomeExpr &atom = *analysis.atom.get()->v;
3856
3857	llvm::errs() << "Analysis {\n";
3858	llvm::errs() << " atom: " << atom.AsFortran() << "\n";
3859	llvm::errs() << " cond: " << exprStr(analysis.cond) << "\n";
3860	llvm::errs() << " op0 {\n";
3861	llvm::errs() << " what: " << whatStr(analysis.op0.what) << "\n";
3862	llvm::errs() << " assign: " << assignStr(analysis.op0.assign) << "\n";
3863	llvm::errs() << " }\n";
3864	llvm::errs() << " op1 {\n";
3865	llvm::errs() << " what: " << whatStr(analysis.op1.what) << "\n";
3866	llvm::errs() << " assign: " << assignStr(analysis.op1.assign) << "\n";
3867	llvm::errs() << " }\n";
3868	llvm::errs() << "}\n";
3869	}
3870
3871	static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
3872	semantics::SemanticsContext &semaCtx,
3873	lower::pft::Evaluation &eval,
3874	const parser::OpenMPAtomicConstruct &construct) {
3875	auto get = [](auto &&typedWrapper) -> decltype(&*typedWrapper.get()->v) {
3876	if (auto *maybe = typedWrapper.get(); maybe && maybe->v) {
3877	return &*maybe->v;
3878	} else {
3879	return nullptr;
3880	}
3881	};
3882
3883	fir::FirOpBuilder &builder = converter.getFirOpBuilder();
3884	auto &dirSpec = std::get<parser::OmpDirectiveSpecification>(construct.t);
3885	List<Clause> clauses = makeClauses(dirSpec.Clauses(), semaCtx);
3886	lower::StatementContext stmtCtx;
3887
3888	const parser::OpenMPAtomicConstruct::Analysis &analysis = construct.analysis;
3889	if (DumpAtomicAnalysis)
3890	dumpAtomicAnalysis(analysis);
3891
3892	const semantics::SomeExpr &atom = *get(analysis.atom);
3893	mlir::Location loc = converter.genLocation(construct.source);
3894	mlir::Value atomAddr =
3895	fir::getBase(converter.genExprAddr(atom, stmtCtx, &loc));
3896	mlir::IntegerAttr hint = getAtomicHint(converter, clauses);
3897	mlir::omp::ClauseMemoryOrderKindAttr memOrder =
3898	getAtomicMemoryOrder(converter, semaCtx, clauses);
3899
3900	if (auto *cond = get(analysis.cond)) {
3901	(void)cond;
3902	TODO(loc, "OpenMP ATOMIC COMPARE");
3903	} else {
3904	int action0 = analysis.op0.what & analysis.Action;
3905	int action1 = analysis.op1.what & analysis.Action;
3906	mlir::Operation *captureOp = nullptr;
3907	fir::FirOpBuilder::InsertPoint preAt = builder.saveInsertionPoint();
3908	fir::FirOpBuilder::InsertPoint atomicAt, postAt;
3909
3910	if (construct.IsCapture()) {
3911	// Capturing operation.
3912	assert(action0 != analysis.None && action1 != analysis.None &&
3913	"Expexcing two actions");
3914	(void)action0;
3915	(void)action1;
3916	captureOp =
3917	builder.create<mlir::omp::AtomicCaptureOp>(loc, hint, memOrder);
3918	// Set the non-atomic insertion point to before the atomic.capture.
3919	preAt = getInsertionPointBefore(captureOp);
3920
3921	mlir::Block *block = builder.createBlock(&captureOp->getRegion(0));
3922	builder.setInsertionPointToEnd(block);
3923	// Set the atomic insertion point to before the terminator inside
3924	// atomic.capture.
3925	mlir::Operation *term = builder.create<mlir::omp::TerminatorOp>(loc);
3926	atomicAt = getInsertionPointBefore(term);
3927	postAt = getInsertionPointAfter(captureOp);
3928	hint = nullptr;
3929	memOrder = nullptr;
3930	} else {
3931	// Non-capturing operation.
3932	assert(action0 != analysis.None && action1 == analysis.None &&
3933	"Expexcing single action");
3934	assert(!(analysis.op0.what & analysis.Condition));
3935	postAt = atomicAt = preAt;
3936	}
3937
3938	// The builder's insertion point needs to be specifically set before
3939	// each call to `genAtomicOperation`.
3940	mlir::Operation *firstOp = genAtomicOperation(
3941	converter, loc, stmtCtx, analysis.op0.what, atomAddr, atom,
3942	*get(analysis.op0.assign), hint, memOrder, preAt, atomicAt, postAt);
3943	assert(firstOp && "Should have created an atomic operation");
3944	atomicAt = getInsertionPointAfter(firstOp);
3945
3946	mlir::Operation *secondOp = nullptr;
3947	if (analysis.op1.what != analysis.None) {
3948	secondOp = genAtomicOperation(converter, loc, stmtCtx, analysis.op1.what,
3949	atomAddr, atom, *get(analysis.op1.assign),
3950	hint, memOrder, preAt, atomicAt, postAt);
3951	}
3952
3953	if (construct.IsCapture()) {
3954	// If this is a capture operation, the first/second ops will be inside
3955	// of it. Set the insertion point to past the capture op itself.
3956	builder.restoreInsertionPoint(postAt);
3957	} else {
3958	if (secondOp) {
3959	builder.setInsertionPointAfter(secondOp);
3960	} else {
3961	builder.setInsertionPointAfter(firstOp);
3962	}
3963	}
3964	}
3965	}
3966
3967	static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
3968	semantics::SemanticsContext &semaCtx,
3969	lower::pft::Evaluation &eval,
3970	const parser::OpenMPBlockConstruct &blockConstruct) {
3971	const auto &beginBlockDirective =
3972	std::get<parser::OmpBeginBlockDirective>(blockConstruct.t);
3973	const auto &endBlockDirective =
3974	std::get<parser::OmpEndBlockDirective>(blockConstruct.t);
3975	mlir::Location currentLocation =
3976	converter.genLocation(beginBlockDirective.source);
3977	const auto origDirective =
3978	std::get<parser::OmpBlockDirective>(beginBlockDirective.t).v;
3979	List<Clause> clauses = makeClauses(
3980	std::get<parser::OmpClauseList>(beginBlockDirective.t), semaCtx);
3981	clauses.append(makeClauses(
3982	std::get<parser::OmpClauseList>(endBlockDirective.t), semaCtx));
3983
3984	assert(llvm::omp::blockConstructSet.test(origDirective) &&
3985	"Expected block construct");
3986	(void)origDirective;
3987
3988	for (const Clause &clause : clauses) {
3989	mlir::Location clauseLocation = converter.genLocation(clause.source);
3990	if (!std::holds_alternative<clause::Affinity>(clause.u) &&
3991	!std::holds_alternative<clause::Allocate>(clause.u) &&
3992	!std::holds_alternative<clause::Copyin>(clause.u) &&
3993	!std::holds_alternative<clause::Copyprivate>(clause.u) &&
3994	!std::holds_alternative<clause::Default>(clause.u) &&
3995	!std::holds_alternative<clause::Defaultmap>(clause.u) &&
3996	!std::holds_alternative<clause::Depend>(clause.u) &&
3997	!std::holds_alternative<clause::Filter>(clause.u) &&
3998	!std::holds_alternative<clause::Final>(clause.u) &&
3999	!std::holds_alternative<clause::Firstprivate>(clause.u) &&
4000	!std::holds_alternative<clause::HasDeviceAddr>(clause.u) &&
4001	!std::holds_alternative<clause::If>(clause.u) &&
4002	!std::holds_alternative<clause::IsDevicePtr>(clause.u) &&
4003	!std::holds_alternative<clause::Map>(clause.u) &&
4004	!std::holds_alternative<clause::Nowait>(clause.u) &&
4005	!std::holds_alternative<clause::NumTeams>(clause.u) &&
4006	!std::holds_alternative<clause::NumThreads>(clause.u) &&
4007	!std::holds_alternative<clause::OmpxBare>(clause.u) &&
4008	!std::holds_alternative<clause::Priority>(clause.u) &&
4009	!std::holds_alternative<clause::Private>(clause.u) &&
4010	!std::holds_alternative<clause::ProcBind>(clause.u) &&
4011	!std::holds_alternative<clause::Reduction>(clause.u) &&
4012	!std::holds_alternative<clause::Shared>(clause.u) &&
4013	!std::holds_alternative<clause::Simd>(clause.u) &&
4014	!std::holds_alternative<clause::ThreadLimit>(clause.u) &&
4015	!std::holds_alternative<clause::Threads>(clause.u) &&
4016	!std::holds_alternative<clause::UseDeviceAddr>(clause.u) &&
4017	!std::holds_alternative<clause::UseDevicePtr>(clause.u) &&
4018	!std::holds_alternative<clause::InReduction>(clause.u) &&
4019	!std::holds_alternative<clause::Mergeable>(clause.u) &&
4020	!std::holds_alternative<clause::Untied>(clause.u) &&
4021	!std::holds_alternative<clause::TaskReduction>(clause.u) &&
4022	!std::holds_alternative<clause::Detach>(clause.u)) {
4023	std::string name =
4024	parser::ToUpperCaseLetters(llvm::omp::getOpenMPClauseName(clause.id));
4025	TODO(clauseLocation, name + " clause is not implemented yet");
4026	}
4027	}
4028
4029	llvm::omp::Directive directive =
4030	std::get<parser::OmpBlockDirective>(beginBlockDirective.t).v;
4031	const parser::CharBlock &source =
4032	std::get<parser::OmpBlockDirective>(beginBlockDirective.t).source;
4033	ConstructQueue queue{
4034	buildConstructQueue(converter.getFirOpBuilder().getModule(), semaCtx,
4035	eval, source, directive, clauses)};
4036	genOMPDispatch(converter, symTable, semaCtx, eval, currentLocation, queue,
4037	queue.begin());
4038	}
4039
4040	static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
4041	semantics::SemanticsContext &semaCtx,
4042	lower::pft::Evaluation &eval,
4043	const parser::OpenMPAssumeConstruct &assumeConstruct) {
4044	mlir::Location clauseLocation = converter.genLocation(assumeConstruct.source);
4045	TODO(clauseLocation, "OpenMP ASSUME construct");
4046	}
4047
4048	static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
4049	semantics::SemanticsContext &semaCtx,
4050	lower::pft::Evaluation &eval,
4051	const parser::OpenMPCriticalConstruct &criticalConstruct) {
4052	const auto &cd = std::get<parser::OmpCriticalDirective>(criticalConstruct.t);
4053	List<Clause> clauses =
4054	makeClauses(std::get<parser::OmpClauseList>(cd.t), semaCtx);
4055
4056	ConstructQueue queue{buildConstructQueue(
4057	converter.getFirOpBuilder().getModule(), semaCtx, eval, cd.source,
4058	llvm::omp::Directive::OMPD_critical, clauses)};
4059
4060	const auto &name = std::get<std::optional<parser::Name>>(cd.t);
4061	mlir::Location currentLocation = converter.getCurrentLocation();
4062	genCriticalOp(converter, symTable, semaCtx, eval, currentLocation, queue,
4063	queue.begin(), name);
4064	}
4065
4066	static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
4067	semantics::SemanticsContext &semaCtx,
4068	lower::pft::Evaluation &eval,
4069	const parser::OpenMPUtilityConstruct &) {
4070	TODO(converter.getCurrentLocation(), "OpenMPUtilityConstruct");
4071	}
4072
4073	static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
4074	semantics::SemanticsContext &semaCtx,
4075	lower::pft::Evaluation &eval,
4076	const parser::OpenMPDispatchConstruct &) {
4077	TODO(converter.getCurrentLocation(), "OpenMPDispatchConstruct");
4078	}
4079
4080	static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
4081	semantics::SemanticsContext &semaCtx,
4082	lower::pft::Evaluation &eval,
4083	const parser::OpenMPExecutableAllocate &execAllocConstruct) {
4084	TODO(converter.getCurrentLocation(), "OpenMPExecutableAllocate");
4085	}
4086
4087	static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
4088	semantics::SemanticsContext &semaCtx,
4089	lower::pft::Evaluation &eval,
4090	const parser::OpenMPLoopConstruct &loopConstruct) {
4091	const auto &beginLoopDirective =
4092	std::get<parser::OmpBeginLoopDirective>(loopConstruct.t);
4093	List<Clause> clauses = makeClauses(
4094	std::get<parser::OmpClauseList>(beginLoopDirective.t), semaCtx);
4095	if (auto &endLoopDirective =
4096	std::get<std::optional<parser::OmpEndLoopDirective>>(
4097	loopConstruct.t)) {
4098	clauses.append(makeClauses(
4099	std::get<parser::OmpClauseList>(endLoopDirective->t), semaCtx));
4100	}
4101
4102	mlir::Location currentLocation =
4103	converter.genLocation(beginLoopDirective.source);
4104
4105	llvm::omp::Directive directive =
4106	std::get<parser::OmpLoopDirective>(beginLoopDirective.t).v;
4107	const parser::CharBlock &source =
4108	std::get<parser::OmpLoopDirective>(beginLoopDirective.t).source;
4109	ConstructQueue queue{
4110	buildConstructQueue(converter.getFirOpBuilder().getModule(), semaCtx,
4111	eval, source, directive, clauses)};
4112	genOMPDispatch(converter, symTable, semaCtx, eval, currentLocation, queue,
4113	queue.begin());
4114	}
4115
4116	static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
4117	semantics::SemanticsContext &semaCtx,
4118	lower::pft::Evaluation &eval,
4119	const parser::OpenMPSectionConstruct &sectionConstruct) {
4120	// Do nothing here. SECTION is lowered inside of the lowering for Sections
4121	}
4122
4123	static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
4124	semantics::SemanticsContext &semaCtx,
4125	lower::pft::Evaluation &eval,
4126	const parser::OpenMPSectionsConstruct &sectionsConstruct) {
4127	const auto &beginSectionsDirective =
4128	std::get<parser::OmpBeginSectionsDirective>(sectionsConstruct.t);
4129	List<Clause> clauses = makeClauses(
4130	std::get<parser::OmpClauseList>(beginSectionsDirective.t), semaCtx);
4131	const auto &endSectionsDirective =
4132	std::get<parser::OmpEndSectionsDirective>(sectionsConstruct.t);
4133	const auto &sectionBlocks =
4134	std::get<parser::OmpSectionBlocks>(sectionsConstruct.t);
4135	clauses.append(makeClauses(
4136	std::get<parser::OmpClauseList>(endSectionsDirective.t), semaCtx));
4137	mlir::Location currentLocation = converter.getCurrentLocation();
4138
4139	llvm::omp::Directive directive =
4140	std::get<parser::OmpSectionsDirective>(beginSectionsDirective.t).v;
4141	const parser::CharBlock &source =
4142	std::get<parser::OmpSectionsDirective>(beginSectionsDirective.t).source;
4143	ConstructQueue queue{
4144	buildConstructQueue(converter.getFirOpBuilder().getModule(), semaCtx,
4145	eval, source, directive, clauses)};
4146	ConstructQueue::iterator next = queue.begin();
4147	// Generate constructs that come first e.g. Parallel
4148	while (next != queue.end() &&
4149	next->id != llvm::omp::Directive::OMPD_sections) {
4150	genOMPDispatch(converter, symTable, semaCtx, eval, currentLocation, queue,
4151	next);
4152	next = std::next(x: next);
4153	}
4154
4155	// call genSectionsOp directly (not via genOMPDispatch) so that we can add the
4156	// sectionBlocks argument
4157	assert(next != queue.end());
4158	assert(next->id == llvm::omp::Directive::OMPD_sections);
4159	genSectionsOp(converter, symTable, semaCtx, eval, currentLocation, queue,
4160	next, sectionBlocks);
4161	assert(std::next(next) == queue.end());
4162	}
4163
4164	static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
4165	semantics::SemanticsContext &semaCtx,
4166	lower::pft::Evaluation &eval,
4167	const parser::OpenMPConstruct &ompConstruct) {
4168	Fortran::common::visit(
4169	[&](auto &&s) { return genOMP(converter, symTable, semaCtx, eval, s); },
4170	ompConstruct.u);
4171	}
4172
4173	//===----------------------------------------------------------------------===//
4174	// Public functions
4175	//===----------------------------------------------------------------------===//
4176
4177	mlir::Operation *Fortran::lower::genOpenMPTerminator(fir::FirOpBuilder &builder,
4178	mlir::Operation *op,
4179	mlir::Location loc) {
4180	if (mlir::isa<mlir::omp::AtomicUpdateOp, mlir::omp::DeclareReductionOp,
4181	mlir::omp::LoopNestOp>(op))
4182	return builder.create<mlir::omp::YieldOp>(loc);
4183	return builder.create<mlir::omp::TerminatorOp>(loc);
4184	}
4185
4186	void Fortran::lower::genOpenMPConstruct(lower::AbstractConverter &converter,
4187	lower::SymMap &symTable,
4188	semantics::SemanticsContext &semaCtx,
4189	lower::pft::Evaluation &eval,
4190	const parser::OpenMPConstruct &omp) {
4191	lower::SymMapScope scope(symTable);
4192	genOMP(converter, symTable, semaCtx, eval, omp);
4193	}
4194
4195	void Fortran::lower::genOpenMPDeclarativeConstruct(
4196	lower::AbstractConverter &converter, lower::SymMap &symTable,
4197	semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
4198	const parser::OpenMPDeclarativeConstruct &omp) {
4199	genOMP(converter, symTable, semaCtx, eval, omp);
4200	genNestedEvaluations(converter, eval);
4201	}
4202
4203	void Fortran::lower::genOpenMPSymbolProperties(
4204	lower::AbstractConverter &converter, const lower::pft::Variable &var) {
4205	assert(var.hasSymbol() && "Expecting Symbol");
4206	const semantics::Symbol &sym = var.getSymbol();
4207
4208	if (sym.test(semantics::Symbol::Flag::OmpThreadprivate))
4209	lower::genThreadprivateOp(converter, var);
4210
4211	if (sym.test(semantics::Symbol::Flag::OmpDeclareTarget))
4212	lower::genDeclareTargetIntGlobal(converter, var);
4213	}
4214
4215	int64_t
4216	Fortran::lower::getCollapseValue(const parser::OmpClauseList &clauseList) {
4217	for (const parser::OmpClause &clause : clauseList.v) {
4218	if (const auto &collapseClause =
4219	std::get_if<parser::OmpClause::Collapse>(&clause.u)) {
4220	const auto *expr = semantics::GetExpr(collapseClause->v);
4221	return evaluate::ToInt64(*expr).value();
4222	}
4223	}
4224	return 1;
4225	}
4226
4227	void Fortran::lower::genThreadprivateOp(lower::AbstractConverter &converter,
4228	const lower::pft::Variable &var) {
4229	fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
4230	mlir::Location currentLocation = converter.getCurrentLocation();
4231
4232	const semantics::Symbol &sym = var.getSymbol();
4233	mlir::Value symThreadprivateValue;
4234	if (const semantics::Symbol *common =
4235	semantics::FindCommonBlockContaining(sym.GetUltimate())) {
4236	mlir::Value commonValue = converter.getSymbolAddress(*common);
4237	if (mlir::isa<mlir::omp::ThreadprivateOp>(commonValue.getDefiningOp())) {
4238	// Generate ThreadprivateOp for a common block instead of its members and
4239	// only do it once for a common block.
4240	return;
4241	}
4242	// Generate ThreadprivateOp and rebind the common block.
4243	mlir::Value commonThreadprivateValue =
4244	firOpBuilder.create<mlir::omp::ThreadprivateOp>(
4245	currentLocation, commonValue.getType(), commonValue);
4246	converter.bindSymbol(*common, commonThreadprivateValue);
4247	// Generate the threadprivate value for the common block member.
4248	symThreadprivateValue = genCommonBlockMember(converter, currentLocation,
4249	sym, commonThreadprivateValue);
4250	} else if (!var.isGlobal()) {
4251	// Non-global variable which can be in threadprivate directive must be one
4252	// variable in main program, and it has implicit SAVE attribute. Take it as
4253	// with SAVE attribute, so to create GlobalOp for it to simplify the
4254	// translation to LLVM IR.
4255	// Avoids performing multiple globalInitializations.
4256	fir::GlobalOp global;
4257	auto module = converter.getModuleOp();
4258	std::string globalName = converter.mangleName(sym);
4259	if (module.lookupSymbol<fir::GlobalOp>(globalName))
4260	global = module.lookupSymbol<fir::GlobalOp>(globalName);
4261	else
4262	global = globalInitialization(converter, firOpBuilder, sym, var,
4263	currentLocation);
4264
4265	mlir::Value symValue = firOpBuilder.create<fir::AddrOfOp>(
4266	currentLocation, global.resultType(), global.getSymbol());
4267	symThreadprivateValue = firOpBuilder.create<mlir::omp::ThreadprivateOp>(
4268	currentLocation, symValue.getType(), symValue);
4269	} else {
4270	mlir::Value symValue = converter.getSymbolAddress(sym);
4271
4272	// The symbol may be use-associated multiple times, and nothing needs to be
4273	// done after the original symbol is mapped to the threadprivatized value
4274	// for the first time. Use the threadprivatized value directly.
4275	mlir::Operation *op;
4276	if (auto declOp = symValue.getDefiningOp<hlfir::DeclareOp>())
4277	op = declOp.getMemref().getDefiningOp();
4278	else
4279	op = symValue.getDefiningOp();
4280	if (mlir::isa<mlir::omp::ThreadprivateOp>(op))
4281	return;
4282
4283	symThreadprivateValue = firOpBuilder.create<mlir::omp::ThreadprivateOp>(
4284	currentLocation, symValue.getType(), symValue);
4285	}
4286
4287	fir::ExtendedValue sexv = converter.getSymbolExtendedValue(sym);
4288	fir::ExtendedValue symThreadprivateExv =
4289	getExtendedValue(sexv, symThreadprivateValue);
4290	converter.bindSymbol(sym, symThreadprivateExv);
4291	}
4292
4293	// This function replicates threadprivate's behaviour of generating
4294	// an internal fir.GlobalOp for non-global variables in the main program
4295	// that have the implicit SAVE attribute, to simplifiy LLVM-IR and MLIR
4296	// generation.
4297	void Fortran::lower::genDeclareTargetIntGlobal(
4298	lower::AbstractConverter &converter, const lower::pft::Variable &var) {
4299	if (!var.isGlobal()) {
4300	// A non-global variable which can be in a declare target directive must
4301	// be a variable in the main program, and it has the implicit SAVE
4302	// attribute. We create a GlobalOp for it to simplify the translation to
4303	// LLVM IR.
4304	globalInitialization(converter, converter.getFirOpBuilder(),
4305	var.getSymbol(), var, converter.getCurrentLocation());
4306	}
4307	}
4308
4309	bool Fortran::lower::isOpenMPTargetConstruct(
4310	const parser::OpenMPConstruct &omp) {
4311	llvm::omp::Directive dir = llvm::omp::Directive::OMPD_unknown;
4312	if (const auto *block = std::get_if<parser::OpenMPBlockConstruct>(&omp.u)) {
4313	const auto &begin = std::get<parser::OmpBeginBlockDirective>(block->t);
4314	dir = std::get<parser::OmpBlockDirective>(begin.t).v;
4315	} else if (const auto *loop =
4316	std::get_if<parser::OpenMPLoopConstruct>(&omp.u)) {
4317	const auto &begin = std::get<parser::OmpBeginLoopDirective>(loop->t);
4318	dir = std::get<parser::OmpLoopDirective>(begin.t).v;
4319	}
4320	return llvm::omp::allTargetSet.test(dir);
4321	}
4322
4323	void Fortran::lower::gatherOpenMPDeferredDeclareTargets(
4324	lower::AbstractConverter &converter, semantics::SemanticsContext &semaCtx,
4325	lower::pft::Evaluation &eval,
4326	const parser::OpenMPDeclarativeConstruct &ompDecl,
4327	llvm::SmallVectorImpl<OMPDeferredDeclareTargetInfo>
4328	&deferredDeclareTarget) {
4329	Fortran::common::visit(
4330	common::visitors{
4331	[&](const parser::OpenMPDeclareTargetConstruct &ompReq) {
4332	collectDeferredDeclareTargets(converter, semaCtx, eval, ompReq,
4333	deferredDeclareTarget);
4334	},
4335	[&](const auto &) {},
4336	},
4337	ompDecl.u);
4338	}
4339
4340	bool Fortran::lower::isOpenMPDeviceDeclareTarget(
4341	lower::AbstractConverter &converter, semantics::SemanticsContext &semaCtx,
4342	lower::pft::Evaluation &eval,
4343	const parser::OpenMPDeclarativeConstruct &ompDecl) {
4344	return Fortran::common::visit(
4345	common::visitors{
4346	[&](const parser::OpenMPDeclareTargetConstruct &ompReq) {
4347	mlir::omp::DeclareTargetDeviceType targetType =
4348	getDeclareTargetFunctionDevice(converter, semaCtx, eval, ompReq)
4349	.value_or(mlir::omp::DeclareTargetDeviceType::host);
4350	return targetType != mlir::omp::DeclareTargetDeviceType::host;
4351	},
4352	[&](const auto &) { return false; },
4353	},
4354	ompDecl.u);
4355	}
4356
4357	// In certain cases such as subroutine or function interfaces which declare
4358	// but do not define or directly call the subroutine or function in the same
4359	// module, their lowering is delayed until after the declare target construct
4360	// itself is processed, so there symbol is not within the table.
4361	//
4362	// This function will also return true if we encounter any device declare
4363	// target cases, to satisfy checking if we require the requires attributes
4364	// on the module.
4365	bool Fortran::lower::markOpenMPDeferredDeclareTargetFunctions(
4366	mlir::Operation *mod,
4367	llvm::SmallVectorImpl<OMPDeferredDeclareTargetInfo> &deferredDeclareTargets,
4368	AbstractConverter &converter) {
4369	bool deviceCodeFound = false;
4370	auto modOp = llvm::cast<mlir::ModuleOp>(mod);
4371	for (auto declTar : deferredDeclareTargets) {
4372	mlir::Operation *op = modOp.lookupSymbol(converter.mangleName(declTar.sym));
4373
4374	// Due to interfaces being optionally emitted on usage in a module,
4375	// not finding an operation at this point cannot be a hard error, we
4376	// simply ignore it for now.
4377	// TODO: Add semantic checks for detecting cases where an erronous
4378	// (undefined) symbol has been supplied to a declare target clause
4379	if (!op)
4380	continue;
4381
4382	auto devType = declTar.declareTargetDeviceType;
4383	if (!deviceCodeFound && devType != mlir::omp::DeclareTargetDeviceType::host)
4384	deviceCodeFound = true;
4385
4386	markDeclareTarget(op, converter, declTar.declareTargetCaptureClause,
4387	devType);
4388	}
4389
4390	return deviceCodeFound;
4391	}
4392
4393	void Fortran::lower::genOpenMPRequires(mlir::Operation *mod,
4394	const semantics::Symbol *symbol) {
4395	using MlirRequires = mlir::omp::ClauseRequires;
4396	using SemaRequires = semantics::WithOmpDeclarative::RequiresFlag;
4397
4398	if (auto offloadMod =
4399	llvm::dyn_cast<mlir::omp::OffloadModuleInterface>(mod)) {
4400	semantics::WithOmpDeclarative::RequiresFlags semaFlags;
4401	if (symbol) {
4402	common::visit(
4403	[&](const auto &details) {
4404	if constexpr (std::is_base_of_v<semantics::WithOmpDeclarative,
4405	std::decay_t<decltype(details)>>) {
4406	if (details.has_ompRequires())
4407	semaFlags = *details.ompRequires();
4408	}
4409	},
4410	symbol->details());
4411	}
4412
4413	// Use pre-populated omp.requires module attribute if it was set, so that
4414	// the "-fopenmp-force-usm" compiler option is honored.
4415	MlirRequires mlirFlags = offloadMod.getRequires();
4416	if (semaFlags.test(SemaRequires::ReverseOffload))
4417	mlirFlags = mlirFlags | MlirRequires::reverse_offload;
4418	if (semaFlags.test(SemaRequires::UnifiedAddress))
4419	mlirFlags = mlirFlags | MlirRequires::unified_address;
4420	if (semaFlags.test(SemaRequires::UnifiedSharedMemory))
4421	mlirFlags = mlirFlags | MlirRequires::unified_shared_memory;
4422	if (semaFlags.test(SemaRequires::DynamicAllocators))
4423	mlirFlags = mlirFlags | MlirRequires::dynamic_allocators;
4424
4425	offloadMod.setRequires(mlirFlags);
4426	}
4427	}
4428