1//===-- OpenMP.cpp -- Open MP directive lowering --------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Coding style: https://mlir.llvm.org/getting_started/DeveloperGuide/
10//
11//===----------------------------------------------------------------------===//
12
13#include "flang/Lower/OpenMP.h"
14
15#include "Atomic.h"
16#include "ClauseProcessor.h"
17#include "DataSharingProcessor.h"
18#include "Decomposer.h"
19#include "Utils.h"
20#include "flang/Common/idioms.h"
21#include "flang/Lower/Bridge.h"
22#include "flang/Lower/ConvertExpr.h"
23#include "flang/Lower/ConvertVariable.h"
24#include "flang/Lower/DirectivesCommon.h"
25#include "flang/Lower/OpenMP/Clauses.h"
26#include "flang/Lower/StatementContext.h"
27#include "flang/Lower/SymbolMap.h"
28#include "flang/Optimizer/Builder/BoxValue.h"
29#include "flang/Optimizer/Builder/FIRBuilder.h"
30#include "flang/Optimizer/Builder/Todo.h"
31#include "flang/Optimizer/Dialect/FIRType.h"
32#include "flang/Optimizer/HLFIR/HLFIROps.h"
33#include "flang/Parser/characters.h"
34#include "flang/Parser/parse-tree.h"
35#include "flang/Semantics/openmp-directive-sets.h"
36#include "flang/Semantics/tools.h"
37#include "flang/Support/Flags.h"
38#include "flang/Support/OpenMP-utils.h"
39#include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h"
40#include "mlir/Dialect/OpenMP/OpenMPDialect.h"
41#include "mlir/Support/StateStack.h"
42#include "mlir/Transforms/RegionUtils.h"
43#include "llvm/ADT/STLExtras.h"
44#include "llvm/Frontend/OpenMP/OMPConstants.h"
45
46using namespace Fortran::lower::omp;
47using namespace Fortran::common::openmp;
48
49//===----------------------------------------------------------------------===//
50// Code generation helper functions
51//===----------------------------------------------------------------------===//
52
53static void genOMPDispatch(lower::AbstractConverter &converter,
54 lower::SymMap &symTable,
55 semantics::SemanticsContext &semaCtx,
56 lower::pft::Evaluation &eval, mlir::Location loc,
57 const ConstructQueue &queue,
58 ConstructQueue::const_iterator item);
59
60static void processHostEvalClauses(lower::AbstractConverter &converter,
61 semantics::SemanticsContext &semaCtx,
62 lower::StatementContext &stmtCtx,
63 lower::pft::Evaluation &eval,
64 mlir::Location loc);
65
66namespace {
67/// Structure holding information that is needed to pass host-evaluated
68/// information to later lowering stages.
69class HostEvalInfo {
70public:
71 // Allow this function access to private members in order to initialize them.
72 friend void ::processHostEvalClauses(lower::AbstractConverter &,
73 semantics::SemanticsContext &,
74 lower::StatementContext &,
75 lower::pft::Evaluation &,
76 mlir::Location);
77
78 /// Fill \c vars with values stored in \c ops.
79 ///
80 /// The order in which values are stored matches the one expected by \see
81 /// bindOperands().
82 void collectValues(llvm::SmallVectorImpl<mlir::Value> &vars) const {
83 vars.append(ops.loopLowerBounds);
84 vars.append(ops.loopUpperBounds);
85 vars.append(ops.loopSteps);
86
87 if (ops.numTeamsLower)
88 vars.push_back(ops.numTeamsLower);
89
90 if (ops.numTeamsUpper)
91 vars.push_back(ops.numTeamsUpper);
92
93 if (ops.numThreads)
94 vars.push_back(ops.numThreads);
95
96 if (ops.threadLimit)
97 vars.push_back(ops.threadLimit);
98 }
99
100 /// Update \c ops, replacing all values with the corresponding block argument
101 /// in \c args.
102 ///
103 /// The order in which values are stored in \c args is the same as the one
104 /// used by \see collectValues().
105 void bindOperands(llvm::ArrayRef<mlir::BlockArgument> args) {
106 assert(args.size() ==
107 ops.loopLowerBounds.size() + ops.loopUpperBounds.size() +
108 ops.loopSteps.size() + (ops.numTeamsLower ? 1 : 0) +
109 (ops.numTeamsUpper ? 1 : 0) + (ops.numThreads ? 1 : 0) +
110 (ops.threadLimit ? 1 : 0) &&
111 "invalid block argument list");
112 int argIndex = 0;
113 for (size_t i = 0; i < ops.loopLowerBounds.size(); ++i)
114 ops.loopLowerBounds[i] = args[argIndex++];
115
116 for (size_t i = 0; i < ops.loopUpperBounds.size(); ++i)
117 ops.loopUpperBounds[i] = args[argIndex++];
118
119 for (size_t i = 0; i < ops.loopSteps.size(); ++i)
120 ops.loopSteps[i] = args[argIndex++];
121
122 if (ops.numTeamsLower)
123 ops.numTeamsLower = args[argIndex++];
124
125 if (ops.numTeamsUpper)
126 ops.numTeamsUpper = args[argIndex++];
127
128 if (ops.numThreads)
129 ops.numThreads = args[argIndex++];
130
131 if (ops.threadLimit)
132 ops.threadLimit = args[argIndex++];
133 }
134
135 /// Update \p clauseOps and \p ivOut with the corresponding host-evaluated
136 /// values and Fortran symbols, respectively, if they have already been
137 /// initialized but not yet applied.
138 ///
139 /// \returns whether an update was performed. If not, these clauses were not
140 /// evaluated in the host device.
141 bool apply(mlir::omp::LoopNestOperands &clauseOps,
142 llvm::SmallVectorImpl<const semantics::Symbol *> &ivOut) {
143 if (iv.empty() || loopNestApplied) {
144 loopNestApplied = true;
145 return false;
146 }
147
148 loopNestApplied = true;
149 clauseOps.loopLowerBounds = ops.loopLowerBounds;
150 clauseOps.loopUpperBounds = ops.loopUpperBounds;
151 clauseOps.loopSteps = ops.loopSteps;
152 ivOut.append(iv);
153 return true;
154 }
155
156 /// Update \p clauseOps with the corresponding host-evaluated values if they
157 /// have already been initialized but not yet applied.
158 ///
159 /// \returns whether an update was performed. If not, these clauses were not
160 /// evaluated in the host device.
161 bool apply(mlir::omp::ParallelOperands &clauseOps) {
162 if (!ops.numThreads || parallelApplied) {
163 parallelApplied = true;
164 return false;
165 }
166
167 parallelApplied = true;
168 clauseOps.numThreads = ops.numThreads;
169 return true;
170 }
171
172 /// Update \p clauseOps with the corresponding host-evaluated values if they
173 /// have already been initialized.
174 ///
175 /// \returns whether an update was performed. If not, these clauses were not
176 /// evaluated in the host device.
177 bool apply(mlir::omp::TeamsOperands &clauseOps) {
178 if (!ops.numTeamsLower && !ops.numTeamsUpper && !ops.threadLimit)
179 return false;
180
181 clauseOps.numTeamsLower = ops.numTeamsLower;
182 clauseOps.numTeamsUpper = ops.numTeamsUpper;
183 clauseOps.threadLimit = ops.threadLimit;
184 return true;
185 }
186
187private:
188 mlir::omp::HostEvaluatedOperands ops;
189 llvm::SmallVector<const semantics::Symbol *> iv;
190 bool loopNestApplied = false, parallelApplied = false;
191};
192} // namespace
193
194/// Stack of \see HostEvalInfo to represent the current nest of \c omp.target
195/// operations being created.
196///
197/// The current implementation prevents nested 'target' regions from breaking
198/// the handling of the outer region by keeping a stack of information
199/// structures, but it will probably still require some further work to support
200/// reverse offloading.
201class HostEvalInfoStackFrame
202 : public mlir::StateStackFrameBase<HostEvalInfoStackFrame> {
203public:
204 MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(HostEvalInfoStackFrame)
205
206 HostEvalInfo info;
207};
208
209static HostEvalInfo *
210getHostEvalInfoStackTop(lower::AbstractConverter &converter) {
211 HostEvalInfoStackFrame *frame =
212 converter.getStateStack().getStackTop<HostEvalInfoStackFrame>();
213 return frame ? &frame->info : nullptr;
214}
215
216/// Stack frame for storing the OpenMPSectionsConstruct currently being
217/// processed so that it can be referred to when lowering the construct.
218class SectionsConstructStackFrame
219 : public mlir::StateStackFrameBase<SectionsConstructStackFrame> {
220public:
221 MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(SectionsConstructStackFrame)
222
223 explicit SectionsConstructStackFrame(
224 const parser::OpenMPSectionsConstruct &sectionsConstruct)
225 : sectionsConstruct{sectionsConstruct} {}
226
227 const parser::OpenMPSectionsConstruct &sectionsConstruct;
228};
229
230static const parser::OpenMPSectionsConstruct *
231getSectionsConstructStackTop(lower::AbstractConverter &converter) {
232 SectionsConstructStackFrame *frame =
233 converter.getStateStack().getStackTop<SectionsConstructStackFrame>();
234 return frame ? &frame->sectionsConstruct : nullptr;
235}
236
237/// Bind symbols to their corresponding entry block arguments.
238///
239/// The binding will be performed inside of the current block, which does not
240/// necessarily have to be part of the operation for which the binding is done.
241/// However, block arguments must be accessible. This enables controlling the
242/// insertion point of any new MLIR operations related to the binding of
243/// arguments of a loop wrapper operation.
244///
245/// \param [in] converter - PFT to MLIR conversion interface.
246/// \param [in] op - owner operation of the block arguments to bind.
247/// \param [in] args - entry block arguments information for the given
248/// operation.
249static void bindEntryBlockArgs(lower::AbstractConverter &converter,
250 mlir::omp::BlockArgOpenMPOpInterface op,
251 const EntryBlockArgs &args) {
252 assert(op != nullptr && "invalid block argument-defining operation");
253 assert(args.isValid() && "invalid args");
254 fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
255
256 auto bindSingleMapLike = [&converter](const semantics::Symbol &sym,
257 const mlir::BlockArgument &arg) {
258 fir::ExtendedValue extVal = converter.getSymbolExtendedValue(sym);
259 auto refType = mlir::dyn_cast<fir::ReferenceType>(arg.getType());
260 if (refType && fir::isa_builtin_cptr_type(refType.getElementType())) {
261 converter.bindSymbol(sym, arg);
262 } else {
263 extVal.match(
264 [&](const fir::BoxValue &v) {
265 converter.bindSymbol(sym, fir::BoxValue(arg, v.getLBounds(),
266 v.getExplicitParameters(),
267 v.getExplicitExtents()));
268 },
269 [&](const fir::MutableBoxValue &v) {
270 converter.bindSymbol(
271 sym, fir::MutableBoxValue(arg, v.getLBounds(),
272 v.getMutableProperties()));
273 },
274 [&](const fir::ArrayBoxValue &v) {
275 converter.bindSymbol(sym, fir::ArrayBoxValue(arg, v.getExtents(),
276 v.getLBounds(),
277 v.getSourceBox()));
278 },
279 [&](const fir::CharArrayBoxValue &v) {
280 converter.bindSymbol(sym, fir::CharArrayBoxValue(arg, v.getLen(),
281 v.getExtents(),
282 v.getLBounds()));
283 },
284 [&](const fir::CharBoxValue &v) {
285 converter.bindSymbol(sym, fir::CharBoxValue(arg, v.getLen()));
286 },
287 [&](const fir::UnboxedValue &v) { converter.bindSymbol(sym, arg); },
288 [&](const auto &) {
289 TODO(converter.getCurrentLocation(),
290 "target map clause operand unsupported type");
291 });
292 }
293 };
294
295 auto bindMapLike =
296 [&bindSingleMapLike](llvm::ArrayRef<const semantics::Symbol *> syms,
297 llvm::ArrayRef<mlir::BlockArgument> args) {
298 // Structure component symbols don't have bindings, and can only be
299 // explicitly mapped individually. If a member is captured implicitly
300 // we map the entirety of the derived type when we find its symbol.
301 llvm::SmallVector<const semantics::Symbol *> processedSyms;
302 llvm::copy_if(syms, std::back_inserter(processedSyms),
303 [](auto *sym) { return !sym->owner().IsDerivedType(); });
304
305 for (auto [sym, arg] : llvm::zip_equal(processedSyms, args))
306 bindSingleMapLike(*sym, arg);
307 };
308
309 auto bindPrivateLike = [&converter, &firOpBuilder](
310 llvm::ArrayRef<const semantics::Symbol *> syms,
311 llvm::ArrayRef<mlir::Value> vars,
312 llvm::ArrayRef<mlir::BlockArgument> args) {
313 llvm::SmallVector<const semantics::Symbol *> processedSyms;
314 for (auto *sym : syms) {
315 if (const auto *commonDet =
316 sym->detailsIf<semantics::CommonBlockDetails>()) {
317 llvm::transform(commonDet->objects(), std::back_inserter(processedSyms),
318 [&](const auto &mem) { return &*mem; });
319 } else {
320 processedSyms.push_back(sym);
321 }
322 }
323
324 for (auto [sym, var, arg] : llvm::zip_equal(processedSyms, vars, args))
325 converter.bindSymbol(
326 *sym,
327 hlfir::translateToExtendedValue(
328 var.getLoc(), firOpBuilder, hlfir::Entity{arg},
329 /*contiguousHint=*/
330 evaluate::IsSimplyContiguous(*sym, converter.getFoldingContext()))
331 .first);
332 };
333
334 // Process in clause name alphabetical order to match block arguments order.
335 // Do not bind host_eval variables because they cannot be used inside of the
336 // corresponding region, except for very specific cases handled separately.
337 bindMapLike(args.hasDeviceAddr.syms, op.getHasDeviceAddrBlockArgs());
338 bindPrivateLike(args.inReduction.syms, args.inReduction.vars,
339 op.getInReductionBlockArgs());
340 bindMapLike(args.map.syms, op.getMapBlockArgs());
341 bindPrivateLike(args.priv.syms, args.priv.vars, op.getPrivateBlockArgs());
342 bindPrivateLike(args.reduction.syms, args.reduction.vars,
343 op.getReductionBlockArgs());
344 bindPrivateLike(args.taskReduction.syms, args.taskReduction.vars,
345 op.getTaskReductionBlockArgs());
346 bindMapLike(args.useDeviceAddr.syms, op.getUseDeviceAddrBlockArgs());
347 bindMapLike(args.useDevicePtr.syms, op.getUseDevicePtrBlockArgs());
348}
349
350/// Get the list of base values that the specified map-like variables point to.
351///
352/// This function must be kept in sync with changes to the `createMapInfoOp`
353/// utility function, since it must take into account the potential introduction
354/// of levels of indirection (i.e. intermediate ops).
355///
356/// \param [in] vars - list of values passed to map-like clauses, returned
357/// by an `omp.map.info` operation.
358/// \param [out] baseOps - populated with the `var_ptr` values of the
359/// corresponding defining operations.
360static void
361extractMappedBaseValues(llvm::ArrayRef<mlir::Value> vars,
362 llvm::SmallVectorImpl<mlir::Value> &baseOps) {
363 llvm::transform(vars, std::back_inserter(baseOps), [](mlir::Value map) {
364 auto mapInfo = map.getDefiningOp<mlir::omp::MapInfoOp>();
365 assert(mapInfo && "expected all map vars to be defined by omp.map.info");
366
367 mlir::Value varPtr = mapInfo.getVarPtr();
368 if (auto boxAddr = varPtr.getDefiningOp<fir::BoxAddrOp>())
369 return boxAddr.getVal();
370
371 return varPtr;
372 });
373}
374
375/// Get the directive enumeration value corresponding to the given OpenMP
376/// construct PFT node.
377llvm::omp::Directive
378extractOmpDirective(const parser::OpenMPConstruct &ompConstruct) {
379 return common::visit(
380 common::visitors{
381 [](const parser::OpenMPAllocatorsConstruct &c) {
382 return llvm::omp::OMPD_allocators;
383 },
384 [](const parser::OpenMPAssumeConstruct &c) {
385 return llvm::omp::OMPD_assume;
386 },
387 [](const parser::OpenMPAtomicConstruct &c) {
388 return llvm::omp::OMPD_atomic;
389 },
390 [](const parser::OpenMPBlockConstruct &c) {
391 return std::get<parser::OmpBlockDirective>(
392 std::get<parser::OmpBeginBlockDirective>(c.t).t)
393 .v;
394 },
395 [](const parser::OpenMPCriticalConstruct &c) {
396 return llvm::omp::OMPD_critical;
397 },
398 [](const parser::OpenMPDeclarativeAllocate &c) {
399 return llvm::omp::OMPD_allocate;
400 },
401 [](const parser::OpenMPDispatchConstruct &c) {
402 return llvm::omp::OMPD_dispatch;
403 },
404 [](const parser::OpenMPExecutableAllocate &c) {
405 return llvm::omp::OMPD_allocate;
406 },
407 [](const parser::OpenMPLoopConstruct &c) {
408 return std::get<parser::OmpLoopDirective>(
409 std::get<parser::OmpBeginLoopDirective>(c.t).t)
410 .v;
411 },
412 [](const parser::OpenMPSectionConstruct &c) {
413 return llvm::omp::OMPD_section;
414 },
415 [](const parser::OpenMPSectionsConstruct &c) {
416 return std::get<parser::OmpSectionsDirective>(
417 std::get<parser::OmpBeginSectionsDirective>(c.t).t)
418 .v;
419 },
420 [](const parser::OpenMPStandaloneConstruct &c) {
421 return common::visit(
422 common::visitors{
423 [](const parser::OpenMPSimpleStandaloneConstruct &c) {
424 return c.v.DirId();
425 },
426 [](const parser::OpenMPFlushConstruct &c) {
427 return llvm::omp::OMPD_flush;
428 },
429 [](const parser::OpenMPCancelConstruct &c) {
430 return llvm::omp::OMPD_cancel;
431 },
432 [](const parser::OpenMPCancellationPointConstruct &c) {
433 return llvm::omp::OMPD_cancellation_point;
434 },
435 [](const parser::OmpMetadirectiveDirective &c) {
436 return llvm::omp::OMPD_metadirective;
437 },
438 [](const parser::OpenMPDepobjConstruct &c) {
439 return llvm::omp::OMPD_depobj;
440 },
441 [](const parser::OpenMPInteropConstruct &c) {
442 return llvm::omp::OMPD_interop;
443 }},
444 c.u);
445 },
446 [](const parser::OpenMPUtilityConstruct &c) {
447 return common::visit(
448 common::visitors{[](const parser::OmpErrorDirective &c) {
449 return llvm::omp::OMPD_error;
450 },
451 [](const parser::OmpNothingDirective &c) {
452 return llvm::omp::OMPD_nothing;
453 }},
454 c.u);
455 }},
456 ompConstruct.u);
457}
458
459/// Populate the global \see hostEvalInfo after processing clauses for the given
460/// \p eval OpenMP target construct, or nested constructs, if these must be
461/// evaluated outside of the target region per the spec.
462///
463/// In particular, this will ensure that in 'target teams' and equivalent nested
464/// constructs, the \c thread_limit and \c num_teams clauses will be evaluated
465/// in the host. Additionally, loop bounds, steps and the \c num_threads clause
466/// will also be evaluated in the host if a target SPMD construct is detected
467/// (i.e. 'target teams distribute parallel do [simd]' or equivalent nesting).
468///
469/// The result, stored as a global, is intended to be used to populate the \c
470/// host_eval operands of the associated \c omp.target operation, and also to be
471/// checked and used by later lowering steps to populate the corresponding
472/// operands of the \c omp.teams, \c omp.parallel or \c omp.loop_nest
473/// operations.
474static void processHostEvalClauses(lower::AbstractConverter &converter,
475 semantics::SemanticsContext &semaCtx,
476 lower::StatementContext &stmtCtx,
477 lower::pft::Evaluation &eval,
478 mlir::Location loc) {
479 // Obtain the list of clauses of the given OpenMP block or loop construct
480 // evaluation. Other evaluations passed to this lambda keep `clauses`
481 // unchanged.
482 auto extractClauses = [&semaCtx](lower::pft::Evaluation &eval,
483 List<Clause> &clauses) {
484 const auto *ompEval = eval.getIf<parser::OpenMPConstruct>();
485 if (!ompEval)
486 return;
487
488 const parser::OmpClauseList *beginClauseList = nullptr;
489 const parser::OmpClauseList *endClauseList = nullptr;
490 common::visit(
491 common::visitors{
492 [&](const parser::OpenMPBlockConstruct &ompConstruct) {
493 const auto &beginDirective =
494 std::get<parser::OmpBeginBlockDirective>(ompConstruct.t);
495 beginClauseList =
496 &std::get<parser::OmpClauseList>(beginDirective.t);
497 endClauseList = &std::get<parser::OmpClauseList>(
498 std::get<parser::OmpEndBlockDirective>(ompConstruct.t).t);
499 },
500 [&](const parser::OpenMPLoopConstruct &ompConstruct) {
501 const auto &beginDirective =
502 std::get<parser::OmpBeginLoopDirective>(ompConstruct.t);
503 beginClauseList =
504 &std::get<parser::OmpClauseList>(beginDirective.t);
505
506 if (auto &endDirective =
507 std::get<std::optional<parser::OmpEndLoopDirective>>(
508 ompConstruct.t))
509 endClauseList =
510 &std::get<parser::OmpClauseList>(endDirective->t);
511 },
512 [&](const auto &) {}},
513 ompEval->u);
514
515 assert(beginClauseList && "expected begin directive");
516 clauses.append(makeClauses(*beginClauseList, semaCtx));
517
518 if (endClauseList)
519 clauses.append(makeClauses(*endClauseList, semaCtx));
520 };
521
522 // Return the directive that is immediately nested inside of the given
523 // `parent` evaluation, if it is its only non-end-statement nested evaluation
524 // and it represents an OpenMP construct.
525 auto extractOnlyOmpNestedDir = [](lower::pft::Evaluation &parent)
526 -> std::optional<llvm::omp::Directive> {
527 if (!parent.hasNestedEvaluations())
528 return std::nullopt;
529
530 llvm::omp::Directive dir;
531 auto &nested = parent.getFirstNestedEvaluation();
532 if (const auto *ompEval = nested.getIf<parser::OpenMPConstruct>())
533 dir = extractOmpDirective(*ompEval);
534 else
535 return std::nullopt;
536
537 for (auto &sibling : parent.getNestedEvaluations())
538 if (&sibling != &nested && !sibling.isEndStmt())
539 return std::nullopt;
540
541 return dir;
542 };
543
544 // Process the given evaluation assuming it's part of a 'target' construct or
545 // captured by one, and store results in the global `hostEvalInfo`.
546 std::function<void(lower::pft::Evaluation &, const List<Clause> &)>
547 processEval;
548 processEval = [&](lower::pft::Evaluation &eval, const List<Clause> &clauses) {
549 using namespace llvm::omp;
550 ClauseProcessor cp(converter, semaCtx, clauses);
551
552 // Call `processEval` recursively with the immediately nested evaluation and
553 // its corresponding clauses if there is a single nested evaluation
554 // representing an OpenMP directive that passes the given test.
555 auto processSingleNestedIf = [&](llvm::function_ref<bool(Directive)> test) {
556 std::optional<Directive> nestedDir = extractOnlyOmpNestedDir(eval);
557 if (!nestedDir || !test(*nestedDir))
558 return;
559
560 lower::pft::Evaluation &nestedEval = eval.getFirstNestedEvaluation();
561 List<lower::omp::Clause> nestedClauses;
562 extractClauses(nestedEval, nestedClauses);
563 processEval(nestedEval, nestedClauses);
564 };
565
566 const auto *ompEval = eval.getIf<parser::OpenMPConstruct>();
567 if (!ompEval)
568 return;
569
570 HostEvalInfo *hostInfo = getHostEvalInfoStackTop(converter);
571 assert(hostInfo && "expected HOST_EVAL info structure");
572
573 switch (extractOmpDirective(*ompEval)) {
574 case OMPD_teams_distribute_parallel_do:
575 case OMPD_teams_distribute_parallel_do_simd:
576 cp.processThreadLimit(stmtCtx, hostInfo->ops);
577 [[fallthrough]];
578 case OMPD_target_teams_distribute_parallel_do:
579 case OMPD_target_teams_distribute_parallel_do_simd:
580 cp.processNumTeams(stmtCtx, hostInfo->ops);
581 [[fallthrough]];
582 case OMPD_distribute_parallel_do:
583 case OMPD_distribute_parallel_do_simd:
584 cp.processNumThreads(stmtCtx, hostInfo->ops);
585 [[fallthrough]];
586 case OMPD_distribute:
587 case OMPD_distribute_simd:
588 cp.processCollapse(loc, eval, hostInfo->ops, hostInfo->iv);
589 break;
590
591 case OMPD_teams:
592 cp.processThreadLimit(stmtCtx, hostInfo->ops);
593 [[fallthrough]];
594 case OMPD_target_teams:
595 cp.processNumTeams(stmtCtx, hostInfo->ops);
596 processSingleNestedIf([](Directive nestedDir) {
597 return topDistributeSet.test(nestedDir) || topLoopSet.test(nestedDir);
598 });
599 break;
600
601 case OMPD_teams_distribute:
602 case OMPD_teams_distribute_simd:
603 cp.processThreadLimit(stmtCtx, hostInfo->ops);
604 [[fallthrough]];
605 case OMPD_target_teams_distribute:
606 case OMPD_target_teams_distribute_simd:
607 cp.processCollapse(loc, eval, hostInfo->ops, hostInfo->iv);
608 cp.processNumTeams(stmtCtx, hostInfo->ops);
609 break;
610
611 case OMPD_teams_loop:
612 cp.processThreadLimit(stmtCtx, hostInfo->ops);
613 [[fallthrough]];
614 case OMPD_target_teams_loop:
615 cp.processNumTeams(stmtCtx, hostInfo->ops);
616 [[fallthrough]];
617 case OMPD_loop:
618 cp.processCollapse(loc, eval, hostInfo->ops, hostInfo->iv);
619 break;
620
621 // Standalone 'target' case.
622 case OMPD_target: {
623 processSingleNestedIf(
624 [](Directive nestedDir) { return topTeamsSet.test(nestedDir); });
625 break;
626 }
627 default:
628 break;
629 }
630 };
631
632 const auto *ompEval = eval.getIf<parser::OpenMPConstruct>();
633 assert(ompEval &&
634 llvm::omp::allTargetSet.test(extractOmpDirective(*ompEval)) &&
635 "expected TARGET construct evaluation");
636 (void)ompEval;
637
638 // Use the whole list of clauses passed to the construct here, rather than the
639 // ones only applied to omp.target.
640 List<lower::omp::Clause> clauses;
641 extractClauses(eval, clauses);
642 processEval(eval, clauses);
643}
644
645static lower::pft::Evaluation *
646getCollapsedLoopEval(lower::pft::Evaluation &eval, int collapseValue) {
647 // Return the Evaluation of the innermost collapsed loop, or the current one
648 // if there was no COLLAPSE.
649 if (collapseValue == 0)
650 return &eval;
651
652 lower::pft::Evaluation *curEval = &eval.getFirstNestedEvaluation();
653 for (int i = 1; i < collapseValue; i++) {
654 // The nested evaluations should be DoConstructs (i.e. they should form
655 // a loop nest). Each DoConstruct is a tuple <NonLabelDoStmt, Block,
656 // EndDoStmt>.
657 assert(curEval->isA<parser::DoConstruct>());
658 curEval = &*std::next(curEval->getNestedEvaluations().begin());
659 }
660 return curEval;
661}
662
663static void genNestedEvaluations(lower::AbstractConverter &converter,
664 lower::pft::Evaluation &eval,
665 int collapseValue = 0) {
666 lower::pft::Evaluation *curEval = getCollapsedLoopEval(eval, collapseValue);
667
668 for (lower::pft::Evaluation &e : curEval->getNestedEvaluations())
669 converter.genEval(e);
670}
671
672static fir::GlobalOp globalInitialization(lower::AbstractConverter &converter,
673 fir::FirOpBuilder &firOpBuilder,
674 const semantics::Symbol &sym,
675 const lower::pft::Variable &var,
676 mlir::Location currentLocation) {
677 std::string globalName = converter.mangleName(sym);
678 mlir::StringAttr linkage = firOpBuilder.createInternalLinkage();
679 return Fortran::lower::defineGlobal(converter, var, globalName, linkage);
680}
681
682// Get the extended value for \p val by extracting additional variable
683// information from \p base.
684static fir::ExtendedValue getExtendedValue(fir::ExtendedValue base,
685 mlir::Value val) {
686 return base.match(
687 [&](const fir::MutableBoxValue &box) -> fir::ExtendedValue {
688 return fir::MutableBoxValue(val, box.nonDeferredLenParams(), {});
689 },
690 [&](const auto &) -> fir::ExtendedValue {
691 return fir::substBase(base, val);
692 });
693}
694
695#ifndef NDEBUG
696static bool isThreadPrivate(lower::SymbolRef sym) {
697 if (const auto *details = sym->detailsIf<semantics::CommonBlockDetails>()) {
698 for (const auto &obj : details->objects())
699 if (!obj->test(semantics::Symbol::Flag::OmpThreadprivate))
700 return false;
701 return true;
702 }
703 return sym->test(semantics::Symbol::Flag::OmpThreadprivate);
704}
705#endif
706
707static void threadPrivatizeVars(lower::AbstractConverter &converter,
708 lower::pft::Evaluation &eval) {
709 fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
710 mlir::Location currentLocation = converter.getCurrentLocation();
711 mlir::OpBuilder::InsertionGuard guard(firOpBuilder);
712 firOpBuilder.setInsertionPointToStart(firOpBuilder.getAllocaBlock());
713
714 // If the symbol corresponds to the original ThreadprivateOp, use the symbol
715 // value from that operation to create one ThreadprivateOp copy operation
716 // inside the parallel region.
717 // In some cases, however, the symbol will correspond to the original,
718 // non-threadprivate variable. This can happen, for instance, with a common
719 // block, declared in a separate module, used by a parent procedure and
720 // privatized in its child procedure.
721 auto genThreadprivateOp = [&](lower::SymbolRef sym) -> mlir::Value {
722 assert(isThreadPrivate(sym));
723 mlir::Value symValue = converter.getSymbolAddress(sym);
724 mlir::Operation *op = symValue.getDefiningOp();
725 if (auto declOp = mlir::dyn_cast<hlfir::DeclareOp>(op))
726 op = declOp.getMemref().getDefiningOp();
727 if (mlir::isa<mlir::omp::ThreadprivateOp>(op))
728 symValue = mlir::dyn_cast<mlir::omp::ThreadprivateOp>(op).getSymAddr();
729 return firOpBuilder.create<mlir::omp::ThreadprivateOp>(
730 currentLocation, symValue.getType(), symValue);
731 };
732
733 llvm::SetVector<const semantics::Symbol *> threadprivateSyms;
734 converter.collectSymbolSet(eval, threadprivateSyms,
735 semantics::Symbol::Flag::OmpThreadprivate,
736 /*collectSymbols=*/true,
737 /*collectHostAssociatedSymbols=*/true);
738 std::set<semantics::SourceName> threadprivateSymNames;
739
740 // For a COMMON block, the ThreadprivateOp is generated for itself instead of
741 // its members, so only bind the value of the new copied ThreadprivateOp
742 // inside the parallel region to the common block symbol only once for
743 // multiple members in one COMMON block.
744 llvm::SetVector<const semantics::Symbol *> commonSyms;
745 for (std::size_t i = 0; i < threadprivateSyms.size(); i++) {
746 const semantics::Symbol *sym = threadprivateSyms[i];
747 mlir::Value symThreadprivateValue;
748 // The variable may be used more than once, and each reference has one
749 // symbol with the same name. Only do once for references of one variable.
750 if (threadprivateSymNames.find(sym->name()) != threadprivateSymNames.end())
751 continue;
752 threadprivateSymNames.insert(sym->name());
753 if (const semantics::Symbol *common =
754 semantics::FindCommonBlockContaining(sym->GetUltimate())) {
755 mlir::Value commonThreadprivateValue;
756 if (commonSyms.contains(common)) {
757 commonThreadprivateValue = converter.getSymbolAddress(*common);
758 } else {
759 commonThreadprivateValue = genThreadprivateOp(*common);
760 converter.bindSymbol(*common, commonThreadprivateValue);
761 commonSyms.insert(common);
762 }
763 symThreadprivateValue = lower::genCommonBlockMember(
764 converter, currentLocation, sym->GetUltimate(),
765 commonThreadprivateValue);
766 } else {
767 symThreadprivateValue = genThreadprivateOp(*sym);
768 }
769
770 fir::ExtendedValue sexv = converter.getSymbolExtendedValue(*sym);
771 fir::ExtendedValue symThreadprivateExv =
772 getExtendedValue(sexv, symThreadprivateValue);
773 converter.bindSymbol(*sym, symThreadprivateExv);
774 }
775}
776
777static mlir::Operation *
778createAndSetPrivatizedLoopVar(lower::AbstractConverter &converter,
779 mlir::Location loc, mlir::Value indexVal,
780 const semantics::Symbol *sym) {
781 fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
782 mlir::OpBuilder::InsertPoint insPt = firOpBuilder.saveInsertionPoint();
783 firOpBuilder.setInsertionPointToStart(firOpBuilder.getAllocaBlock());
784
785 mlir::Type tempTy = converter.genType(*sym);
786
787 assert(converter.isPresentShallowLookup(*sym) &&
788 "Expected symbol to be in symbol table.");
789
790 firOpBuilder.restoreInsertionPoint(insPt);
791 mlir::Value cvtVal = firOpBuilder.createConvert(loc, tempTy, indexVal);
792 hlfir::Entity lhs{converter.getSymbolAddress(*sym)};
793
794 lhs = hlfir::derefPointersAndAllocatables(loc, firOpBuilder, lhs);
795
796 mlir::Operation *storeOp =
797 firOpBuilder.create<hlfir::AssignOp>(loc, cvtVal, lhs);
798 return storeOp;
799}
800
801// This helper function implements the functionality of "promoting" non-CPTR
802// arguments of use_device_ptr to use_device_addr arguments (automagic
803// conversion of use_device_ptr -> use_device_addr in these cases). The way we
804// do so currently is through the shuffling of operands from the
805// devicePtrOperands to deviceAddrOperands, as well as the types, locations and
806// symbols.
807//
808// This effectively implements some deprecated OpenMP functionality that some
809// legacy applications unfortunately depend on (deprecated in specification
810// version 5.2):
811//
812// "If a list item in a use_device_ptr clause is not of type C_PTR, the behavior
813// is as if the list item appeared in a use_device_addr clause. Support for
814// such list items in a use_device_ptr clause is deprecated."
815static void promoteNonCPtrUseDevicePtrArgsToUseDeviceAddr(
816 llvm::SmallVectorImpl<mlir::Value> &useDeviceAddrVars,
817 llvm::SmallVectorImpl<const semantics::Symbol *> &useDeviceAddrSyms,
818 llvm::SmallVectorImpl<mlir::Value> &useDevicePtrVars,
819 llvm::SmallVectorImpl<const semantics::Symbol *> &useDevicePtrSyms) {
820 // Iterate over our use_device_ptr list and shift all non-cptr arguments into
821 // use_device_addr.
822 auto *varIt = useDevicePtrVars.begin();
823 auto *symIt = useDevicePtrSyms.begin();
824 while (varIt != useDevicePtrVars.end()) {
825 if (fir::isa_builtin_cptr_type(fir::unwrapRefType(varIt->getType()))) {
826 ++varIt;
827 ++symIt;
828 continue;
829 }
830
831 useDeviceAddrVars.push_back(*varIt);
832 useDeviceAddrSyms.push_back(*symIt);
833
834 varIt = useDevicePtrVars.erase(varIt);
835 symIt = useDevicePtrSyms.erase(symIt);
836 }
837}
838
839/// Extract the list of function and variable symbols affected by the given
840/// 'declare target' directive and return the intended device type for them.
841static void getDeclareTargetInfo(
842 lower::AbstractConverter &converter, semantics::SemanticsContext &semaCtx,
843 lower::pft::Evaluation &eval,
844 const parser::OpenMPDeclareTargetConstruct &declareTargetConstruct,
845 mlir::omp::DeclareTargetOperands &clauseOps,
846 llvm::SmallVectorImpl<DeclareTargetCapturePair> &symbolAndClause) {
847 const auto &spec =
848 std::get<parser::OmpDeclareTargetSpecifier>(declareTargetConstruct.t);
849 if (const auto *objectList{parser::Unwrap<parser::OmpObjectList>(spec.u)}) {
850 ObjectList objects{makeObjects(*objectList, semaCtx)};
851 // Case: declare target(func, var1, var2)
852 gatherFuncAndVarSyms(objects, mlir::omp::DeclareTargetCaptureClause::to,
853 symbolAndClause);
854 } else if (const auto *clauseList{
855 parser::Unwrap<parser::OmpClauseList>(spec.u)}) {
856 List<Clause> clauses = makeClauses(*clauseList, semaCtx);
857 if (clauses.empty()) {
858 Fortran::lower::pft::FunctionLikeUnit *owningProc =
859 eval.getOwningProcedure();
860 if (owningProc && (!owningProc->isMainProgram() ||
861 owningProc->getMainProgramSymbol())) {
862 // Case: declare target, implicit capture of function
863 symbolAndClause.emplace_back(mlir::omp::DeclareTargetCaptureClause::to,
864 owningProc->getSubprogramSymbol());
865 }
866 }
867
868 ClauseProcessor cp(converter, semaCtx, clauses);
869 cp.processDeviceType(clauseOps);
870 cp.processEnter(symbolAndClause);
871 cp.processLink(symbolAndClause);
872 cp.processTo(symbolAndClause);
873
874 cp.processTODO<clause::Indirect>(converter.getCurrentLocation(),
875 llvm::omp::Directive::OMPD_declare_target);
876 }
877}
878
879static void collectDeferredDeclareTargets(
880 lower::AbstractConverter &converter, semantics::SemanticsContext &semaCtx,
881 lower::pft::Evaluation &eval,
882 const parser::OpenMPDeclareTargetConstruct &declareTargetConstruct,
883 llvm::SmallVectorImpl<lower::OMPDeferredDeclareTargetInfo>
884 &deferredDeclareTarget) {
885 mlir::omp::DeclareTargetOperands clauseOps;
886 llvm::SmallVector<DeclareTargetCapturePair> symbolAndClause;
887 getDeclareTargetInfo(converter, semaCtx, eval, declareTargetConstruct,
888 clauseOps, symbolAndClause);
889 // Return the device type only if at least one of the targets for the
890 // directive is a function or subroutine
891 mlir::ModuleOp mod = converter.getFirOpBuilder().getModule();
892
893 for (const DeclareTargetCapturePair &symClause : symbolAndClause) {
894 mlir::Operation *op = mod.lookupSymbol(
895 converter.mangleName(std::get<const semantics::Symbol &>(symClause)));
896
897 if (!op) {
898 deferredDeclareTarget.push_back({std::get<0>(symClause),
899 clauseOps.deviceType,
900 std::get<1>(symClause)});
901 }
902 }
903}
904
905static std::optional<mlir::omp::DeclareTargetDeviceType>
906getDeclareTargetFunctionDevice(
907 lower::AbstractConverter &converter, semantics::SemanticsContext &semaCtx,
908 lower::pft::Evaluation &eval,
909 const parser::OpenMPDeclareTargetConstruct &declareTargetConstruct) {
910 mlir::omp::DeclareTargetOperands clauseOps;
911 llvm::SmallVector<DeclareTargetCapturePair> symbolAndClause;
912 getDeclareTargetInfo(converter, semaCtx, eval, declareTargetConstruct,
913 clauseOps, symbolAndClause);
914
915 // Return the device type only if at least one of the targets for the
916 // directive is a function or subroutine
917 mlir::ModuleOp mod = converter.getFirOpBuilder().getModule();
918 for (const DeclareTargetCapturePair &symClause : symbolAndClause) {
919 mlir::Operation *op = mod.lookupSymbol(
920 converter.mangleName(std::get<const semantics::Symbol &>(symClause)));
921
922 if (mlir::isa_and_nonnull<mlir::func::FuncOp>(op))
923 return clauseOps.deviceType;
924 }
925
926 return std::nullopt;
927}
928
929/// Set up the entry block of the given `omp.loop_nest` operation, adding a
930/// block argument for each loop induction variable and allocating and
931/// initializing a private value to hold each of them.
932///
933/// This function can also bind the symbols of any variables that should match
934/// block arguments on parent loop wrapper operations attached to the same
935/// loop. This allows the introduction of any necessary `hlfir.declare`
936/// operations inside of the entry block of the `omp.loop_nest` operation and
937/// not directly under any of the wrappers, which would invalidate them.
938///
939/// \param [in] op - the loop nest operation.
940/// \param [in] converter - PFT to MLIR conversion interface.
941/// \param [in] loc - location.
942/// \param [in] args - symbols of induction variables.
943/// \param [in] wrapperArgs - list of parent loop wrappers and their associated
944/// entry block arguments.
945static void genLoopVars(
946 mlir::Operation *op, lower::AbstractConverter &converter,
947 mlir::Location &loc, llvm::ArrayRef<const semantics::Symbol *> args,
948 llvm::ArrayRef<
949 std::pair<mlir::omp::BlockArgOpenMPOpInterface, const EntryBlockArgs &>>
950 wrapperArgs = {}) {
951 fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
952 auto &region = op->getRegion(0);
953
954 std::size_t loopVarTypeSize = 0;
955 for (const semantics::Symbol *arg : args)
956 loopVarTypeSize = std::max(loopVarTypeSize, arg->GetUltimate().size());
957 mlir::Type loopVarType = getLoopVarType(converter, loopVarTypeSize);
958 llvm::SmallVector<mlir::Type> tiv(args.size(), loopVarType);
959 llvm::SmallVector<mlir::Location> locs(args.size(), loc);
960 firOpBuilder.createBlock(&region, {}, tiv, locs);
961
962 // Update nested wrapper operands if parent wrappers have mapped these values
963 // to block arguments.
964 //
965 // Binding these values earlier would take care of this, but we cannot rely on
966 // that approach because binding in between the creation of a wrapper and the
967 // next one would result in 'hlfir.declare' operations being introduced inside
968 // of a wrapper, which is illegal.
969 mlir::IRMapping mapper;
970 for (auto [argGeneratingOp, blockArgs] : wrapperArgs) {
971 for (mlir::OpOperand &operand : argGeneratingOp->getOpOperands())
972 operand.set(mapper.lookupOrDefault(operand.get()));
973
974 for (const auto [arg, var] : llvm::zip_equal(
975 argGeneratingOp->getRegion(0).getArguments(), blockArgs.getVars()))
976 mapper.map(var, arg);
977 }
978
979 // Bind the entry block arguments of parent wrappers to the corresponding
980 // symbols.
981 for (auto [argGeneratingOp, blockArgs] : wrapperArgs)
982 bindEntryBlockArgs(converter, argGeneratingOp, blockArgs);
983
984 // The argument is not currently in memory, so make a temporary for the
985 // argument, and store it there, then bind that location to the argument.
986 mlir::Operation *storeOp = nullptr;
987 for (auto [argIndex, argSymbol] : llvm::enumerate(args)) {
988 mlir::Value indexVal = fir::getBase(region.front().getArgument(argIndex));
989 storeOp =
990 createAndSetPrivatizedLoopVar(converter, loc, indexVal, argSymbol);
991 }
992 firOpBuilder.setInsertionPointAfter(storeOp);
993}
994
995static clause::Defaultmap::ImplicitBehavior
996getDefaultmapIfPresent(const DefaultMapsTy &defaultMaps, mlir::Type varType) {
997 using DefMap = clause::Defaultmap;
998
999 if (defaultMaps.empty())
1000 return DefMap::ImplicitBehavior::Default;
1001
1002 if (llvm::is_contained(defaultMaps, DefMap::VariableCategory::All))
1003 return defaultMaps.at(DefMap::VariableCategory::All);
1004
1005 // NOTE: Unsure if complex and/or vector falls into a scalar type
1006 // or aggregate, but the current default implicit behaviour is to
1007 // treat them as such (c_ptr has its own behaviour, so perhaps
1008 // being lumped in as a scalar isn't the right thing).
1009 if ((fir::isa_trivial(varType) || fir::isa_char(varType) ||
1010 fir::isa_builtin_cptr_type(varType)) &&
1011 llvm::is_contained(defaultMaps, DefMap::VariableCategory::Scalar))
1012 return defaultMaps.at(DefMap::VariableCategory::Scalar);
1013
1014 if (fir::isPointerType(varType) &&
1015 llvm::is_contained(defaultMaps, DefMap::VariableCategory::Pointer))
1016 return defaultMaps.at(DefMap::VariableCategory::Pointer);
1017
1018 if (fir::isAllocatableType(varType) &&
1019 llvm::is_contained(defaultMaps, DefMap::VariableCategory::Allocatable))
1020 return defaultMaps.at(DefMap::VariableCategory::Allocatable);
1021
1022 if (fir::isa_aggregate(varType) &&
1023 llvm::is_contained(defaultMaps, DefMap::VariableCategory::Aggregate))
1024 return defaultMaps.at(DefMap::VariableCategory::Aggregate);
1025
1026 return DefMap::ImplicitBehavior::Default;
1027}
1028
1029static std::pair<llvm::omp::OpenMPOffloadMappingFlags,
1030 mlir::omp::VariableCaptureKind>
1031getImplicitMapTypeAndKind(fir::FirOpBuilder &firOpBuilder,
1032 lower::AbstractConverter &converter,
1033 const DefaultMapsTy &defaultMaps, mlir::Type varType,
1034 mlir::Location loc, const semantics::Symbol &sym) {
1035 using DefMap = clause::Defaultmap;
1036 // Check if a value of type `type` can be passed to the kernel by value.
1037 // All kernel parameters are of pointer type, so if the value can be
1038 // represented inside of a pointer, then it can be passed by value.
1039 auto isLiteralType = [&](mlir::Type type) {
1040 const mlir::DataLayout &dl = firOpBuilder.getDataLayout();
1041 mlir::Type ptrTy =
1042 mlir::LLVM::LLVMPointerType::get(&converter.getMLIRContext());
1043 uint64_t ptrSize = dl.getTypeSize(ptrTy);
1044 uint64_t ptrAlign = dl.getTypePreferredAlignment(ptrTy);
1045
1046 auto [size, align] = fir::getTypeSizeAndAlignmentOrCrash(
1047 loc, type, dl, converter.getKindMap());
1048 return size <= ptrSize && align <= ptrAlign;
1049 };
1050
1051 llvm::omp::OpenMPOffloadMappingFlags mapFlag =
1052 llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT;
1053
1054 auto implicitBehaviour = getDefaultmapIfPresent(defaultMaps, varType);
1055 if (implicitBehaviour == DefMap::ImplicitBehavior::Default) {
1056 mlir::omp::VariableCaptureKind captureKind =
1057 mlir::omp::VariableCaptureKind::ByRef;
1058
1059 // If a variable is specified in declare target link and if device
1060 // type is not specified as `nohost`, it needs to be mapped tofrom
1061 mlir::ModuleOp mod = firOpBuilder.getModule();
1062 mlir::Operation *op = mod.lookupSymbol(converter.mangleName(sym));
1063 auto declareTargetOp =
1064 llvm::dyn_cast_if_present<mlir::omp::DeclareTargetInterface>(op);
1065 if (declareTargetOp && declareTargetOp.isDeclareTarget()) {
1066 if (declareTargetOp.getDeclareTargetCaptureClause() ==
1067 mlir::omp::DeclareTargetCaptureClause::link &&
1068 declareTargetOp.getDeclareTargetDeviceType() !=
1069 mlir::omp::DeclareTargetDeviceType::nohost) {
1070 mapFlag |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TO;
1071 mapFlag |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_FROM;
1072 }
1073 } else if (fir::isa_trivial(varType) || fir::isa_char(varType)) {
1074 // Scalars behave as if they were "firstprivate".
1075 // TODO: Handle objects that are shared/lastprivate or were listed
1076 // in an in_reduction clause.
1077 if (isLiteralType(varType)) {
1078 captureKind = mlir::omp::VariableCaptureKind::ByCopy;
1079 } else {
1080 mapFlag |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TO;
1081 }
1082 } else if (!fir::isa_builtin_cptr_type(varType)) {
1083 mapFlag |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TO;
1084 mapFlag |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_FROM;
1085 }
1086 return std::make_pair(mapFlag, captureKind);
1087 }
1088
1089 switch (implicitBehaviour) {
1090 case DefMap::ImplicitBehavior::Alloc:
1091 return std::make_pair(llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_NONE,
1092 mlir::omp::VariableCaptureKind::ByRef);
1093 break;
1094 case DefMap::ImplicitBehavior::Firstprivate:
1095 case DefMap::ImplicitBehavior::None:
1096 TODO(loc, "Firstprivate and None are currently unsupported defaultmap "
1097 "behaviour");
1098 break;
1099 case DefMap::ImplicitBehavior::From:
1100 return std::make_pair(mapFlag |=
1101 llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_FROM,
1102 mlir::omp::VariableCaptureKind::ByRef);
1103 break;
1104 case DefMap::ImplicitBehavior::Present:
1105 return std::make_pair(mapFlag |=
1106 llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_PRESENT,
1107 mlir::omp::VariableCaptureKind::ByRef);
1108 break;
1109 case DefMap::ImplicitBehavior::To:
1110 return std::make_pair(mapFlag |=
1111 llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TO,
1112 (fir::isa_trivial(varType) || fir::isa_char(varType))
1113 ? mlir::omp::VariableCaptureKind::ByCopy
1114 : mlir::omp::VariableCaptureKind::ByRef);
1115 break;
1116 case DefMap::ImplicitBehavior::Tofrom:
1117 return std::make_pair(mapFlag |=
1118 llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_FROM |
1119 llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TO,
1120 mlir::omp::VariableCaptureKind::ByRef);
1121 break;
1122 case DefMap::ImplicitBehavior::Default:
1123 llvm_unreachable(
1124 "Implicit None Behaviour Should Have Been Handled Earlier");
1125 break;
1126 }
1127
1128 return std::make_pair(mapFlag |=
1129 llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_FROM |
1130 llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TO,
1131 mlir::omp::VariableCaptureKind::ByRef);
1132}
1133
1134static void
1135markDeclareTarget(mlir::Operation *op, lower::AbstractConverter &converter,
1136 mlir::omp::DeclareTargetCaptureClause captureClause,
1137 mlir::omp::DeclareTargetDeviceType deviceType) {
1138 // TODO: Add support for program local variables with declare target applied
1139 auto declareTargetOp = llvm::dyn_cast<mlir::omp::DeclareTargetInterface>(op);
1140 if (!declareTargetOp)
1141 fir::emitFatalError(
1142 converter.getCurrentLocation(),
1143 "Attempt to apply declare target on unsupported operation");
1144
1145 // The function or global already has a declare target applied to it, very
1146 // likely through implicit capture (usage in another declare target
1147 // function/subroutine). It should be marked as any if it has been assigned
1148 // both host and nohost, else we skip, as there is no change
1149 if (declareTargetOp.isDeclareTarget()) {
1150 if (declareTargetOp.getDeclareTargetDeviceType() != deviceType)
1151 declareTargetOp.setDeclareTarget(mlir::omp::DeclareTargetDeviceType::any,
1152 captureClause);
1153 return;
1154 }
1155
1156 declareTargetOp.setDeclareTarget(deviceType, captureClause);
1157}
1158
1159//===----------------------------------------------------------------------===//
1160// Op body generation helper structures and functions
1161//===----------------------------------------------------------------------===//
1162
1163struct OpWithBodyGenInfo {
1164 /// A type for a code-gen callback function. This takes as argument the op for
1165 /// which the code is being generated and returns the arguments of the op's
1166 /// region.
1167 using GenOMPRegionEntryCBFn =
1168 std::function<llvm::SmallVector<const semantics::Symbol *>(
1169 mlir::Operation *)>;
1170
1171 OpWithBodyGenInfo(lower::AbstractConverter &converter,
1172 lower::SymMap &symTable,
1173 semantics::SemanticsContext &semaCtx, mlir::Location loc,
1174 lower::pft::Evaluation &eval, llvm::omp::Directive dir)
1175 : converter(converter), symTable(symTable), semaCtx(semaCtx), loc(loc),
1176 eval(eval), dir(dir) {}
1177
1178 OpWithBodyGenInfo &setClauses(const List<Clause> *value) {
1179 clauses = value;
1180 return *this;
1181 }
1182
1183 OpWithBodyGenInfo &setDataSharingProcessor(DataSharingProcessor *value) {
1184 dsp = value;
1185 return *this;
1186 }
1187
1188 OpWithBodyGenInfo &setEntryBlockArgs(const EntryBlockArgs *value) {
1189 blockArgs = value;
1190 return *this;
1191 }
1192
1193 OpWithBodyGenInfo &setGenRegionEntryCb(GenOMPRegionEntryCBFn value) {
1194 genRegionEntryCB = value;
1195 return *this;
1196 }
1197
1198 OpWithBodyGenInfo &setGenSkeletonOnly(bool value) {
1199 genSkeletonOnly = value;
1200 return *this;
1201 }
1202
1203 /// [inout] converter to use for the clauses.
1204 lower::AbstractConverter &converter;
1205 /// [in] Symbol table
1206 lower::SymMap &symTable;
1207 /// [in] Semantics context
1208 semantics::SemanticsContext &semaCtx;
1209 /// [in] location in source code.
1210 mlir::Location loc;
1211 /// [in] current PFT node/evaluation.
1212 lower::pft::Evaluation &eval;
1213 /// [in] leaf directive for which to generate the op body.
1214 llvm::omp::Directive dir;
1215 /// [in] list of clauses to process.
1216 const List<Clause> *clauses = nullptr;
1217 /// [in] if provided, processes the construct's data-sharing attributes.
1218 DataSharingProcessor *dsp = nullptr;
1219 /// [in] if provided, it is used to create the op's region entry block. It is
1220 /// overriden when a \see genRegionEntryCB is provided. This is only valid for
1221 /// operations implementing the \see mlir::omp::BlockArgOpenMPOpInterface.
1222 const EntryBlockArgs *blockArgs = nullptr;
1223 /// [in] if provided, it overrides the default op's region entry block
1224 /// creation.
1225 GenOMPRegionEntryCBFn genRegionEntryCB = nullptr;
1226 /// [in] if set to `true`, skip generating nested evaluations and dispatching
1227 /// any further leaf constructs.
1228 bool genSkeletonOnly = false;
1229};
1230
1231/// Create the body (block) for an OpenMP Operation.
1232///
1233/// \param [in] op - the operation the body belongs to.
1234/// \param [in] info - options controlling code-gen for the construction.
1235/// \param [in] queue - work queue with nested constructs.
1236/// \param [in] item - item in the queue to generate body for.
1237static void createBodyOfOp(mlir::Operation &op, const OpWithBodyGenInfo &info,
1238 const ConstructQueue &queue,
1239 ConstructQueue::const_iterator item) {
1240 fir::FirOpBuilder &firOpBuilder = info.converter.getFirOpBuilder();
1241
1242 auto insertMarker = [](fir::FirOpBuilder &builder) {
1243 mlir::Value undef = builder.create<fir::UndefOp>(builder.getUnknownLoc(),
1244 builder.getIndexType());
1245 return undef.getDefiningOp();
1246 };
1247
1248 // Create the entry block for the region and collect its arguments for use
1249 // within the region. The entry block will be created as follows:
1250 // - By default, it will be empty and have no arguments.
1251 // - Operations implementing the omp::BlockArgOpenMPOpInterface can set the
1252 // `info.blockArgs` pointer so that block arguments will be those
1253 // corresponding to entry block argument-generating clauses. Binding of
1254 // Fortran symbols to the new MLIR values is done automatically.
1255 // - If the `info.genRegionEntryCB` callback is set, it takes precedence and
1256 // allows callers to manually create the entry block with its intended
1257 // list of arguments and to bind these arguments to their corresponding
1258 // Fortran symbols. This is used for e.g. loop induction variables.
1259 auto regionArgs = [&]() -> llvm::SmallVector<const semantics::Symbol *> {
1260 if (info.genRegionEntryCB)
1261 return info.genRegionEntryCB(&op);
1262
1263 if (info.blockArgs) {
1264 genEntryBlock(firOpBuilder, *info.blockArgs, op.getRegion(0));
1265 bindEntryBlockArgs(info.converter,
1266 llvm::cast<mlir::omp::BlockArgOpenMPOpInterface>(op),
1267 *info.blockArgs);
1268 return llvm::to_vector(info.blockArgs->getSyms());
1269 }
1270
1271 firOpBuilder.createBlock(&op.getRegion(0));
1272 return {};
1273 }();
1274
1275 // Mark the earliest insertion point.
1276 mlir::Operation *marker = insertMarker(firOpBuilder);
1277
1278 // If it is an unstructured region, create empty blocks for all evaluations.
1279 if (lower::omp::isLastItemInQueue(item, queue) &&
1280 info.eval.lowerAsUnstructured()) {
1281 lower::createEmptyRegionBlocks<mlir::omp::TerminatorOp, mlir::omp::YieldOp>(
1282 firOpBuilder, info.eval.getNestedEvaluations());
1283 }
1284
1285 // Start with privatization, so that the lowering of the nested
1286 // code will use the right symbols.
1287 bool isLoop = llvm::omp::getDirectiveAssociation(D: info.dir) ==
1288 llvm::omp::Association::Loop;
1289 bool privatize = info.clauses;
1290
1291 firOpBuilder.setInsertionPoint(marker);
1292 std::optional<DataSharingProcessor> tempDsp;
1293 if (privatize && !info.dsp) {
1294 tempDsp.emplace(info.converter, info.semaCtx, *info.clauses, info.eval,
1295 Fortran::lower::omp::isLastItemInQueue(item, queue),
1296 /*useDelayedPrivatization=*/false, info.symTable);
1297 tempDsp->processStep1();
1298 }
1299
1300 if (info.dir == llvm::omp::Directive::OMPD_parallel) {
1301 threadPrivatizeVars(info.converter, info.eval);
1302 if (info.clauses) {
1303 firOpBuilder.setInsertionPoint(marker);
1304 ClauseProcessor(info.converter, info.semaCtx, *info.clauses)
1305 .processCopyin();
1306 }
1307 }
1308
1309 if (!info.genSkeletonOnly) {
1310 if (ConstructQueue::const_iterator next = std::next(item);
1311 next != queue.end()) {
1312 genOMPDispatch(info.converter, info.symTable, info.semaCtx, info.eval,
1313 info.loc, queue, next);
1314 } else {
1315 // genFIR(Evaluation&) tries to patch up unterminated blocks, causing
1316 // a lot of complications for our approach if the terminator generation
1317 // is delayed past this point. Insert a temporary terminator here, then
1318 // delete it.
1319 firOpBuilder.setInsertionPointToEnd(&op.getRegion(0).back());
1320 auto *temp = lower::genOpenMPTerminator(firOpBuilder, &op, info.loc);
1321 firOpBuilder.setInsertionPointAfter(marker);
1322 genNestedEvaluations(info.converter, info.eval);
1323 temp->erase();
1324 }
1325 }
1326
1327 // Get or create a unique exiting block from the given region, or
1328 // return nullptr if there is no exiting block.
1329 auto getUniqueExit = [&](mlir::Region &region) -> mlir::Block * {
1330 // Find the blocks where the OMP terminator should go. In simple cases
1331 // it is the single block in the operation's region. When the region
1332 // is more complicated, especially with unstructured control flow, there
1333 // may be multiple blocks, and some of them may have non-OMP terminators
1334 // resulting from lowering of the code contained within the operation.
1335 // All the remaining blocks are potential exit points from the op's region.
1336 //
1337 // Explicit control flow cannot exit any OpenMP region (other than via
1338 // STOP), and that is enforced by semantic checks prior to lowering. STOP
1339 // statements are lowered to a function call.
1340
1341 // Collect unterminated blocks.
1342 llvm::SmallVector<mlir::Block *> exits;
1343 for (mlir::Block &b : region) {
1344 if (b.empty() || !b.back().hasTrait<mlir::OpTrait::IsTerminator>())
1345 exits.push_back(&b);
1346 }
1347
1348 if (exits.empty())
1349 return nullptr;
1350 // If there already is a unique exiting block, do not create another one.
1351 // Additionally, some ops (e.g. omp.sections) require only 1 block in
1352 // its region.
1353 if (exits.size() == 1)
1354 return exits[0];
1355 mlir::Block *exit = firOpBuilder.createBlock(&region);
1356 for (mlir::Block *b : exits) {
1357 firOpBuilder.setInsertionPointToEnd(b);
1358 firOpBuilder.create<mlir::cf::BranchOp>(info.loc, exit);
1359 }
1360 return exit;
1361 };
1362
1363 if (auto *exitBlock = getUniqueExit(op.getRegion(0))) {
1364 firOpBuilder.setInsertionPointToEnd(exitBlock);
1365 auto *term = lower::genOpenMPTerminator(firOpBuilder, &op, info.loc);
1366 // Only insert lastprivate code when there actually is an exit block.
1367 // Such a block may not exist if the nested code produced an infinite
1368 // loop (this may not make sense in production code, but a user could
1369 // write that and we should handle it).
1370 firOpBuilder.setInsertionPoint(term);
1371 if (privatize) {
1372 // DataSharingProcessor::processStep2() may create operations before/after
1373 // the one passed as argument. We need to treat loop wrappers and their
1374 // nested loop as a unit, so we need to pass the bottom level wrapper (if
1375 // present). Otherwise, these operations will be inserted within a
1376 // wrapper region.
1377 mlir::Operation *privatizationBottomLevelOp = &op;
1378 if (auto loopNest = llvm::dyn_cast<mlir::omp::LoopNestOp>(op)) {
1379 llvm::SmallVector<mlir::omp::LoopWrapperInterface> wrappers;
1380 loopNest.gatherWrappers(wrappers);
1381 if (!wrappers.empty())
1382 privatizationBottomLevelOp = &*wrappers.front();
1383 }
1384
1385 if (!info.dsp) {
1386 assert(tempDsp.has_value());
1387 tempDsp->processStep2(privatizationBottomLevelOp, isLoop);
1388 } else {
1389 if (isLoop && regionArgs.size() > 0) {
1390 for (const auto &regionArg : regionArgs) {
1391 info.dsp->pushLoopIV(info.converter.getSymbolAddress(*regionArg));
1392 }
1393 }
1394 info.dsp->processStep2(privatizationBottomLevelOp, isLoop);
1395 }
1396 }
1397 }
1398
1399 firOpBuilder.setInsertionPointAfter(marker);
1400 marker->erase();
1401}
1402
1403static void genBodyOfTargetDataOp(
1404 lower::AbstractConverter &converter, lower::SymMap &symTable,
1405 semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
1406 mlir::omp::TargetDataOp &dataOp, const EntryBlockArgs &args,
1407 const mlir::Location &currentLocation, const ConstructQueue &queue,
1408 ConstructQueue::const_iterator item) {
1409 fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
1410
1411 genEntryBlock(firOpBuilder, args, dataOp.getRegion());
1412 bindEntryBlockArgs(converter, dataOp, args);
1413
1414 // Insert dummy instruction to remember the insertion position. The
1415 // marker will be deleted by clean up passes since there are no uses.
1416 // Remembering the position for further insertion is important since
1417 // there are hlfir.declares inserted above while setting block arguments
1418 // and new code from the body should be inserted after that.
1419 mlir::Value undefMarker = firOpBuilder.create<fir::UndefOp>(
1420 dataOp.getLoc(), firOpBuilder.getIndexType());
1421
1422 // Create blocks for unstructured regions. This has to be done since
1423 // blocks are initially allocated with the function as the parent region.
1424 if (eval.lowerAsUnstructured()) {
1425 lower::createEmptyRegionBlocks<mlir::omp::TerminatorOp, mlir::omp::YieldOp>(
1426 firOpBuilder, eval.getNestedEvaluations());
1427 }
1428
1429 firOpBuilder.create<mlir::omp::TerminatorOp>(currentLocation);
1430
1431 // Set the insertion point after the marker.
1432 firOpBuilder.setInsertionPointAfter(undefMarker.getDefiningOp());
1433
1434 if (ConstructQueue::const_iterator next = std::next(item);
1435 next != queue.end()) {
1436 genOMPDispatch(converter, symTable, semaCtx, eval, currentLocation, queue,
1437 next);
1438 } else {
1439 genNestedEvaluations(converter, eval);
1440 }
1441}
1442
1443// This generates intermediate common block member accesses within a region
1444// and then rebinds the members symbol to the intermediate accessors we have
1445// generated so that subsequent code generation will utilise these instead.
1446//
1447// When the scope changes, the bindings to the intermediate accessors should
1448// be dropped in place of the original symbol bindings.
1449//
1450// This is for utilisation with TargetOp.
1451static void genIntermediateCommonBlockAccessors(
1452 Fortran::lower::AbstractConverter &converter,
1453 const mlir::Location &currentLocation,
1454 llvm::ArrayRef<mlir::BlockArgument> mapBlockArgs,
1455 llvm::ArrayRef<const Fortran::semantics::Symbol *> mapSyms) {
1456 // Iterate over the symbol list, which will be shorter than the list of
1457 // arguments if new entry block arguments were introduced to implicitly map
1458 // outside values used by the bounds cloned into the target region. In that
1459 // case, the additional block arguments do not need processing here.
1460 for (auto [mapSym, mapArg] : llvm::zip_first(mapSyms, mapBlockArgs)) {
1461 auto *details = mapSym->detailsIf<Fortran::semantics::CommonBlockDetails>();
1462 if (!details)
1463 continue;
1464
1465 for (auto obj : details->objects()) {
1466 auto targetCBMemberBind = Fortran::lower::genCommonBlockMember(
1467 converter, currentLocation, *obj, mapArg);
1468 fir::ExtendedValue sexv = converter.getSymbolExtendedValue(*obj);
1469 fir::ExtendedValue targetCBExv =
1470 getExtendedValue(sexv, targetCBMemberBind);
1471 converter.bindSymbol(*obj, targetCBExv);
1472 }
1473 }
1474}
1475
1476// This functions creates a block for the body of the targetOp's region. It adds
1477// all the symbols present in mapSymbols as block arguments to this block.
1478static void genBodyOfTargetOp(
1479 lower::AbstractConverter &converter, lower::SymMap &symTable,
1480 semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
1481 mlir::omp::TargetOp &targetOp, const EntryBlockArgs &args,
1482 const mlir::Location &currentLocation, const ConstructQueue &queue,
1483 ConstructQueue::const_iterator item, DataSharingProcessor &dsp) {
1484 fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
1485 auto argIface = llvm::cast<mlir::omp::BlockArgOpenMPOpInterface>(*targetOp);
1486
1487 mlir::Region &region = targetOp.getRegion();
1488 mlir::Block *entryBlock = genEntryBlock(firOpBuilder, args, region);
1489 bindEntryBlockArgs(converter, targetOp, args);
1490 if (HostEvalInfo *hostEvalInfo = getHostEvalInfoStackTop(converter))
1491 hostEvalInfo->bindOperands(argIface.getHostEvalBlockArgs());
1492
1493 // Check if cloning the bounds introduced any dependency on the outer region.
1494 // If so, then either clone them as well if they are MemoryEffectFree, or else
1495 // copy them to a new temporary and add them to the map and block_argument
1496 // lists and replace their uses with the new temporary.
1497 llvm::SetVector<mlir::Value> valuesDefinedAbove;
1498 mlir::getUsedValuesDefinedAbove(region, valuesDefinedAbove);
1499 while (!valuesDefinedAbove.empty()) {
1500 for (mlir::Value val : valuesDefinedAbove) {
1501 mlir::Operation *valOp = val.getDefiningOp();
1502
1503 // NOTE: We skip BoxDimsOp's as the lesser of two evils is to map the
1504 // indices separately, as the alternative is to eventually map the Box,
1505 // which comes with a fairly large overhead comparatively. We could be
1506 // more robust about this and check using a BackwardsSlice to see if we
1507 // run the risk of mapping a box.
1508 if (valOp && mlir::isMemoryEffectFree(valOp) &&
1509 !mlir::isa<fir::BoxDimsOp>(valOp)) {
1510 mlir::Operation *clonedOp = valOp->clone();
1511 entryBlock->push_front(clonedOp);
1512
1513 auto replace = [entryBlock](mlir::OpOperand &use) {
1514 return use.getOwner()->getBlock() == entryBlock;
1515 };
1516
1517 valOp->getResults().replaceUsesWithIf(clonedOp->getResults(), replace);
1518 valOp->replaceUsesWithIf(clonedOp, replace);
1519 } else {
1520 auto savedIP = firOpBuilder.getInsertionPoint();
1521
1522 if (valOp)
1523 firOpBuilder.setInsertionPointAfter(valOp);
1524 else
1525 // This means val is a block argument
1526 firOpBuilder.setInsertionPoint(targetOp);
1527
1528 auto copyVal =
1529 firOpBuilder.createTemporary(val.getLoc(), val.getType());
1530 firOpBuilder.createStoreWithConvert(copyVal.getLoc(), val, copyVal);
1531
1532 fir::factory::AddrAndBoundsInfo info =
1533 fir::factory::getDataOperandBaseAddr(
1534 firOpBuilder, val, /*isOptional=*/false, val.getLoc());
1535 llvm::SmallVector<mlir::Value> bounds =
1536 fir::factory::genImplicitBoundsOps<mlir::omp::MapBoundsOp,
1537 mlir::omp::MapBoundsType>(
1538 firOpBuilder, info,
1539 hlfir::translateToExtendedValue(val.getLoc(), firOpBuilder,
1540 hlfir::Entity{val})
1541 .first,
1542 /*dataExvIsAssumedSize=*/false, val.getLoc());
1543
1544 std::stringstream name;
1545 firOpBuilder.setInsertionPoint(targetOp);
1546
1547 llvm::omp::OpenMPOffloadMappingFlags mapFlag =
1548 llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT;
1549 mlir::omp::VariableCaptureKind captureKind =
1550 mlir::omp::VariableCaptureKind::ByRef;
1551
1552 mlir::Type eleType = copyVal.getType();
1553 if (auto refType =
1554 mlir::dyn_cast<fir::ReferenceType>(copyVal.getType()))
1555 eleType = refType.getElementType();
1556
1557 if (fir::isa_trivial(eleType) || fir::isa_char(eleType)) {
1558 captureKind = mlir::omp::VariableCaptureKind::ByCopy;
1559 } else if (!fir::isa_builtin_cptr_type(eleType)) {
1560 mapFlag |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TO;
1561 }
1562
1563 mlir::Value mapOp = createMapInfoOp(
1564 firOpBuilder, copyVal.getLoc(), copyVal,
1565 /*varPtrPtr=*/mlir::Value{}, name.str(), bounds,
1566 /*members=*/llvm::SmallVector<mlir::Value>{},
1567 /*membersIndex=*/mlir::ArrayAttr{},
1568 static_cast<
1569 std::underlying_type_t<llvm::omp::OpenMPOffloadMappingFlags>>(
1570 mapFlag),
1571 captureKind, copyVal.getType());
1572
1573 // Get the index of the first non-map argument before modifying mapVars,
1574 // then append an element to mapVars and an associated entry block
1575 // argument at that index.
1576 unsigned insertIndex =
1577 argIface.getMapBlockArgsStart() + argIface.numMapBlockArgs();
1578 targetOp.getMapVarsMutable().append(mapOp);
1579 mlir::Value clonedValArg = region.insertArgument(
1580 insertIndex, copyVal.getType(), copyVal.getLoc());
1581
1582 firOpBuilder.setInsertionPointToStart(entryBlock);
1583 auto loadOp = firOpBuilder.create<fir::LoadOp>(clonedValArg.getLoc(),
1584 clonedValArg);
1585 val.replaceUsesWithIf(loadOp->getResult(0),
1586 [entryBlock](mlir::OpOperand &use) {
1587 return use.getOwner()->getBlock() == entryBlock;
1588 });
1589 firOpBuilder.setInsertionPoint(entryBlock, savedIP);
1590 }
1591 }
1592 valuesDefinedAbove.clear();
1593 mlir::getUsedValuesDefinedAbove(region, valuesDefinedAbove);
1594 }
1595
1596 // Insert dummy instruction to remember the insertion position. The
1597 // marker will be deleted since there are not uses.
1598 // In the HLFIR flow there are hlfir.declares inserted above while
1599 // setting block arguments.
1600 mlir::Value undefMarker = firOpBuilder.create<fir::UndefOp>(
1601 targetOp.getLoc(), firOpBuilder.getIndexType());
1602
1603 // Create blocks for unstructured regions. This has to be done since
1604 // blocks are initially allocated with the function as the parent region.
1605 if (lower::omp::isLastItemInQueue(item, queue) &&
1606 eval.lowerAsUnstructured()) {
1607 lower::createEmptyRegionBlocks<mlir::omp::TerminatorOp, mlir::omp::YieldOp>(
1608 firOpBuilder, eval.getNestedEvaluations());
1609 }
1610
1611 firOpBuilder.create<mlir::omp::TerminatorOp>(currentLocation);
1612
1613 // Create the insertion point after the marker.
1614 firOpBuilder.setInsertionPointAfter(undefMarker.getDefiningOp());
1615
1616 // If we map a common block using it's symbol e.g. map(tofrom: /common_block/)
1617 // and accessing its members within the target region, there is a large
1618 // chance we will end up with uses external to the region accessing the common
1619 // resolve these, we do so by generating new common block member accesses
1620 // within the region, binding them to the member symbol for the scope of the
1621 // region so that subsequent code generation within the region will utilise
1622 // our new member accesses we have created.
1623 genIntermediateCommonBlockAccessors(
1624 converter, currentLocation, argIface.getMapBlockArgs(), args.map.syms);
1625
1626 if (ConstructQueue::const_iterator next = std::next(item);
1627 next != queue.end()) {
1628 genOMPDispatch(converter, symTable, semaCtx, eval, currentLocation, queue,
1629 next);
1630 } else {
1631 genNestedEvaluations(converter, eval);
1632 }
1633
1634 dsp.processStep2(targetOp, /*isLoop=*/false);
1635}
1636
1637template <typename OpTy, typename... Args>
1638static OpTy genOpWithBody(const OpWithBodyGenInfo &info,
1639 const ConstructQueue &queue,
1640 ConstructQueue::const_iterator item, Args &&...args) {
1641 auto op = info.converter.getFirOpBuilder().create<OpTy>(
1642 info.loc, std::forward<Args>(args)...);
1643 createBodyOfOp(*op, info, queue, item);
1644 return op;
1645}
1646
1647template <typename OpTy, typename ClauseOpsTy>
1648static OpTy genWrapperOp(lower::AbstractConverter &converter,
1649 mlir::Location loc, const ClauseOpsTy &clauseOps,
1650 const EntryBlockArgs &args) {
1651 static_assert(
1652 OpTy::template hasTrait<mlir::omp::LoopWrapperInterface::Trait>(),
1653 "expected a loop wrapper");
1654 fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
1655
1656 // Create wrapper.
1657 auto op = firOpBuilder.create<OpTy>(loc, clauseOps);
1658
1659 // Create entry block with arguments.
1660 genEntryBlock(firOpBuilder, args, op.getRegion());
1661
1662 return op;
1663}
1664
1665//===----------------------------------------------------------------------===//
1666// Code generation functions for clauses
1667//===----------------------------------------------------------------------===//
1668
1669static void genCancelClauses(lower::AbstractConverter &converter,
1670 semantics::SemanticsContext &semaCtx,
1671 const List<Clause> &clauses, mlir::Location loc,
1672 mlir::omp::CancelOperands &clauseOps) {
1673 ClauseProcessor cp(converter, semaCtx, clauses);
1674 cp.processCancelDirectiveName(clauseOps);
1675 cp.processIf(llvm::omp::Directive::OMPD_cancel, clauseOps);
1676}
1677
1678static void
1679genCancellationPointClauses(lower::AbstractConverter &converter,
1680 semantics::SemanticsContext &semaCtx,
1681 const List<Clause> &clauses, mlir::Location loc,
1682 mlir::omp::CancellationPointOperands &clauseOps) {
1683 ClauseProcessor cp(converter, semaCtx, clauses);
1684 cp.processCancelDirectiveName(clauseOps);
1685}
1686
1687static void genCriticalDeclareClauses(
1688 lower::AbstractConverter &converter, semantics::SemanticsContext &semaCtx,
1689 const List<Clause> &clauses, mlir::Location loc,
1690 mlir::omp::CriticalDeclareOperands &clauseOps, llvm::StringRef name) {
1691 ClauseProcessor cp(converter, semaCtx, clauses);
1692 cp.processHint(clauseOps);
1693 clauseOps.symName =
1694 mlir::StringAttr::get(converter.getFirOpBuilder().getContext(), name);
1695}
1696
1697static void genDistributeClauses(lower::AbstractConverter &converter,
1698 semantics::SemanticsContext &semaCtx,
1699 lower::StatementContext &stmtCtx,
1700 const List<Clause> &clauses,
1701 mlir::Location loc,
1702 mlir::omp::DistributeOperands &clauseOps) {
1703 ClauseProcessor cp(converter, semaCtx, clauses);
1704 cp.processAllocate(clauseOps);
1705 cp.processDistSchedule(stmtCtx, clauseOps);
1706 cp.processOrder(clauseOps);
1707}
1708
1709static void genFlushClauses(lower::AbstractConverter &converter,
1710 semantics::SemanticsContext &semaCtx,
1711 const ObjectList &objects,
1712 const List<Clause> &clauses, mlir::Location loc,
1713 llvm::SmallVectorImpl<mlir::Value> &operandRange) {
1714 if (!objects.empty())
1715 genObjectList(objects, converter, operandRange);
1716
1717 ClauseProcessor cp(converter, semaCtx, clauses);
1718 cp.processTODO<clause::AcqRel, clause::Acquire, clause::Release,
1719 clause::SeqCst>(loc, llvm::omp::OMPD_flush);
1720}
1721
1722static void
1723genLoopNestClauses(lower::AbstractConverter &converter,
1724 semantics::SemanticsContext &semaCtx,
1725 lower::pft::Evaluation &eval, const List<Clause> &clauses,
1726 mlir::Location loc, mlir::omp::LoopNestOperands &clauseOps,
1727 llvm::SmallVectorImpl<const semantics::Symbol *> &iv) {
1728 ClauseProcessor cp(converter, semaCtx, clauses);
1729
1730 HostEvalInfo *hostEvalInfo = getHostEvalInfoStackTop(converter);
1731 if (!hostEvalInfo || !hostEvalInfo->apply(clauseOps, iv))
1732 cp.processCollapse(loc, eval, clauseOps, iv);
1733
1734 clauseOps.loopInclusive = converter.getFirOpBuilder().getUnitAttr();
1735}
1736
1737static void genLoopClauses(
1738 lower::AbstractConverter &converter, semantics::SemanticsContext &semaCtx,
1739 const List<Clause> &clauses, mlir::Location loc,
1740 mlir::omp::LoopOperands &clauseOps,
1741 llvm::SmallVectorImpl<const semantics::Symbol *> &reductionSyms) {
1742 ClauseProcessor cp(converter, semaCtx, clauses);
1743 cp.processBind(clauseOps);
1744 cp.processOrder(clauseOps);
1745 cp.processReduction(loc, clauseOps, reductionSyms);
1746 cp.processTODO<clause::Lastprivate>(loc, llvm::omp::Directive::OMPD_loop);
1747}
1748
1749static void genMaskedClauses(lower::AbstractConverter &converter,
1750 semantics::SemanticsContext &semaCtx,
1751 lower::StatementContext &stmtCtx,
1752 const List<Clause> &clauses, mlir::Location loc,
1753 mlir::omp::MaskedOperands &clauseOps) {
1754 ClauseProcessor cp(converter, semaCtx, clauses);
1755 cp.processFilter(stmtCtx, clauseOps);
1756}
1757
1758static void
1759genOrderedRegionClauses(lower::AbstractConverter &converter,
1760 semantics::SemanticsContext &semaCtx,
1761 const List<Clause> &clauses, mlir::Location loc,
1762 mlir::omp::OrderedRegionOperands &clauseOps) {
1763 ClauseProcessor cp(converter, semaCtx, clauses);
1764 cp.processTODO<clause::Simd>(loc, llvm::omp::Directive::OMPD_ordered);
1765}
1766
1767static void genParallelClauses(
1768 lower::AbstractConverter &converter, semantics::SemanticsContext &semaCtx,
1769 lower::StatementContext &stmtCtx, const List<Clause> &clauses,
1770 mlir::Location loc, mlir::omp::ParallelOperands &clauseOps,
1771 llvm::SmallVectorImpl<const semantics::Symbol *> &reductionSyms) {
1772 ClauseProcessor cp(converter, semaCtx, clauses);
1773 cp.processAllocate(clauseOps);
1774 cp.processIf(llvm::omp::Directive::OMPD_parallel, clauseOps);
1775
1776 HostEvalInfo *hostEvalInfo = getHostEvalInfoStackTop(converter);
1777 if (!hostEvalInfo || !hostEvalInfo->apply(clauseOps))
1778 cp.processNumThreads(stmtCtx, clauseOps);
1779
1780 cp.processProcBind(clauseOps);
1781 cp.processReduction(loc, clauseOps, reductionSyms);
1782}
1783
1784static void genScanClauses(lower::AbstractConverter &converter,
1785 semantics::SemanticsContext &semaCtx,
1786 const List<Clause> &clauses, mlir::Location loc,
1787 mlir::omp::ScanOperands &clauseOps) {
1788 ClauseProcessor cp(converter, semaCtx, clauses);
1789 cp.processInclusive(loc, clauseOps);
1790 cp.processExclusive(loc, clauseOps);
1791}
1792
1793static void genSectionsClauses(
1794 lower::AbstractConverter &converter, semantics::SemanticsContext &semaCtx,
1795 const List<Clause> &clauses, mlir::Location loc,
1796 mlir::omp::SectionsOperands &clauseOps,
1797 llvm::SmallVectorImpl<const semantics::Symbol *> &reductionSyms) {
1798 ClauseProcessor cp(converter, semaCtx, clauses);
1799 cp.processAllocate(clauseOps);
1800 cp.processNowait(clauseOps);
1801 cp.processReduction(loc, clauseOps, reductionSyms);
1802 // TODO Support delayed privatization.
1803}
1804
1805static void genSimdClauses(
1806 lower::AbstractConverter &converter, semantics::SemanticsContext &semaCtx,
1807 const List<Clause> &clauses, mlir::Location loc,
1808 mlir::omp::SimdOperands &clauseOps,
1809 llvm::SmallVectorImpl<const semantics::Symbol *> &reductionSyms) {
1810 ClauseProcessor cp(converter, semaCtx, clauses);
1811 cp.processAligned(clauseOps);
1812 cp.processIf(llvm::omp::Directive::OMPD_simd, clauseOps);
1813 cp.processNontemporal(clauseOps);
1814 cp.processOrder(clauseOps);
1815 cp.processReduction(loc, clauseOps, reductionSyms);
1816 cp.processSafelen(clauseOps);
1817 cp.processSimdlen(clauseOps);
1818
1819 cp.processTODO<clause::Linear>(loc, llvm::omp::Directive::OMPD_simd);
1820}
1821
1822static void genSingleClauses(lower::AbstractConverter &converter,
1823 semantics::SemanticsContext &semaCtx,
1824 const List<Clause> &clauses, mlir::Location loc,
1825 mlir::omp::SingleOperands &clauseOps) {
1826 ClauseProcessor cp(converter, semaCtx, clauses);
1827 cp.processAllocate(clauseOps);
1828 cp.processCopyprivate(loc, clauseOps);
1829 cp.processNowait(clauseOps);
1830 // TODO Support delayed privatization.
1831}
1832
1833static void genTargetClauses(
1834 lower::AbstractConverter &converter, semantics::SemanticsContext &semaCtx,
1835 lower::SymMap &symTable, lower::StatementContext &stmtCtx,
1836 lower::pft::Evaluation &eval, const List<Clause> &clauses,
1837 mlir::Location loc, mlir::omp::TargetOperands &clauseOps,
1838 DefaultMapsTy &defaultMaps,
1839 llvm::SmallVectorImpl<const semantics::Symbol *> &hasDeviceAddrSyms,
1840 llvm::SmallVectorImpl<const semantics::Symbol *> &isDevicePtrSyms,
1841 llvm::SmallVectorImpl<const semantics::Symbol *> &mapSyms) {
1842 ClauseProcessor cp(converter, semaCtx, clauses);
1843 cp.processBare(clauseOps);
1844 cp.processDefaultMap(stmtCtx, defaultMaps);
1845 cp.processDepend(symTable, stmtCtx, clauseOps);
1846 cp.processDevice(stmtCtx, clauseOps);
1847 cp.processHasDeviceAddr(stmtCtx, clauseOps, hasDeviceAddrSyms);
1848 if (HostEvalInfo *hostEvalInfo = getHostEvalInfoStackTop(converter)) {
1849 // Only process host_eval if compiling for the host device.
1850 processHostEvalClauses(converter, semaCtx, stmtCtx, eval, loc);
1851 hostEvalInfo->collectValues(clauseOps.hostEvalVars);
1852 }
1853 cp.processIf(llvm::omp::Directive::OMPD_target, clauseOps);
1854 cp.processIsDevicePtr(clauseOps, isDevicePtrSyms);
1855 cp.processMap(loc, stmtCtx, clauseOps, llvm::omp::Directive::OMPD_unknown,
1856 &mapSyms);
1857 cp.processNowait(clauseOps);
1858 cp.processThreadLimit(stmtCtx, clauseOps);
1859
1860 cp.processTODO<clause::Allocate, clause::InReduction, clause::UsesAllocators>(
1861 loc, llvm::omp::Directive::OMPD_target);
1862
1863 // `target private(..)` is only supported in delayed privatization mode.
1864 if (!enableDelayedPrivatizationStaging)
1865 cp.processTODO<clause::Firstprivate, clause::Private>(
1866 loc, llvm::omp::Directive::OMPD_target);
1867}
1868
1869static void genTargetDataClauses(
1870 lower::AbstractConverter &converter, semantics::SemanticsContext &semaCtx,
1871 lower::StatementContext &stmtCtx, const List<Clause> &clauses,
1872 mlir::Location loc, mlir::omp::TargetDataOperands &clauseOps,
1873 llvm::SmallVectorImpl<const semantics::Symbol *> &useDeviceAddrSyms,
1874 llvm::SmallVectorImpl<const semantics::Symbol *> &useDevicePtrSyms) {
1875 ClauseProcessor cp(converter, semaCtx, clauses);
1876 cp.processDevice(stmtCtx, clauseOps);
1877 cp.processIf(llvm::omp::Directive::OMPD_target_data, clauseOps);
1878 cp.processMap(loc, stmtCtx, clauseOps);
1879 cp.processUseDeviceAddr(stmtCtx, clauseOps, useDeviceAddrSyms);
1880 cp.processUseDevicePtr(stmtCtx, clauseOps, useDevicePtrSyms);
1881
1882 // This function implements the deprecated functionality of use_device_ptr
1883 // that allows users to provide non-CPTR arguments to it with the caveat
1884 // that the compiler will treat them as use_device_addr. A lot of legacy
1885 // code may still depend on this functionality, so we should support it
1886 // in some manner. We do so currently by simply shifting non-cptr operands
1887 // from the use_device_ptr lists into the use_device_addr lists.
1888 // TODO: Perhaps create a user provideable compiler option that will
1889 // re-introduce a hard-error rather than a warning in these cases.
1890 promoteNonCPtrUseDevicePtrArgsToUseDeviceAddr(
1891 clauseOps.useDeviceAddrVars, useDeviceAddrSyms,
1892 clauseOps.useDevicePtrVars, useDevicePtrSyms);
1893}
1894
1895static void genTargetEnterExitUpdateDataClauses(
1896 lower::AbstractConverter &converter, semantics::SemanticsContext &semaCtx,
1897 lower::SymMap &symTable, lower::StatementContext &stmtCtx,
1898 const List<Clause> &clauses, mlir::Location loc,
1899 llvm::omp::Directive directive,
1900 mlir::omp::TargetEnterExitUpdateDataOperands &clauseOps) {
1901 ClauseProcessor cp(converter, semaCtx, clauses);
1902 cp.processDepend(symTable, stmtCtx, clauseOps);
1903 cp.processDevice(stmtCtx, clauseOps);
1904 cp.processIf(directive, clauseOps);
1905
1906 if (directive == llvm::omp::Directive::OMPD_target_update)
1907 cp.processMotionClauses(stmtCtx, clauseOps);
1908 else
1909 cp.processMap(loc, stmtCtx, clauseOps, directive);
1910
1911 cp.processNowait(clauseOps);
1912}
1913
1914static void genTaskClauses(
1915 lower::AbstractConverter &converter, semantics::SemanticsContext &semaCtx,
1916 lower::SymMap &symTable, lower::StatementContext &stmtCtx,
1917 const List<Clause> &clauses, mlir::Location loc,
1918 mlir::omp::TaskOperands &clauseOps,
1919 llvm::SmallVectorImpl<const semantics::Symbol *> &inReductionSyms) {
1920 ClauseProcessor cp(converter, semaCtx, clauses);
1921 cp.processAllocate(clauseOps);
1922 cp.processDepend(symTable, stmtCtx, clauseOps);
1923 cp.processFinal(stmtCtx, clauseOps);
1924 cp.processIf(llvm::omp::Directive::OMPD_task, clauseOps);
1925 cp.processInReduction(loc, clauseOps, inReductionSyms);
1926 cp.processMergeable(clauseOps);
1927 cp.processPriority(stmtCtx, clauseOps);
1928 cp.processUntied(clauseOps);
1929 cp.processDetach(clauseOps);
1930
1931 cp.processTODO<clause::Affinity>(loc, llvm::omp::Directive::OMPD_task);
1932}
1933
1934static void genTaskgroupClauses(
1935 lower::AbstractConverter &converter, semantics::SemanticsContext &semaCtx,
1936 const List<Clause> &clauses, mlir::Location loc,
1937 mlir::omp::TaskgroupOperands &clauseOps,
1938 llvm::SmallVectorImpl<const semantics::Symbol *> &taskReductionSyms) {
1939 ClauseProcessor cp(converter, semaCtx, clauses);
1940 cp.processAllocate(clauseOps);
1941 cp.processTaskReduction(loc, clauseOps, taskReductionSyms);
1942}
1943
1944static void genTaskloopClauses(lower::AbstractConverter &converter,
1945 semantics::SemanticsContext &semaCtx,
1946 lower::StatementContext &stmtCtx,
1947 const List<Clause> &clauses, mlir::Location loc,
1948 mlir::omp::TaskloopOperands &clauseOps) {
1949
1950 ClauseProcessor cp(converter, semaCtx, clauses);
1951 cp.processGrainsize(stmtCtx, clauseOps);
1952 cp.processNumTasks(stmtCtx, clauseOps);
1953
1954 cp.processTODO<clause::Allocate, clause::Collapse, clause::Default,
1955 clause::Final, clause::If, clause::InReduction,
1956 clause::Lastprivate, clause::Mergeable, clause::Nogroup,
1957 clause::Priority, clause::Reduction, clause::Shared,
1958 clause::Untied>(loc, llvm::omp::Directive::OMPD_taskloop);
1959}
1960
1961static void genTaskwaitClauses(lower::AbstractConverter &converter,
1962 semantics::SemanticsContext &semaCtx,
1963 const List<Clause> &clauses, mlir::Location loc,
1964 mlir::omp::TaskwaitOperands &clauseOps) {
1965 ClauseProcessor cp(converter, semaCtx, clauses);
1966 cp.processTODO<clause::Depend, clause::Nowait>(
1967 loc, llvm::omp::Directive::OMPD_taskwait);
1968}
1969
1970static void genWorkshareClauses(lower::AbstractConverter &converter,
1971 semantics::SemanticsContext &semaCtx,
1972 lower::StatementContext &stmtCtx,
1973 const List<Clause> &clauses, mlir::Location loc,
1974 mlir::omp::WorkshareOperands &clauseOps) {
1975 ClauseProcessor cp(converter, semaCtx, clauses);
1976 cp.processNowait(clauseOps);
1977}
1978
1979static void genTeamsClauses(
1980 lower::AbstractConverter &converter, semantics::SemanticsContext &semaCtx,
1981 lower::StatementContext &stmtCtx, const List<Clause> &clauses,
1982 mlir::Location loc, mlir::omp::TeamsOperands &clauseOps,
1983 llvm::SmallVectorImpl<const semantics::Symbol *> &reductionSyms) {
1984 ClauseProcessor cp(converter, semaCtx, clauses);
1985 cp.processAllocate(clauseOps);
1986 cp.processIf(llvm::omp::Directive::OMPD_teams, clauseOps);
1987
1988 HostEvalInfo *hostEvalInfo = getHostEvalInfoStackTop(converter);
1989 if (!hostEvalInfo || !hostEvalInfo->apply(clauseOps)) {
1990 cp.processNumTeams(stmtCtx, clauseOps);
1991 cp.processThreadLimit(stmtCtx, clauseOps);
1992 }
1993
1994 cp.processReduction(loc, clauseOps, reductionSyms);
1995 // TODO Support delayed privatization.
1996}
1997
1998static void genWsloopClauses(
1999 lower::AbstractConverter &converter, semantics::SemanticsContext &semaCtx,
2000 lower::StatementContext &stmtCtx, const List<Clause> &clauses,
2001 mlir::Location loc, mlir::omp::WsloopOperands &clauseOps,
2002 llvm::SmallVectorImpl<const semantics::Symbol *> &reductionSyms) {
2003 ClauseProcessor cp(converter, semaCtx, clauses);
2004 cp.processNowait(clauseOps);
2005 cp.processOrder(clauseOps);
2006 cp.processOrdered(clauseOps);
2007 cp.processReduction(loc, clauseOps, reductionSyms);
2008 cp.processSchedule(stmtCtx, clauseOps);
2009
2010 cp.processTODO<clause::Allocate, clause::Linear>(
2011 loc, llvm::omp::Directive::OMPD_do);
2012}
2013
2014//===----------------------------------------------------------------------===//
2015// Code generation functions for leaf constructs
2016//===----------------------------------------------------------------------===//
2017
2018static mlir::omp::BarrierOp
2019genBarrierOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
2020 semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
2021 mlir::Location loc, const ConstructQueue &queue,
2022 ConstructQueue::const_iterator item) {
2023 return converter.getFirOpBuilder().create<mlir::omp::BarrierOp>(loc);
2024}
2025
2026static mlir::omp::CancelOp genCancelOp(lower::AbstractConverter &converter,
2027 semantics::SemanticsContext &semaCtx,
2028 lower::pft::Evaluation &eval,
2029 mlir::Location loc,
2030 const ConstructQueue &queue,
2031 ConstructQueue::const_iterator item) {
2032 mlir::omp::CancelOperands clauseOps;
2033 genCancelClauses(converter, semaCtx, item->clauses, loc, clauseOps);
2034
2035 return converter.getFirOpBuilder().create<mlir::omp::CancelOp>(loc,
2036 clauseOps);
2037}
2038
2039static mlir::omp::CancellationPointOp genCancellationPointOp(
2040 lower::AbstractConverter &converter, semantics::SemanticsContext &semaCtx,
2041 lower::pft::Evaluation &eval, mlir::Location loc,
2042 const ConstructQueue &queue, ConstructQueue::const_iterator item) {
2043 mlir::omp::CancellationPointOperands clauseOps;
2044 genCancellationPointClauses(converter, semaCtx, item->clauses, loc,
2045 clauseOps);
2046
2047 return converter.getFirOpBuilder().create<mlir::omp::CancellationPointOp>(
2048 loc, clauseOps);
2049}
2050
2051static mlir::omp::CriticalOp
2052genCriticalOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
2053 semantics::SemanticsContext &semaCtx,
2054 lower::pft::Evaluation &eval, mlir::Location loc,
2055 const ConstructQueue &queue, ConstructQueue::const_iterator item,
2056 const std::optional<parser::Name> &name) {
2057 fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
2058 mlir::FlatSymbolRefAttr nameAttr;
2059
2060 if (name) {
2061 std::string nameStr = name->ToString();
2062 mlir::ModuleOp mod = firOpBuilder.getModule();
2063 auto global = mod.lookupSymbol<mlir::omp::CriticalDeclareOp>(nameStr);
2064 if (!global) {
2065 mlir::omp::CriticalDeclareOperands clauseOps;
2066 genCriticalDeclareClauses(converter, semaCtx, item->clauses, loc,
2067 clauseOps, nameStr);
2068
2069 mlir::OpBuilder modBuilder(mod.getBodyRegion());
2070 global = modBuilder.create<mlir::omp::CriticalDeclareOp>(loc, clauseOps);
2071 }
2072 nameAttr = mlir::FlatSymbolRefAttr::get(firOpBuilder.getContext(),
2073 global.getSymName());
2074 }
2075
2076 return genOpWithBody<mlir::omp::CriticalOp>(
2077 OpWithBodyGenInfo(converter, symTable, semaCtx, loc, eval,
2078 llvm::omp::Directive::OMPD_critical),
2079 queue, item, nameAttr);
2080}
2081
2082static mlir::omp::FlushOp
2083genFlushOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
2084 semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
2085 mlir::Location loc, const ObjectList &objects,
2086 const ConstructQueue &queue, ConstructQueue::const_iterator item) {
2087 llvm::SmallVector<mlir::Value> operandRange;
2088 genFlushClauses(converter, semaCtx, objects, item->clauses, loc,
2089 operandRange);
2090
2091 return converter.getFirOpBuilder().create<mlir::omp::FlushOp>(
2092 converter.getCurrentLocation(), operandRange);
2093}
2094
2095static mlir::omp::LoopNestOp genLoopNestOp(
2096 lower::AbstractConverter &converter, lower::SymMap &symTable,
2097 semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
2098 mlir::Location loc, const ConstructQueue &queue,
2099 ConstructQueue::const_iterator item, mlir::omp::LoopNestOperands &clauseOps,
2100 llvm::ArrayRef<const semantics::Symbol *> iv,
2101 llvm::ArrayRef<
2102 std::pair<mlir::omp::BlockArgOpenMPOpInterface, const EntryBlockArgs &>>
2103 wrapperArgs,
2104 llvm::omp::Directive directive, DataSharingProcessor &dsp) {
2105 auto ivCallback = [&](mlir::Operation *op) {
2106 genLoopVars(op, converter, loc, iv, wrapperArgs);
2107 return llvm::SmallVector<const semantics::Symbol *>(iv);
2108 };
2109
2110 auto *nestedEval =
2111 getCollapsedLoopEval(eval, getCollapseValue(item->clauses));
2112
2113 return genOpWithBody<mlir::omp::LoopNestOp>(
2114 OpWithBodyGenInfo(converter, symTable, semaCtx, loc, *nestedEval,
2115 directive)
2116 .setClauses(&item->clauses)
2117 .setDataSharingProcessor(&dsp)
2118 .setGenRegionEntryCb(ivCallback),
2119 queue, item, clauseOps);
2120}
2121
2122static mlir::omp::LoopOp
2123genLoopOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
2124 semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
2125 mlir::Location loc, const ConstructQueue &queue,
2126 ConstructQueue::const_iterator item) {
2127 mlir::omp::LoopOperands loopClauseOps;
2128 llvm::SmallVector<const semantics::Symbol *> loopReductionSyms;
2129 genLoopClauses(converter, semaCtx, item->clauses, loc, loopClauseOps,
2130 loopReductionSyms);
2131
2132 DataSharingProcessor dsp(converter, semaCtx, item->clauses, eval,
2133 /*shouldCollectPreDeterminedSymbols=*/true,
2134 /*useDelayedPrivatization=*/true, symTable);
2135 dsp.processStep1(clauseOps: &loopClauseOps);
2136
2137 mlir::omp::LoopNestOperands loopNestClauseOps;
2138 llvm::SmallVector<const semantics::Symbol *> iv;
2139 genLoopNestClauses(converter, semaCtx, eval, item->clauses, loc,
2140 loopNestClauseOps, iv);
2141
2142 EntryBlockArgs loopArgs;
2143 loopArgs.priv.syms = dsp.getDelayedPrivSymbols();
2144 loopArgs.priv.vars = loopClauseOps.privateVars;
2145 loopArgs.reduction.syms = loopReductionSyms;
2146 loopArgs.reduction.vars = loopClauseOps.reductionVars;
2147
2148 auto loopOp =
2149 genWrapperOp<mlir::omp::LoopOp>(converter, loc, loopClauseOps, loopArgs);
2150 genLoopNestOp(converter, symTable, semaCtx, eval, loc, queue, item,
2151 loopNestClauseOps, iv, {{loopOp, loopArgs}},
2152 llvm::omp::Directive::OMPD_loop, dsp);
2153 return loopOp;
2154}
2155
2156static mlir::omp::MaskedOp
2157genMaskedOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
2158 lower::StatementContext &stmtCtx,
2159 semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
2160 mlir::Location loc, const ConstructQueue &queue,
2161 ConstructQueue::const_iterator item) {
2162 mlir::omp::MaskedOperands clauseOps;
2163 genMaskedClauses(converter, semaCtx, stmtCtx, item->clauses, loc, clauseOps);
2164
2165 return genOpWithBody<mlir::omp::MaskedOp>(
2166 OpWithBodyGenInfo(converter, symTable, semaCtx, loc, eval,
2167 llvm::omp::Directive::OMPD_masked),
2168 queue, item, clauseOps);
2169}
2170
2171static mlir::omp::MasterOp
2172genMasterOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
2173 semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
2174 mlir::Location loc, const ConstructQueue &queue,
2175 ConstructQueue::const_iterator item) {
2176 return genOpWithBody<mlir::omp::MasterOp>(
2177 OpWithBodyGenInfo(converter, symTable, semaCtx, loc, eval,
2178 llvm::omp::Directive::OMPD_master),
2179 queue, item);
2180}
2181
2182static mlir::omp::OrderedOp
2183genOrderedOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
2184 semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
2185 mlir::Location loc, const ConstructQueue &queue,
2186 ConstructQueue::const_iterator item) {
2187 TODO(loc, "OMPD_ordered");
2188 return nullptr;
2189}
2190
2191static mlir::omp::OrderedRegionOp
2192genOrderedRegionOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
2193 semantics::SemanticsContext &semaCtx,
2194 lower::pft::Evaluation &eval, mlir::Location loc,
2195 const ConstructQueue &queue,
2196 ConstructQueue::const_iterator item) {
2197 mlir::omp::OrderedRegionOperands clauseOps;
2198 genOrderedRegionClauses(converter, semaCtx, item->clauses, loc, clauseOps);
2199
2200 return genOpWithBody<mlir::omp::OrderedRegionOp>(
2201 OpWithBodyGenInfo(converter, symTable, semaCtx, loc, eval,
2202 llvm::omp::Directive::OMPD_ordered),
2203 queue, item, clauseOps);
2204}
2205
2206static mlir::omp::ParallelOp
2207genParallelOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
2208 semantics::SemanticsContext &semaCtx,
2209 lower::pft::Evaluation &eval, mlir::Location loc,
2210 const ConstructQueue &queue, ConstructQueue::const_iterator item,
2211 mlir::omp::ParallelOperands &clauseOps,
2212 const EntryBlockArgs &args, DataSharingProcessor *dsp,
2213 bool isComposite = false) {
2214 assert((!enableDelayedPrivatization || dsp) &&
2215 "expected valid DataSharingProcessor");
2216
2217 OpWithBodyGenInfo genInfo =
2218 OpWithBodyGenInfo(converter, symTable, semaCtx, loc, eval,
2219 llvm::omp::Directive::OMPD_parallel)
2220 .setClauses(&item->clauses)
2221 .setEntryBlockArgs(&args)
2222 .setGenSkeletonOnly(isComposite)
2223 .setDataSharingProcessor(dsp);
2224
2225 auto parallelOp =
2226 genOpWithBody<mlir::omp::ParallelOp>(genInfo, queue, item, clauseOps);
2227 parallelOp.setComposite(isComposite);
2228 return parallelOp;
2229}
2230
2231static mlir::omp::ScanOp
2232genScanOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
2233 semantics::SemanticsContext &semaCtx, mlir::Location loc,
2234 const ConstructQueue &queue, ConstructQueue::const_iterator item) {
2235 mlir::omp::ScanOperands clauseOps;
2236 genScanClauses(converter, semaCtx, item->clauses, loc, clauseOps);
2237 return converter.getFirOpBuilder().create<mlir::omp::ScanOp>(
2238 converter.getCurrentLocation(), clauseOps);
2239}
2240
2241static mlir::omp::SectionsOp
2242genSectionsOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
2243 semantics::SemanticsContext &semaCtx,
2244 lower::pft::Evaluation &eval, mlir::Location loc,
2245 const ConstructQueue &queue,
2246 ConstructQueue::const_iterator item) {
2247 const parser::OpenMPSectionsConstruct *sectionsConstruct =
2248 getSectionsConstructStackTop(converter);
2249 assert(sectionsConstruct && "Missing additional parsing information");
2250
2251 const auto &sectionBlocks =
2252 std::get<parser::OmpSectionBlocks>(sectionsConstruct->t);
2253 mlir::omp::SectionsOperands clauseOps;
2254 llvm::SmallVector<const semantics::Symbol *> reductionSyms;
2255 genSectionsClauses(converter, semaCtx, item->clauses, loc, clauseOps,
2256 reductionSyms);
2257
2258 auto &builder = converter.getFirOpBuilder();
2259
2260 // Insert privatizations before SECTIONS
2261 lower::SymMapScope scope(symTable);
2262 DataSharingProcessor dsp(converter, semaCtx, item->clauses, eval,
2263 lower::omp::isLastItemInQueue(item, queue),
2264 /*useDelayedPrivatization=*/false, symTable);
2265 dsp.processStep1();
2266
2267 List<Clause> nonDsaClauses;
2268 List<const clause::Lastprivate *> lastprivates;
2269
2270 for (const Clause &clause : item->clauses) {
2271 if (clause.id == llvm::omp::Clause::OMPC_lastprivate) {
2272 auto &lastp = std::get<clause::Lastprivate>(clause.u);
2273 lastprivateModifierNotSupported(lastp, converter.getCurrentLocation());
2274 lastprivates.push_back(&lastp);
2275 } else {
2276 switch (clause.id) {
2277 case llvm::omp::Clause::OMPC_firstprivate:
2278 case llvm::omp::Clause::OMPC_private:
2279 case llvm::omp::Clause::OMPC_shared:
2280 break;
2281 default:
2282 nonDsaClauses.push_back(clause);
2283 }
2284 }
2285 }
2286
2287 // SECTIONS construct.
2288 auto sectionsOp = builder.create<mlir::omp::SectionsOp>(loc, clauseOps);
2289
2290 // Create entry block with reduction variables as arguments.
2291 EntryBlockArgs args;
2292 // TODO: Add private syms and vars.
2293 args.reduction.syms = reductionSyms;
2294 args.reduction.vars = clauseOps.reductionVars;
2295
2296 genEntryBlock(builder, args, sectionsOp.getRegion());
2297 mlir::Operation *terminator =
2298 lower::genOpenMPTerminator(builder, sectionsOp, loc);
2299
2300 // Generate nested SECTION constructs.
2301 // This is done here rather than in genOMP([...], OpenMPSectionConstruct )
2302 // because we need to run genReductionVars on each omp.section so that the
2303 // reduction variable gets mapped to the private version
2304 for (auto [construct, nestedEval] :
2305 llvm::zip(sectionBlocks.v, eval.getNestedEvaluations())) {
2306 const auto *sectionConstruct =
2307 std::get_if<parser::OpenMPSectionConstruct>(&construct.u);
2308 if (!sectionConstruct) {
2309 assert(false &&
2310 "unexpected construct nested inside of SECTIONS construct");
2311 continue;
2312 }
2313
2314 ConstructQueue sectionQueue{buildConstructQueue(
2315 converter.getFirOpBuilder().getModule(), semaCtx, nestedEval,
2316 sectionConstruct->source, llvm::omp::Directive::OMPD_section, {})};
2317
2318 builder.setInsertionPoint(terminator);
2319 genOpWithBody<mlir::omp::SectionOp>(
2320 OpWithBodyGenInfo(converter, symTable, semaCtx, loc, nestedEval,
2321 llvm::omp::Directive::OMPD_section)
2322 .setClauses(&sectionQueue.begin()->clauses)
2323 .setDataSharingProcessor(&dsp)
2324 .setEntryBlockArgs(&args),
2325 sectionQueue, sectionQueue.begin());
2326 }
2327
2328 if (!lastprivates.empty()) {
2329 mlir::Region &sectionsBody = sectionsOp.getRegion();
2330 assert(sectionsBody.hasOneBlock());
2331 mlir::Block &body = sectionsBody.front();
2332
2333 auto lastSectionOp = llvm::find_if(
2334 llvm::reverse(body.getOperations()), [](const mlir::Operation &op) {
2335 return llvm::isa<mlir::omp::SectionOp>(op);
2336 });
2337 assert(lastSectionOp != body.rend());
2338
2339 for (const clause::Lastprivate *lastp : lastprivates) {
2340 builder.setInsertionPoint(
2341 lastSectionOp->getRegion(0).back().getTerminator());
2342 mlir::OpBuilder::InsertPoint insp = builder.saveInsertionPoint();
2343 const auto &objList = std::get<ObjectList>(lastp->t);
2344 for (const Object &object : objList) {
2345 semantics::Symbol *sym = object.sym();
2346 if (const auto *common =
2347 sym->detailsIf<semantics::CommonBlockDetails>()) {
2348 for (const auto &obj : common->objects())
2349 converter.copyHostAssociateVar(*obj, &insp, /*hostIsSource=*/false);
2350 } else {
2351 converter.copyHostAssociateVar(*sym, &insp, /*hostIsSource=*/false);
2352 }
2353 }
2354 }
2355 }
2356
2357 // Perform DataSharingProcessor's step2 out of SECTIONS
2358 builder.setInsertionPointAfter(sectionsOp.getOperation());
2359 dsp.processStep2(sectionsOp, false);
2360 // Emit implicit barrier to synchronize threads and avoid data
2361 // races on post-update of lastprivate variables when `nowait`
2362 // clause is present.
2363 if (clauseOps.nowait && !lastprivates.empty())
2364 builder.create<mlir::omp::BarrierOp>(loc);
2365
2366 return sectionsOp;
2367}
2368
2369static mlir::Operation *
2370genScopeOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
2371 semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
2372 mlir::Location loc, const ConstructQueue &queue,
2373 ConstructQueue::const_iterator item) {
2374 TODO(loc, "Scope construct");
2375 return nullptr;
2376}
2377
2378static mlir::omp::SingleOp
2379genSingleOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
2380 semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
2381 mlir::Location loc, const ConstructQueue &queue,
2382 ConstructQueue::const_iterator item) {
2383 mlir::omp::SingleOperands clauseOps;
2384 genSingleClauses(converter, semaCtx, item->clauses, loc, clauseOps);
2385
2386 return genOpWithBody<mlir::omp::SingleOp>(
2387 OpWithBodyGenInfo(converter, symTable, semaCtx, loc, eval,
2388 llvm::omp::Directive::OMPD_single)
2389 .setClauses(&item->clauses),
2390 queue, item, clauseOps);
2391}
2392
2393static mlir::omp::TargetOp
2394genTargetOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
2395 lower::StatementContext &stmtCtx,
2396 semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
2397 mlir::Location loc, const ConstructQueue &queue,
2398 ConstructQueue::const_iterator item) {
2399 fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
2400 bool isTargetDevice =
2401 llvm::cast<mlir::omp::OffloadModuleInterface>(*converter.getModuleOp())
2402 .getIsTargetDevice();
2403
2404 // Introduce a new host_eval information structure for this target region.
2405 if (!isTargetDevice)
2406 converter.getStateStack().stackPush<HostEvalInfoStackFrame>();
2407
2408 mlir::omp::TargetOperands clauseOps;
2409 DefaultMapsTy defaultMaps;
2410 llvm::SmallVector<const semantics::Symbol *> mapSyms, isDevicePtrSyms,
2411 hasDeviceAddrSyms;
2412 genTargetClauses(converter, semaCtx, symTable, stmtCtx, eval, item->clauses,
2413 loc, clauseOps, defaultMaps, hasDeviceAddrSyms,
2414 isDevicePtrSyms, mapSyms);
2415
2416 DataSharingProcessor dsp(converter, semaCtx, item->clauses, eval,
2417 /*shouldCollectPreDeterminedSymbols=*/
2418 lower::omp::isLastItemInQueue(item, queue),
2419 /*useDelayedPrivatization=*/true, symTable);
2420 dsp.processStep1(clauseOps: &clauseOps);
2421
2422 // 5.8.1 Implicit Data-Mapping Attribute Rules
2423 // The following code follows the implicit data-mapping rules to map all the
2424 // symbols used inside the region that do not have explicit data-environment
2425 // attribute clauses (neither data-sharing; e.g. `private`, nor `map`
2426 // clauses).
2427 auto captureImplicitMap = [&](const semantics::Symbol &sym) {
2428 if (dsp.getAllSymbolsToPrivatize().contains(&sym))
2429 return;
2430
2431 // Skip parameters/constants as they do not need to be mapped.
2432 if (semantics::IsNamedConstant(sym))
2433 return;
2434
2435 // These symbols are mapped individually in processHasDeviceAddr.
2436 if (llvm::is_contained(hasDeviceAddrSyms, &sym))
2437 return;
2438
2439 // Structure component symbols don't have bindings, and can only be
2440 // explicitly mapped individually. If a member is captured implicitly
2441 // we map the entirety of the derived type when we find its symbol.
2442 if (sym.owner().IsDerivedType())
2443 return;
2444
2445 // if the symbol is part of an already mapped common block, do not make a
2446 // map for it.
2447 if (const Fortran::semantics::Symbol *common =
2448 Fortran::semantics::FindCommonBlockContaining(sym.GetUltimate()))
2449 if (llvm::is_contained(mapSyms, common))
2450 return;
2451
2452 // If we come across a symbol without a symbol address, we
2453 // return as we cannot process it, this is intended as a
2454 // catch all early exit for symbols that do not have a
2455 // corresponding extended value. Such as subroutines,
2456 // interfaces and named blocks.
2457 if (!converter.getSymbolAddress(sym))
2458 return;
2459
2460 if (!llvm::is_contained(mapSyms, &sym)) {
2461 if (const auto *details =
2462 sym.template detailsIf<semantics::HostAssocDetails>())
2463 converter.copySymbolBinding(details->symbol(), sym);
2464 std::stringstream name;
2465 fir::ExtendedValue dataExv = converter.getSymbolExtendedValue(sym);
2466 name << sym.name().ToString();
2467
2468 mlir::FlatSymbolRefAttr mapperId;
2469 if (sym.GetType()->category() == semantics::DeclTypeSpec::TypeDerived) {
2470 auto &typeSpec = sym.GetType()->derivedTypeSpec();
2471 std::string mapperIdName =
2472 typeSpec.name().ToString() + llvm::omp::OmpDefaultMapperName;
2473 if (auto *sym = converter.getCurrentScope().FindSymbol(mapperIdName))
2474 mapperIdName = converter.mangleName(mapperIdName, sym->owner());
2475 if (converter.getModuleOp().lookupSymbol(mapperIdName))
2476 mapperId = mlir::FlatSymbolRefAttr::get(&converter.getMLIRContext(),
2477 mapperIdName);
2478 }
2479
2480 fir::factory::AddrAndBoundsInfo info =
2481 Fortran::lower::getDataOperandBaseAddr(
2482 converter, firOpBuilder, sym.GetUltimate(),
2483 converter.getCurrentLocation());
2484 llvm::SmallVector<mlir::Value> bounds =
2485 fir::factory::genImplicitBoundsOps<mlir::omp::MapBoundsOp,
2486 mlir::omp::MapBoundsType>(
2487 firOpBuilder, info, dataExv,
2488 semantics::IsAssumedSizeArray(sym.GetUltimate()),
2489 converter.getCurrentLocation());
2490 mlir::Value baseOp = info.rawInput;
2491 mlir::Type eleType = baseOp.getType();
2492 if (auto refType = mlir::dyn_cast<fir::ReferenceType>(baseOp.getType()))
2493 eleType = refType.getElementType();
2494
2495 std::pair<llvm::omp::OpenMPOffloadMappingFlags,
2496 mlir::omp::VariableCaptureKind>
2497 mapFlagAndKind = getImplicitMapTypeAndKind(
2498 firOpBuilder, converter, defaultMaps, eleType, loc, sym);
2499
2500 mlir::Value mapOp = createMapInfoOp(
2501 firOpBuilder, converter.getCurrentLocation(), baseOp,
2502 /*varPtrPtr=*/mlir::Value{}, name.str(), bounds, /*members=*/{},
2503 /*membersIndex=*/mlir::ArrayAttr{},
2504 static_cast<
2505 std::underlying_type_t<llvm::omp::OpenMPOffloadMappingFlags>>(
2506 std::get<0>(mapFlagAndKind)),
2507 std::get<1>(mapFlagAndKind), baseOp.getType(),
2508 /*partialMap=*/false, mapperId);
2509
2510 clauseOps.mapVars.push_back(mapOp);
2511 mapSyms.push_back(&sym);
2512 }
2513 };
2514 lower::pft::visitAllSymbols(eval, captureImplicitMap);
2515
2516 auto targetOp = firOpBuilder.create<mlir::omp::TargetOp>(loc, clauseOps);
2517
2518 llvm::SmallVector<mlir::Value> hasDeviceAddrBaseValues, mapBaseValues;
2519 extractMappedBaseValues(clauseOps.hasDeviceAddrVars, hasDeviceAddrBaseValues);
2520 extractMappedBaseValues(clauseOps.mapVars, mapBaseValues);
2521
2522 EntryBlockArgs args;
2523 args.hasDeviceAddr.syms = hasDeviceAddrSyms;
2524 args.hasDeviceAddr.vars = hasDeviceAddrBaseValues;
2525 args.hostEvalVars = clauseOps.hostEvalVars;
2526 // TODO: Add in_reduction syms and vars.
2527 args.map.syms = mapSyms;
2528 args.map.vars = mapBaseValues;
2529 args.priv.syms = dsp.getDelayedPrivSymbols();
2530 args.priv.vars = clauseOps.privateVars;
2531
2532 genBodyOfTargetOp(converter, symTable, semaCtx, eval, targetOp, args, loc,
2533 queue, item, dsp);
2534
2535 // Remove the host_eval information structure created for this target region.
2536 if (!isTargetDevice)
2537 converter.getStateStack().stackPop();
2538 return targetOp;
2539}
2540
2541static mlir::omp::TargetDataOp genTargetDataOp(
2542 lower::AbstractConverter &converter, lower::SymMap &symTable,
2543 lower::StatementContext &stmtCtx, semantics::SemanticsContext &semaCtx,
2544 lower::pft::Evaluation &eval, mlir::Location loc,
2545 const ConstructQueue &queue, ConstructQueue::const_iterator item) {
2546 mlir::omp::TargetDataOperands clauseOps;
2547 llvm::SmallVector<const semantics::Symbol *> useDeviceAddrSyms,
2548 useDevicePtrSyms;
2549 genTargetDataClauses(converter, semaCtx, stmtCtx, item->clauses, loc,
2550 clauseOps, useDeviceAddrSyms, useDevicePtrSyms);
2551
2552 auto targetDataOp =
2553 converter.getFirOpBuilder().create<mlir::omp::TargetDataOp>(loc,
2554 clauseOps);
2555
2556 llvm::SmallVector<mlir::Value> useDeviceAddrBaseValues,
2557 useDevicePtrBaseValues;
2558 extractMappedBaseValues(clauseOps.useDeviceAddrVars, useDeviceAddrBaseValues);
2559 extractMappedBaseValues(clauseOps.useDevicePtrVars, useDevicePtrBaseValues);
2560
2561 EntryBlockArgs args;
2562 args.useDeviceAddr.syms = useDeviceAddrSyms;
2563 args.useDeviceAddr.vars = useDeviceAddrBaseValues;
2564 args.useDevicePtr.syms = useDevicePtrSyms;
2565 args.useDevicePtr.vars = useDevicePtrBaseValues;
2566
2567 genBodyOfTargetDataOp(converter, symTable, semaCtx, eval, targetDataOp, args,
2568 loc, queue, item);
2569 return targetDataOp;
2570}
2571
2572template <typename OpTy>
2573static OpTy genTargetEnterExitUpdateDataOp(
2574 lower::AbstractConverter &converter, lower::SymMap &symTable,
2575 lower::StatementContext &stmtCtx, semantics::SemanticsContext &semaCtx,
2576 mlir::Location loc, const ConstructQueue &queue,
2577 ConstructQueue::const_iterator item) {
2578 fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
2579
2580 // GCC 9.3.0 emits a (probably) bogus warning about an unused variable.
2581 [[maybe_unused]] llvm::omp::Directive directive;
2582 if constexpr (std::is_same_v<OpTy, mlir::omp::TargetEnterDataOp>) {
2583 directive = llvm::omp::Directive::OMPD_target_enter_data;
2584 } else if constexpr (std::is_same_v<OpTy, mlir::omp::TargetExitDataOp>) {
2585 directive = llvm::omp::Directive::OMPD_target_exit_data;
2586 } else if constexpr (std::is_same_v<OpTy, mlir::omp::TargetUpdateOp>) {
2587 directive = llvm::omp::Directive::OMPD_target_update;
2588 } else {
2589 llvm_unreachable("Unexpected TARGET DATA construct");
2590 }
2591
2592 mlir::omp::TargetEnterExitUpdateDataOperands clauseOps;
2593 genTargetEnterExitUpdateDataClauses(converter, semaCtx, symTable, stmtCtx,
2594 item->clauses, loc, directive, clauseOps);
2595
2596 return firOpBuilder.create<OpTy>(loc, clauseOps);
2597}
2598
2599static mlir::omp::TaskOp
2600genTaskOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
2601 lower::StatementContext &stmtCtx,
2602 semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
2603 mlir::Location loc, const ConstructQueue &queue,
2604 ConstructQueue::const_iterator item) {
2605 mlir::omp::TaskOperands clauseOps;
2606 llvm::SmallVector<const semantics::Symbol *> inReductionSyms;
2607 genTaskClauses(converter, semaCtx, symTable, stmtCtx, item->clauses, loc,
2608 clauseOps, inReductionSyms);
2609
2610 if (!enableDelayedPrivatization)
2611 return genOpWithBody<mlir::omp::TaskOp>(
2612 OpWithBodyGenInfo(converter, symTable, semaCtx, loc, eval,
2613 llvm::omp::Directive::OMPD_task)
2614 .setClauses(&item->clauses),
2615 queue, item, clauseOps);
2616
2617 DataSharingProcessor dsp(converter, semaCtx, item->clauses, eval,
2618 lower::omp::isLastItemInQueue(item, queue),
2619 /*useDelayedPrivatization=*/true, symTable);
2620 dsp.processStep1(clauseOps: &clauseOps);
2621
2622 EntryBlockArgs taskArgs;
2623 taskArgs.priv.syms = dsp.getDelayedPrivSymbols();
2624 taskArgs.priv.vars = clauseOps.privateVars;
2625 taskArgs.inReduction.syms = inReductionSyms;
2626 taskArgs.inReduction.vars = clauseOps.inReductionVars;
2627
2628 return genOpWithBody<mlir::omp::TaskOp>(
2629 OpWithBodyGenInfo(converter, symTable, semaCtx, loc, eval,
2630 llvm::omp::Directive::OMPD_task)
2631 .setClauses(&item->clauses)
2632 .setDataSharingProcessor(&dsp)
2633 .setEntryBlockArgs(&taskArgs),
2634 queue, item, clauseOps);
2635}
2636
2637static mlir::omp::TaskgroupOp
2638genTaskgroupOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
2639 semantics::SemanticsContext &semaCtx,
2640 lower::pft::Evaluation &eval, mlir::Location loc,
2641 const ConstructQueue &queue,
2642 ConstructQueue::const_iterator item) {
2643 mlir::omp::TaskgroupOperands clauseOps;
2644 llvm::SmallVector<const semantics::Symbol *> taskReductionSyms;
2645 genTaskgroupClauses(converter, semaCtx, item->clauses, loc, clauseOps,
2646 taskReductionSyms);
2647
2648 EntryBlockArgs taskgroupArgs;
2649 taskgroupArgs.taskReduction.syms = taskReductionSyms;
2650 taskgroupArgs.taskReduction.vars = clauseOps.taskReductionVars;
2651
2652 return genOpWithBody<mlir::omp::TaskgroupOp>(
2653 OpWithBodyGenInfo(converter, symTable, semaCtx, loc, eval,
2654 llvm::omp::Directive::OMPD_taskgroup)
2655 .setClauses(&item->clauses)
2656 .setEntryBlockArgs(&taskgroupArgs),
2657 queue, item, clauseOps);
2658}
2659
2660static mlir::omp::TaskwaitOp
2661genTaskwaitOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
2662 semantics::SemanticsContext &semaCtx,
2663 lower::pft::Evaluation &eval, mlir::Location loc,
2664 const ConstructQueue &queue,
2665 ConstructQueue::const_iterator item) {
2666 mlir::omp::TaskwaitOperands clauseOps;
2667 genTaskwaitClauses(converter, semaCtx, item->clauses, loc, clauseOps);
2668 return converter.getFirOpBuilder().create<mlir::omp::TaskwaitOp>(loc,
2669 clauseOps);
2670}
2671
2672static mlir::omp::TaskyieldOp
2673genTaskyieldOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
2674 semantics::SemanticsContext &semaCtx,
2675 lower::pft::Evaluation &eval, mlir::Location loc,
2676 const ConstructQueue &queue,
2677 ConstructQueue::const_iterator item) {
2678 return converter.getFirOpBuilder().create<mlir::omp::TaskyieldOp>(loc);
2679}
2680
2681static mlir::omp::WorkshareOp genWorkshareOp(
2682 lower::AbstractConverter &converter, lower::SymMap &symTable,
2683 lower::StatementContext &stmtCtx, semantics::SemanticsContext &semaCtx,
2684 lower::pft::Evaluation &eval, mlir::Location loc,
2685 const ConstructQueue &queue, ConstructQueue::const_iterator item) {
2686 mlir::omp::WorkshareOperands clauseOps;
2687 genWorkshareClauses(converter, semaCtx, stmtCtx, item->clauses, loc,
2688 clauseOps);
2689
2690 return genOpWithBody<mlir::omp::WorkshareOp>(
2691 OpWithBodyGenInfo(converter, symTable, semaCtx, loc, eval,
2692 llvm::omp::Directive::OMPD_workshare)
2693 .setClauses(&item->clauses),
2694 queue, item, clauseOps);
2695}
2696
2697static mlir::omp::TeamsOp
2698genTeamsOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
2699 lower::StatementContext &stmtCtx,
2700 semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
2701 mlir::Location loc, const ConstructQueue &queue,
2702 ConstructQueue::const_iterator item) {
2703 lower::SymMapScope scope(symTable);
2704 mlir::omp::TeamsOperands clauseOps;
2705 llvm::SmallVector<const semantics::Symbol *> reductionSyms;
2706 genTeamsClauses(converter, semaCtx, stmtCtx, item->clauses, loc, clauseOps,
2707 reductionSyms);
2708
2709 EntryBlockArgs args;
2710 // TODO: Add private syms and vars.
2711 args.reduction.syms = reductionSyms;
2712 args.reduction.vars = clauseOps.reductionVars;
2713
2714 return genOpWithBody<mlir::omp::TeamsOp>(
2715 OpWithBodyGenInfo(converter, symTable, semaCtx, loc, eval,
2716 llvm::omp::Directive::OMPD_teams)
2717 .setClauses(&item->clauses)
2718 .setEntryBlockArgs(&args),
2719 queue, item, clauseOps);
2720}
2721
2722//===----------------------------------------------------------------------===//
2723// Code generation functions for the standalone version of constructs that can
2724// also be a leaf of a composite construct
2725//===----------------------------------------------------------------------===//
2726
2727static mlir::omp::DistributeOp genStandaloneDistribute(
2728 lower::AbstractConverter &converter, lower::SymMap &symTable,
2729 lower::StatementContext &stmtCtx, semantics::SemanticsContext &semaCtx,
2730 lower::pft::Evaluation &eval, mlir::Location loc,
2731 const ConstructQueue &queue, ConstructQueue::const_iterator item) {
2732 mlir::omp::DistributeOperands distributeClauseOps;
2733 genDistributeClauses(converter, semaCtx, stmtCtx, item->clauses, loc,
2734 distributeClauseOps);
2735
2736 DataSharingProcessor dsp(converter, semaCtx, item->clauses, eval,
2737 /*shouldCollectPreDeterminedSymbols=*/true,
2738 enableDelayedPrivatization, symTable);
2739 dsp.processStep1(clauseOps: &distributeClauseOps);
2740
2741 mlir::omp::LoopNestOperands loopNestClauseOps;
2742 llvm::SmallVector<const semantics::Symbol *> iv;
2743 genLoopNestClauses(converter, semaCtx, eval, item->clauses, loc,
2744 loopNestClauseOps, iv);
2745
2746 EntryBlockArgs distributeArgs;
2747 distributeArgs.priv.syms = dsp.getDelayedPrivSymbols();
2748 distributeArgs.priv.vars = distributeClauseOps.privateVars;
2749 auto distributeOp = genWrapperOp<mlir::omp::DistributeOp>(
2750 converter, loc, distributeClauseOps, distributeArgs);
2751
2752 genLoopNestOp(converter, symTable, semaCtx, eval, loc, queue, item,
2753 loopNestClauseOps, iv, {{distributeOp, distributeArgs}},
2754 llvm::omp::Directive::OMPD_distribute, dsp);
2755 return distributeOp;
2756}
2757
2758static mlir::omp::WsloopOp genStandaloneDo(
2759 lower::AbstractConverter &converter, lower::SymMap &symTable,
2760 lower::StatementContext &stmtCtx, semantics::SemanticsContext &semaCtx,
2761 lower::pft::Evaluation &eval, mlir::Location loc,
2762 const ConstructQueue &queue, ConstructQueue::const_iterator item) {
2763 mlir::omp::WsloopOperands wsloopClauseOps;
2764 llvm::SmallVector<const semantics::Symbol *> wsloopReductionSyms;
2765 genWsloopClauses(converter, semaCtx, stmtCtx, item->clauses, loc,
2766 wsloopClauseOps, wsloopReductionSyms);
2767
2768 DataSharingProcessor dsp(converter, semaCtx, item->clauses, eval,
2769 /*shouldCollectPreDeterminedSymbols=*/true,
2770 enableDelayedPrivatization, symTable);
2771 dsp.processStep1(clauseOps: &wsloopClauseOps);
2772
2773 mlir::omp::LoopNestOperands loopNestClauseOps;
2774 llvm::SmallVector<const semantics::Symbol *> iv;
2775 genLoopNestClauses(converter, semaCtx, eval, item->clauses, loc,
2776 loopNestClauseOps, iv);
2777
2778 EntryBlockArgs wsloopArgs;
2779 wsloopArgs.priv.syms = dsp.getDelayedPrivSymbols();
2780 wsloopArgs.priv.vars = wsloopClauseOps.privateVars;
2781 wsloopArgs.reduction.syms = wsloopReductionSyms;
2782 wsloopArgs.reduction.vars = wsloopClauseOps.reductionVars;
2783 auto wsloopOp = genWrapperOp<mlir::omp::WsloopOp>(
2784 converter, loc, wsloopClauseOps, wsloopArgs);
2785
2786 genLoopNestOp(converter, symTable, semaCtx, eval, loc, queue, item,
2787 loopNestClauseOps, iv, {{wsloopOp, wsloopArgs}},
2788 llvm::omp::Directive::OMPD_do, dsp);
2789 return wsloopOp;
2790}
2791
2792static mlir::omp::ParallelOp genStandaloneParallel(
2793 lower::AbstractConverter &converter, lower::SymMap &symTable,
2794 lower::StatementContext &stmtCtx, semantics::SemanticsContext &semaCtx,
2795 lower::pft::Evaluation &eval, mlir::Location loc,
2796 const ConstructQueue &queue, ConstructQueue::const_iterator item) {
2797 lower::SymMapScope scope(symTable);
2798 mlir::omp::ParallelOperands parallelClauseOps;
2799 llvm::SmallVector<const semantics::Symbol *> parallelReductionSyms;
2800 genParallelClauses(converter, semaCtx, stmtCtx, item->clauses, loc,
2801 parallelClauseOps, parallelReductionSyms);
2802
2803 std::optional<DataSharingProcessor> dsp;
2804 if (enableDelayedPrivatization) {
2805 dsp.emplace(converter, semaCtx, item->clauses, eval,
2806 lower::omp::isLastItemInQueue(item, queue),
2807 /*useDelayedPrivatization=*/true, symTable);
2808 dsp->processStep1(clauseOps: &parallelClauseOps);
2809 }
2810
2811 EntryBlockArgs parallelArgs;
2812 if (dsp)
2813 parallelArgs.priv.syms = dsp->getDelayedPrivSymbols();
2814 parallelArgs.priv.vars = parallelClauseOps.privateVars;
2815 parallelArgs.reduction.syms = parallelReductionSyms;
2816 parallelArgs.reduction.vars = parallelClauseOps.reductionVars;
2817 return genParallelOp(converter, symTable, semaCtx, eval, loc, queue, item,
2818 parallelClauseOps, parallelArgs,
2819 enableDelayedPrivatization ? &dsp.value() : nullptr);
2820}
2821
2822static mlir::omp::SimdOp
2823genStandaloneSimd(lower::AbstractConverter &converter, lower::SymMap &symTable,
2824 semantics::SemanticsContext &semaCtx,
2825 lower::pft::Evaluation &eval, mlir::Location loc,
2826 const ConstructQueue &queue,
2827 ConstructQueue::const_iterator item) {
2828 mlir::omp::SimdOperands simdClauseOps;
2829 llvm::SmallVector<const semantics::Symbol *> simdReductionSyms;
2830 genSimdClauses(converter, semaCtx, item->clauses, loc, simdClauseOps,
2831 simdReductionSyms);
2832
2833 DataSharingProcessor dsp(converter, semaCtx, item->clauses, eval,
2834 /*shouldCollectPreDeterminedSymbols=*/true,
2835 enableDelayedPrivatization, symTable);
2836 dsp.processStep1(clauseOps: &simdClauseOps);
2837
2838 mlir::omp::LoopNestOperands loopNestClauseOps;
2839 llvm::SmallVector<const semantics::Symbol *> iv;
2840 genLoopNestClauses(converter, semaCtx, eval, item->clauses, loc,
2841 loopNestClauseOps, iv);
2842
2843 EntryBlockArgs simdArgs;
2844 simdArgs.priv.syms = dsp.getDelayedPrivSymbols();
2845 simdArgs.priv.vars = simdClauseOps.privateVars;
2846 simdArgs.reduction.syms = simdReductionSyms;
2847 simdArgs.reduction.vars = simdClauseOps.reductionVars;
2848 auto simdOp =
2849 genWrapperOp<mlir::omp::SimdOp>(converter, loc, simdClauseOps, simdArgs);
2850
2851 genLoopNestOp(converter, symTable, semaCtx, eval, loc, queue, item,
2852 loopNestClauseOps, iv, {{simdOp, simdArgs}},
2853 llvm::omp::Directive::OMPD_simd, dsp);
2854 return simdOp;
2855}
2856
2857static mlir::omp::TaskloopOp genStandaloneTaskloop(
2858 lower::AbstractConverter &converter, lower::SymMap &symTable,
2859 lower::StatementContext &stmtCtx, semantics::SemanticsContext &semaCtx,
2860 lower::pft::Evaluation &eval, mlir::Location loc,
2861 const ConstructQueue &queue, ConstructQueue::const_iterator item) {
2862 mlir::omp::TaskloopOperands taskloopClauseOps;
2863 genTaskloopClauses(converter, semaCtx, stmtCtx, item->clauses, loc,
2864 taskloopClauseOps);
2865 DataSharingProcessor dsp(converter, semaCtx, item->clauses, eval,
2866 /*shouldCollectPreDeterminedSymbols=*/true,
2867 enableDelayedPrivatization, symTable);
2868 dsp.processStep1(clauseOps: &taskloopClauseOps);
2869
2870 mlir::omp::LoopNestOperands loopNestClauseOps;
2871 llvm::SmallVector<const semantics::Symbol *> iv;
2872 genLoopNestClauses(converter, semaCtx, eval, item->clauses, loc,
2873 loopNestClauseOps, iv);
2874
2875 EntryBlockArgs taskloopArgs;
2876 taskloopArgs.priv.syms = dsp.getDelayedPrivSymbols();
2877 taskloopArgs.priv.vars = taskloopClauseOps.privateVars;
2878
2879 auto taskLoopOp = genWrapperOp<mlir::omp::TaskloopOp>(
2880 converter, loc, taskloopClauseOps, taskloopArgs);
2881
2882 genLoopNestOp(converter, symTable, semaCtx, eval, loc, queue, item,
2883 loopNestClauseOps, iv, {{taskLoopOp, taskloopArgs}},
2884 llvm::omp::Directive::OMPD_taskloop, dsp);
2885 return taskLoopOp;
2886}
2887
2888//===----------------------------------------------------------------------===//
2889// Code generation functions for composite constructs
2890//===----------------------------------------------------------------------===//
2891
2892static mlir::omp::DistributeOp genCompositeDistributeParallelDo(
2893 lower::AbstractConverter &converter, lower::SymMap &symTable,
2894 lower::StatementContext &stmtCtx, semantics::SemanticsContext &semaCtx,
2895 lower::pft::Evaluation &eval, mlir::Location loc,
2896 const ConstructQueue &queue, ConstructQueue::const_iterator item) {
2897 assert(std::distance(item, queue.end()) == 3 && "Invalid leaf constructs");
2898 ConstructQueue::const_iterator distributeItem = item;
2899 ConstructQueue::const_iterator parallelItem = std::next(distributeItem);
2900 ConstructQueue::const_iterator doItem = std::next(parallelItem);
2901
2902 // Create parent omp.parallel first.
2903 mlir::omp::ParallelOperands parallelClauseOps;
2904 llvm::SmallVector<const semantics::Symbol *> parallelReductionSyms;
2905 genParallelClauses(converter, semaCtx, stmtCtx, parallelItem->clauses, loc,
2906 parallelClauseOps, parallelReductionSyms);
2907
2908 DataSharingProcessor dsp(converter, semaCtx, doItem->clauses, eval,
2909 /*shouldCollectPreDeterminedSymbols=*/true,
2910 /*useDelayedPrivatization=*/true, symTable);
2911 dsp.processStep1(clauseOps: &parallelClauseOps);
2912
2913 EntryBlockArgs parallelArgs;
2914 parallelArgs.priv.syms = dsp.getDelayedPrivSymbols();
2915 parallelArgs.priv.vars = parallelClauseOps.privateVars;
2916 parallelArgs.reduction.syms = parallelReductionSyms;
2917 parallelArgs.reduction.vars = parallelClauseOps.reductionVars;
2918 genParallelOp(converter, symTable, semaCtx, eval, loc, queue, parallelItem,
2919 parallelClauseOps, parallelArgs, &dsp, /*isComposite=*/true);
2920
2921 // Clause processing.
2922 mlir::omp::DistributeOperands distributeClauseOps;
2923 genDistributeClauses(converter, semaCtx, stmtCtx, distributeItem->clauses,
2924 loc, distributeClauseOps);
2925
2926 mlir::omp::WsloopOperands wsloopClauseOps;
2927 llvm::SmallVector<const semantics::Symbol *> wsloopReductionSyms;
2928 genWsloopClauses(converter, semaCtx, stmtCtx, doItem->clauses, loc,
2929 wsloopClauseOps, wsloopReductionSyms);
2930
2931 mlir::omp::LoopNestOperands loopNestClauseOps;
2932 llvm::SmallVector<const semantics::Symbol *> iv;
2933 genLoopNestClauses(converter, semaCtx, eval, doItem->clauses, loc,
2934 loopNestClauseOps, iv);
2935
2936 // Operation creation.
2937 EntryBlockArgs distributeArgs;
2938 // TODO: Add private syms and vars.
2939 auto distributeOp = genWrapperOp<mlir::omp::DistributeOp>(
2940 converter, loc, distributeClauseOps, distributeArgs);
2941 distributeOp.setComposite(/*val=*/true);
2942
2943 EntryBlockArgs wsloopArgs;
2944 // TODO: Add private syms and vars.
2945 wsloopArgs.reduction.syms = wsloopReductionSyms;
2946 wsloopArgs.reduction.vars = wsloopClauseOps.reductionVars;
2947 auto wsloopOp = genWrapperOp<mlir::omp::WsloopOp>(
2948 converter, loc, wsloopClauseOps, wsloopArgs);
2949 wsloopOp.setComposite(/*val=*/true);
2950
2951 genLoopNestOp(converter, symTable, semaCtx, eval, loc, queue, doItem,
2952 loopNestClauseOps, iv,
2953 {{distributeOp, distributeArgs}, {wsloopOp, wsloopArgs}},
2954 llvm::omp::Directive::OMPD_distribute_parallel_do, dsp);
2955 return distributeOp;
2956}
2957
2958static mlir::omp::DistributeOp genCompositeDistributeParallelDoSimd(
2959 lower::AbstractConverter &converter, lower::SymMap &symTable,
2960 lower::StatementContext &stmtCtx, semantics::SemanticsContext &semaCtx,
2961 lower::pft::Evaluation &eval, mlir::Location loc,
2962 const ConstructQueue &queue, ConstructQueue::const_iterator item) {
2963 assert(std::distance(item, queue.end()) == 4 && "Invalid leaf constructs");
2964 ConstructQueue::const_iterator distributeItem = item;
2965 ConstructQueue::const_iterator parallelItem = std::next(distributeItem);
2966 ConstructQueue::const_iterator doItem = std::next(parallelItem);
2967 ConstructQueue::const_iterator simdItem = std::next(doItem);
2968
2969 // Create parent omp.parallel first.
2970 mlir::omp::ParallelOperands parallelClauseOps;
2971 llvm::SmallVector<const semantics::Symbol *> parallelReductionSyms;
2972 genParallelClauses(converter, semaCtx, stmtCtx, parallelItem->clauses, loc,
2973 parallelClauseOps, parallelReductionSyms);
2974
2975 DataSharingProcessor parallelItemDSP(
2976 converter, semaCtx, parallelItem->clauses, eval,
2977 /*shouldCollectPreDeterminedSymbols=*/false,
2978 /*useDelayedPrivatization=*/true, symTable);
2979 parallelItemDSP.processStep1(clauseOps: &parallelClauseOps);
2980
2981 EntryBlockArgs parallelArgs;
2982 parallelArgs.priv.syms = parallelItemDSP.getDelayedPrivSymbols();
2983 parallelArgs.priv.vars = parallelClauseOps.privateVars;
2984 parallelArgs.reduction.syms = parallelReductionSyms;
2985 parallelArgs.reduction.vars = parallelClauseOps.reductionVars;
2986 genParallelOp(converter, symTable, semaCtx, eval, loc, queue, parallelItem,
2987 parallelClauseOps, parallelArgs, &parallelItemDSP,
2988 /*isComposite=*/true);
2989
2990 // Clause processing.
2991 mlir::omp::DistributeOperands distributeClauseOps;
2992 genDistributeClauses(converter, semaCtx, stmtCtx, distributeItem->clauses,
2993 loc, distributeClauseOps);
2994
2995 mlir::omp::WsloopOperands wsloopClauseOps;
2996 llvm::SmallVector<const semantics::Symbol *> wsloopReductionSyms;
2997 genWsloopClauses(converter, semaCtx, stmtCtx, doItem->clauses, loc,
2998 wsloopClauseOps, wsloopReductionSyms);
2999
3000 mlir::omp::SimdOperands simdClauseOps;
3001 llvm::SmallVector<const semantics::Symbol *> simdReductionSyms;
3002 genSimdClauses(converter, semaCtx, simdItem->clauses, loc, simdClauseOps,
3003 simdReductionSyms);
3004
3005 DataSharingProcessor simdItemDSP(converter, semaCtx, simdItem->clauses, eval,
3006 /*shouldCollectPreDeterminedSymbols=*/true,
3007 /*useDelayedPrivatization=*/true, symTable);
3008 simdItemDSP.processStep1(clauseOps: &simdClauseOps);
3009
3010 mlir::omp::LoopNestOperands loopNestClauseOps;
3011 llvm::SmallVector<const semantics::Symbol *> iv;
3012 genLoopNestClauses(converter, semaCtx, eval, simdItem->clauses, loc,
3013 loopNestClauseOps, iv);
3014
3015 // Operation creation.
3016 EntryBlockArgs distributeArgs;
3017 // TODO: Add private syms and vars.
3018 auto distributeOp = genWrapperOp<mlir::omp::DistributeOp>(
3019 converter, loc, distributeClauseOps, distributeArgs);
3020 distributeOp.setComposite(/*val=*/true);
3021
3022 EntryBlockArgs wsloopArgs;
3023 // TODO: Add private syms and vars.
3024 wsloopArgs.reduction.syms = wsloopReductionSyms;
3025 wsloopArgs.reduction.vars = wsloopClauseOps.reductionVars;
3026 auto wsloopOp = genWrapperOp<mlir::omp::WsloopOp>(
3027 converter, loc, wsloopClauseOps, wsloopArgs);
3028 wsloopOp.setComposite(/*val=*/true);
3029
3030 EntryBlockArgs simdArgs;
3031 simdArgs.priv.syms = simdItemDSP.getDelayedPrivSymbols();
3032 simdArgs.priv.vars = simdClauseOps.privateVars;
3033 simdArgs.reduction.syms = simdReductionSyms;
3034 simdArgs.reduction.vars = simdClauseOps.reductionVars;
3035 auto simdOp =
3036 genWrapperOp<mlir::omp::SimdOp>(converter, loc, simdClauseOps, simdArgs);
3037 simdOp.setComposite(/*val=*/true);
3038
3039 genLoopNestOp(converter, symTable, semaCtx, eval, loc, queue, simdItem,
3040 loopNestClauseOps, iv,
3041 {{distributeOp, distributeArgs},
3042 {wsloopOp, wsloopArgs},
3043 {simdOp, simdArgs}},
3044 llvm::omp::Directive::OMPD_distribute_parallel_do_simd,
3045 simdItemDSP);
3046 return distributeOp;
3047}
3048
3049static mlir::omp::DistributeOp genCompositeDistributeSimd(
3050 lower::AbstractConverter &converter, lower::SymMap &symTable,
3051 lower::StatementContext &stmtCtx, semantics::SemanticsContext &semaCtx,
3052 lower::pft::Evaluation &eval, mlir::Location loc,
3053 const ConstructQueue &queue, ConstructQueue::const_iterator item) {
3054 assert(std::distance(item, queue.end()) == 2 && "Invalid leaf constructs");
3055 ConstructQueue::const_iterator distributeItem = item;
3056 ConstructQueue::const_iterator simdItem = std::next(distributeItem);
3057
3058 // Clause processing.
3059 mlir::omp::DistributeOperands distributeClauseOps;
3060 genDistributeClauses(converter, semaCtx, stmtCtx, distributeItem->clauses,
3061 loc, distributeClauseOps);
3062
3063 mlir::omp::SimdOperands simdClauseOps;
3064 llvm::SmallVector<const semantics::Symbol *> simdReductionSyms;
3065 genSimdClauses(converter, semaCtx, simdItem->clauses, loc, simdClauseOps,
3066 simdReductionSyms);
3067
3068 // TODO: Support delayed privatization.
3069 DataSharingProcessor dsp(converter, semaCtx, simdItem->clauses, eval,
3070 /*shouldCollectPreDeterminedSymbols=*/true,
3071 /*useDelayedPrivatization=*/false, symTable);
3072 dsp.processStep1();
3073
3074 // Pass the innermost leaf construct's clauses because that's where COLLAPSE
3075 // is placed by construct decomposition.
3076 mlir::omp::LoopNestOperands loopNestClauseOps;
3077 llvm::SmallVector<const semantics::Symbol *> iv;
3078 genLoopNestClauses(converter, semaCtx, eval, simdItem->clauses, loc,
3079 loopNestClauseOps, iv);
3080
3081 // Operation creation.
3082 EntryBlockArgs distributeArgs;
3083 // TODO: Add private syms and vars.
3084 auto distributeOp = genWrapperOp<mlir::omp::DistributeOp>(
3085 converter, loc, distributeClauseOps, distributeArgs);
3086 distributeOp.setComposite(/*val=*/true);
3087
3088 EntryBlockArgs simdArgs;
3089 // TODO: Add private syms and vars.
3090 simdArgs.reduction.syms = simdReductionSyms;
3091 simdArgs.reduction.vars = simdClauseOps.reductionVars;
3092 auto simdOp =
3093 genWrapperOp<mlir::omp::SimdOp>(converter, loc, simdClauseOps, simdArgs);
3094 simdOp.setComposite(/*val=*/true);
3095
3096 genLoopNestOp(converter, symTable, semaCtx, eval, loc, queue, simdItem,
3097 loopNestClauseOps, iv,
3098 {{distributeOp, distributeArgs}, {simdOp, simdArgs}},
3099 llvm::omp::Directive::OMPD_distribute_simd, dsp);
3100 return distributeOp;
3101}
3102
3103static mlir::omp::WsloopOp genCompositeDoSimd(
3104 lower::AbstractConverter &converter, lower::SymMap &symTable,
3105 lower::StatementContext &stmtCtx, semantics::SemanticsContext &semaCtx,
3106 lower::pft::Evaluation &eval, mlir::Location loc,
3107 const ConstructQueue &queue, ConstructQueue::const_iterator item) {
3108 assert(std::distance(item, queue.end()) == 2 && "Invalid leaf constructs");
3109 ConstructQueue::const_iterator doItem = item;
3110 ConstructQueue::const_iterator simdItem = std::next(doItem);
3111
3112 // Clause processing.
3113 mlir::omp::WsloopOperands wsloopClauseOps;
3114 llvm::SmallVector<const semantics::Symbol *> wsloopReductionSyms;
3115 genWsloopClauses(converter, semaCtx, stmtCtx, doItem->clauses, loc,
3116 wsloopClauseOps, wsloopReductionSyms);
3117
3118 mlir::omp::SimdOperands simdClauseOps;
3119 llvm::SmallVector<const semantics::Symbol *> simdReductionSyms;
3120 genSimdClauses(converter, semaCtx, simdItem->clauses, loc, simdClauseOps,
3121 simdReductionSyms);
3122
3123 // TODO: Support delayed privatization.
3124 DataSharingProcessor dsp(converter, semaCtx, simdItem->clauses, eval,
3125 /*shouldCollectPreDeterminedSymbols=*/true,
3126 /*useDelayedPrivatization=*/false, symTable);
3127 dsp.processStep1();
3128
3129 // Pass the innermost leaf construct's clauses because that's where COLLAPSE
3130 // is placed by construct decomposition.
3131 mlir::omp::LoopNestOperands loopNestClauseOps;
3132 llvm::SmallVector<const semantics::Symbol *> iv;
3133 genLoopNestClauses(converter, semaCtx, eval, simdItem->clauses, loc,
3134 loopNestClauseOps, iv);
3135
3136 // Operation creation.
3137 EntryBlockArgs wsloopArgs;
3138 // TODO: Add private syms and vars.
3139 wsloopArgs.reduction.syms = wsloopReductionSyms;
3140 wsloopArgs.reduction.vars = wsloopClauseOps.reductionVars;
3141 auto wsloopOp = genWrapperOp<mlir::omp::WsloopOp>(
3142 converter, loc, wsloopClauseOps, wsloopArgs);
3143 wsloopOp.setComposite(/*val=*/true);
3144
3145 EntryBlockArgs simdArgs;
3146 // TODO: Add private syms and vars.
3147 simdArgs.reduction.syms = simdReductionSyms;
3148 simdArgs.reduction.vars = simdClauseOps.reductionVars;
3149 auto simdOp =
3150 genWrapperOp<mlir::omp::SimdOp>(converter, loc, simdClauseOps, simdArgs);
3151 simdOp.setComposite(/*val=*/true);
3152
3153 genLoopNestOp(converter, symTable, semaCtx, eval, loc, queue, simdItem,
3154 loopNestClauseOps, iv,
3155 {{wsloopOp, wsloopArgs}, {simdOp, simdArgs}},
3156 llvm::omp::Directive::OMPD_do_simd, dsp);
3157 return wsloopOp;
3158}
3159
3160static mlir::omp::TaskloopOp genCompositeTaskloopSimd(
3161 lower::AbstractConverter &converter, lower::SymMap &symTable,
3162 lower::StatementContext &stmtCtx, semantics::SemanticsContext &semaCtx,
3163 lower::pft::Evaluation &eval, mlir::Location loc,
3164 const ConstructQueue &queue, ConstructQueue::const_iterator item) {
3165 assert(std::distance(item, queue.end()) == 2 && "Invalid leaf constructs");
3166 TODO(loc, "Composite TASKLOOP SIMD");
3167 return nullptr;
3168}
3169
3170//===----------------------------------------------------------------------===//
3171// Dispatch
3172//===----------------------------------------------------------------------===//
3173
3174static bool genOMPCompositeDispatch(
3175 lower::AbstractConverter &converter, lower::SymMap &symTable,
3176 lower::StatementContext &stmtCtx, semantics::SemanticsContext &semaCtx,
3177 lower::pft::Evaluation &eval, mlir::Location loc,
3178 const ConstructQueue &queue, ConstructQueue::const_iterator item,
3179 mlir::Operation *&newOp) {
3180 using llvm::omp::Directive;
3181 using lower::omp::matchLeafSequence;
3182
3183 // TODO: Privatization for composite constructs is currently only done based
3184 // on the clauses for their last leaf construct, which may not always be
3185 // correct. Consider per-leaf privatization of composite constructs once
3186 // delayed privatization is supported by all participating ops.
3187 if (matchLeafSequence(item, queue, Directive::OMPD_distribute_parallel_do))
3188 newOp = genCompositeDistributeParallelDo(converter, symTable, stmtCtx,
3189 semaCtx, eval, loc, queue, item);
3190 else if (matchLeafSequence(item, queue,
3191 Directive::OMPD_distribute_parallel_do_simd))
3192 newOp = genCompositeDistributeParallelDoSimd(
3193 converter, symTable, stmtCtx, semaCtx, eval, loc, queue, item);
3194 else if (matchLeafSequence(item, queue, Directive::OMPD_distribute_simd))
3195 newOp = genCompositeDistributeSimd(converter, symTable, stmtCtx, semaCtx,
3196 eval, loc, queue, item);
3197 else if (matchLeafSequence(item, queue, Directive::OMPD_do_simd))
3198 newOp = genCompositeDoSimd(converter, symTable, stmtCtx, semaCtx, eval, loc,
3199 queue, item);
3200 else if (matchLeafSequence(item, queue, Directive::OMPD_taskloop_simd))
3201 newOp = genCompositeTaskloopSimd(converter, symTable, stmtCtx, semaCtx,
3202 eval, loc, queue, item);
3203 else
3204 return false;
3205
3206 return true;
3207}
3208
3209static void genOMPDispatch(lower::AbstractConverter &converter,
3210 lower::SymMap &symTable,
3211 semantics::SemanticsContext &semaCtx,
3212 lower::pft::Evaluation &eval, mlir::Location loc,
3213 const ConstructQueue &queue,
3214 ConstructQueue::const_iterator item) {
3215 assert(item != queue.end());
3216
3217 lower::StatementContext stmtCtx;
3218 mlir::Operation *newOp = nullptr;
3219
3220 // Generate cleanup code for the stmtCtx after newOp
3221 auto finalizeStmtCtx = [&]() {
3222 if (newOp) {
3223 fir::FirOpBuilder &builder = converter.getFirOpBuilder();
3224 fir::FirOpBuilder::InsertionGuard guard(builder);
3225 builder.setInsertionPointAfter(newOp);
3226 stmtCtx.finalizeAndPop();
3227 }
3228 };
3229
3230 bool loopLeaf = llvm::omp::getDirectiveAssociation(D: item->id) ==
3231 llvm::omp::Association::Loop;
3232 if (loopLeaf) {
3233 symTable.pushScope();
3234 if (genOMPCompositeDispatch(converter, symTable, stmtCtx, semaCtx, eval,
3235 loc, queue, item, newOp)) {
3236 symTable.popScope();
3237 finalizeStmtCtx();
3238 return;
3239 }
3240 }
3241
3242 switch (llvm::omp::Directive dir = item->id) {
3243 case llvm::omp::Directive::OMPD_barrier:
3244 newOp = genBarrierOp(converter, symTable, semaCtx, eval, loc, queue, item);
3245 break;
3246 case llvm::omp::Directive::OMPD_distribute:
3247 newOp = genStandaloneDistribute(converter, symTable, stmtCtx, semaCtx, eval,
3248 loc, queue, item);
3249 break;
3250 case llvm::omp::Directive::OMPD_do:
3251 newOp = genStandaloneDo(converter, symTable, stmtCtx, semaCtx, eval, loc,
3252 queue, item);
3253 break;
3254 case llvm::omp::Directive::OMPD_loop:
3255 newOp = genLoopOp(converter, symTable, semaCtx, eval, loc, queue, item);
3256 break;
3257 case llvm::omp::Directive::OMPD_masked:
3258 newOp = genMaskedOp(converter, symTable, stmtCtx, semaCtx, eval, loc, queue,
3259 item);
3260 break;
3261 case llvm::omp::Directive::OMPD_master:
3262 newOp = genMasterOp(converter, symTable, semaCtx, eval, loc, queue, item);
3263 break;
3264 case llvm::omp::Directive::OMPD_ordered:
3265 // Block-associated "ordered" construct.
3266 newOp = genOrderedRegionOp(converter, symTable, semaCtx, eval, loc, queue,
3267 item);
3268 break;
3269 case llvm::omp::Directive::OMPD_parallel:
3270 newOp = genStandaloneParallel(converter, symTable, stmtCtx, semaCtx, eval,
3271 loc, queue, item);
3272 break;
3273 case llvm::omp::Directive::OMPD_scan:
3274 newOp = genScanOp(converter, symTable, semaCtx, loc, queue, item);
3275 break;
3276 case llvm::omp::Directive::OMPD_section:
3277 llvm_unreachable("genOMPDispatch: OMPD_section");
3278 // Lowered in the enclosing genSectionsOp.
3279 break;
3280 case llvm::omp::Directive::OMPD_sections:
3281 genSectionsOp(converter, symTable, semaCtx, eval, loc, queue, item);
3282 break;
3283 case llvm::omp::Directive::OMPD_simd:
3284 newOp =
3285 genStandaloneSimd(converter, symTable, semaCtx, eval, loc, queue, item);
3286 break;
3287 case llvm::omp::Directive::OMPD_scope:
3288 newOp = genScopeOp(converter, symTable, semaCtx, eval, loc, queue, item);
3289 break;
3290 case llvm::omp::Directive::OMPD_single:
3291 newOp = genSingleOp(converter, symTable, semaCtx, eval, loc, queue, item);
3292 break;
3293 case llvm::omp::Directive::OMPD_target:
3294 newOp = genTargetOp(converter, symTable, stmtCtx, semaCtx, eval, loc, queue,
3295 item);
3296 break;
3297 case llvm::omp::Directive::OMPD_target_data:
3298 newOp = genTargetDataOp(converter, symTable, stmtCtx, semaCtx, eval, loc,
3299 queue, item);
3300 break;
3301 case llvm::omp::Directive::OMPD_target_enter_data:
3302 newOp = genTargetEnterExitUpdateDataOp<mlir::omp::TargetEnterDataOp>(
3303 converter, symTable, stmtCtx, semaCtx, loc, queue, item);
3304 break;
3305 case llvm::omp::Directive::OMPD_target_exit_data:
3306 newOp = genTargetEnterExitUpdateDataOp<mlir::omp::TargetExitDataOp>(
3307 converter, symTable, stmtCtx, semaCtx, loc, queue, item);
3308 break;
3309 case llvm::omp::Directive::OMPD_target_update:
3310 newOp = genTargetEnterExitUpdateDataOp<mlir::omp::TargetUpdateOp>(
3311 converter, symTable, stmtCtx, semaCtx, loc, queue, item);
3312 break;
3313 case llvm::omp::Directive::OMPD_task:
3314 newOp = genTaskOp(converter, symTable, stmtCtx, semaCtx, eval, loc, queue,
3315 item);
3316 break;
3317 case llvm::omp::Directive::OMPD_taskgroup:
3318 newOp =
3319 genTaskgroupOp(converter, symTable, semaCtx, eval, loc, queue, item);
3320 break;
3321 case llvm::omp::Directive::OMPD_taskloop:
3322 newOp = genStandaloneTaskloop(converter, symTable, stmtCtx, semaCtx, eval,
3323 loc, queue, item);
3324 break;
3325 case llvm::omp::Directive::OMPD_taskwait:
3326 newOp = genTaskwaitOp(converter, symTable, semaCtx, eval, loc, queue, item);
3327 break;
3328 case llvm::omp::Directive::OMPD_taskyield:
3329 newOp =
3330 genTaskyieldOp(converter, symTable, semaCtx, eval, loc, queue, item);
3331 break;
3332 case llvm::omp::Directive::OMPD_teams:
3333 newOp = genTeamsOp(converter, symTable, stmtCtx, semaCtx, eval, loc, queue,
3334 item);
3335 break;
3336 case llvm::omp::Directive::OMPD_tile:
3337 case llvm::omp::Directive::OMPD_unroll: {
3338 unsigned version = semaCtx.langOptions().OpenMPVersion;
3339 TODO(loc, "Unhandled loop directive (" +
3340 llvm::omp::getOpenMPDirectiveName(D: dir, Ver: version) + ")");
3341 }
3342 // case llvm::omp::Directive::OMPD_workdistribute:
3343 case llvm::omp::Directive::OMPD_workshare:
3344 newOp = genWorkshareOp(converter, symTable, stmtCtx, semaCtx, eval, loc,
3345 queue, item);
3346 break;
3347 default:
3348 // Combined and composite constructs should have been split into a sequence
3349 // of leaf constructs when building the construct queue.
3350 assert(!llvm::omp::isLeafConstruct(dir) &&
3351 "Unexpected compound construct.");
3352 break;
3353 }
3354
3355 finalizeStmtCtx();
3356 if (loopLeaf)
3357 symTable.popScope();
3358}
3359
3360//===----------------------------------------------------------------------===//
3361// OpenMPDeclarativeConstruct visitors
3362//===----------------------------------------------------------------------===//
3363static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
3364 semantics::SemanticsContext &semaCtx,
3365 lower::pft::Evaluation &eval,
3366 const parser::OpenMPUtilityConstruct &);
3367
3368static void
3369genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
3370 semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
3371 const parser::OpenMPDeclarativeAllocate &declarativeAllocate) {
3372 TODO(converter.getCurrentLocation(), "OpenMPDeclarativeAllocate");
3373}
3374
3375static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
3376 semantics::SemanticsContext &semaCtx,
3377 lower::pft::Evaluation &eval,
3378 const parser::OpenMPDeclarativeAssumes &assumesConstruct) {
3379 TODO(converter.getCurrentLocation(), "OpenMP ASSUMES declaration");
3380}
3381
3382static void
3383genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
3384 semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
3385 const parser::OmpDeclareVariantDirective &declareVariantDirective) {
3386 TODO(converter.getCurrentLocation(), "OmpDeclareVariantDirective");
3387}
3388
3389static void genOMP(
3390 lower::AbstractConverter &converter, lower::SymMap &symTable,
3391 semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
3392 const parser::OpenMPDeclareReductionConstruct &declareReductionConstruct) {
3393 TODO(converter.getCurrentLocation(), "OpenMPDeclareReductionConstruct");
3394}
3395
3396static void
3397genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
3398 semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
3399 const parser::OpenMPDeclareSimdConstruct &declareSimdConstruct) {
3400 TODO(converter.getCurrentLocation(), "OpenMPDeclareSimdConstruct");
3401}
3402
3403static void
3404genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
3405 semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
3406 const parser::OpenMPDeclareMapperConstruct &declareMapperConstruct) {
3407 mlir::Location loc = converter.genLocation(declareMapperConstruct.source);
3408 fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
3409 lower::StatementContext stmtCtx;
3410 const auto &spec =
3411 std::get<parser::OmpMapperSpecifier>(declareMapperConstruct.t);
3412 const auto &mapperName{std::get<std::string>(spec.t)};
3413 const auto &varType{std::get<parser::TypeSpec>(spec.t)};
3414 const auto &varName{std::get<parser::Name>(spec.t)};
3415 assert(varType.declTypeSpec->category() ==
3416 semantics::DeclTypeSpec::Category::TypeDerived &&
3417 "Expected derived type");
3418
3419 std::string mapperNameStr = mapperName;
3420 if (auto *sym = converter.getCurrentScope().FindSymbol(mapperNameStr))
3421 mapperNameStr = converter.mangleName(mapperNameStr, sym->owner());
3422
3423 // Save current insertion point before moving to the module scope to create
3424 // the DeclareMapperOp
3425 mlir::OpBuilder::InsertionGuard guard(firOpBuilder);
3426
3427 firOpBuilder.setInsertionPointToStart(converter.getModuleOp().getBody());
3428 auto mlirType = converter.genType(varType.declTypeSpec->derivedTypeSpec());
3429 auto declMapperOp = firOpBuilder.create<mlir::omp::DeclareMapperOp>(
3430 loc, mapperNameStr, mlirType);
3431 auto &region = declMapperOp.getRegion();
3432 firOpBuilder.createBlock(&region);
3433 auto varVal = region.addArgument(firOpBuilder.getRefType(mlirType), loc);
3434 converter.bindSymbol(*varName.symbol, varVal);
3435
3436 // Populate the declareMapper region with the map information.
3437 mlir::omp::DeclareMapperInfoOperands clauseOps;
3438 const auto *clauseList{
3439 parser::Unwrap<parser::OmpClauseList>(declareMapperConstruct.t)};
3440 List<Clause> clauses = makeClauses(*clauseList, semaCtx);
3441 ClauseProcessor cp(converter, semaCtx, clauses);
3442 cp.processMap(loc, stmtCtx, clauseOps);
3443 firOpBuilder.create<mlir::omp::DeclareMapperInfoOp>(loc, clauseOps.mapVars);
3444}
3445
3446static void
3447genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
3448 semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
3449 const parser::OpenMPDeclareTargetConstruct &declareTargetConstruct) {
3450 mlir::omp::DeclareTargetOperands clauseOps;
3451 llvm::SmallVector<DeclareTargetCapturePair> symbolAndClause;
3452 mlir::ModuleOp mod = converter.getFirOpBuilder().getModule();
3453 getDeclareTargetInfo(converter, semaCtx, eval, declareTargetConstruct,
3454 clauseOps, symbolAndClause);
3455
3456 for (const DeclareTargetCapturePair &symClause : symbolAndClause) {
3457 mlir::Operation *op = mod.lookupSymbol(
3458 converter.mangleName(std::get<const semantics::Symbol &>(symClause)));
3459
3460 // Some symbols are deferred until later in the module, these are handled
3461 // upon finalization of the module for OpenMP inside of Bridge, so we simply
3462 // skip for now.
3463 if (!op)
3464 continue;
3465
3466 markDeclareTarget(
3467 op, converter,
3468 std::get<mlir::omp::DeclareTargetCaptureClause>(symClause),
3469 clauseOps.deviceType);
3470 }
3471}
3472
3473static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
3474 semantics::SemanticsContext &semaCtx,
3475 lower::pft::Evaluation &eval,
3476 const parser::OpenMPRequiresConstruct &requiresConstruct) {
3477 // Requires directives are gathered and processed in semantics and
3478 // then combined in the lowering bridge before triggering codegen
3479 // just once. Hence, there is no need to lower each individual
3480 // occurrence here.
3481}
3482
3483static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
3484 semantics::SemanticsContext &semaCtx,
3485 lower::pft::Evaluation &eval,
3486 const parser::OpenMPThreadprivate &threadprivate) {
3487 // The directive is lowered when instantiating the variable to
3488 // support the case of threadprivate variable declared in module.
3489}
3490
3491static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
3492 semantics::SemanticsContext &semaCtx,
3493 lower::pft::Evaluation &eval,
3494 const parser::OmpMetadirectiveDirective &meta) {
3495 TODO(converter.getCurrentLocation(), "METADIRECTIVE");
3496}
3497
3498static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
3499 semantics::SemanticsContext &semaCtx,
3500 lower::pft::Evaluation &eval,
3501 const parser::OpenMPDeclarativeConstruct &ompDeclConstruct) {
3502 Fortran::common::visit(
3503 [&](auto &&s) { return genOMP(converter, symTable, semaCtx, eval, s); },
3504 ompDeclConstruct.u);
3505}
3506
3507//===----------------------------------------------------------------------===//
3508// OpenMPStandaloneConstruct visitors
3509//===----------------------------------------------------------------------===//
3510
3511static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
3512 semantics::SemanticsContext &semaCtx,
3513 lower::pft::Evaluation &eval,
3514 const parser::OpenMPSimpleStandaloneConstruct &construct) {
3515 const auto &directive = std::get<parser::OmpDirectiveName>(construct.v.t);
3516 List<Clause> clauses = makeClauses(construct.v.Clauses(), semaCtx);
3517 mlir::Location currentLocation = converter.genLocation(directive.source);
3518
3519 ConstructQueue queue{
3520 buildConstructQueue(converter.getFirOpBuilder().getModule(), semaCtx,
3521 eval, directive.source, directive.v, clauses)};
3522 if (directive.v == llvm::omp::Directive::OMPD_ordered) {
3523 // Standalone "ordered" directive.
3524 genOrderedOp(converter, symTable, semaCtx, eval, currentLocation, queue,
3525 queue.begin());
3526 } else {
3527 // Dispatch handles the "block-associated" variant of "ordered".
3528 genOMPDispatch(converter, symTable, semaCtx, eval, currentLocation, queue,
3529 queue.begin());
3530 }
3531}
3532
3533static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
3534 semantics::SemanticsContext &semaCtx,
3535 lower::pft::Evaluation &eval,
3536 const parser::OpenMPFlushConstruct &construct) {
3537 const auto &argumentList = construct.v.Arguments();
3538 const auto &clauseList = construct.v.Clauses();
3539 ObjectList objects = makeObjects(argumentList, semaCtx);
3540 List<Clause> clauses =
3541 makeList(clauseList.v, [&](auto &&s) { return makeClause(s, semaCtx); });
3542 mlir::Location currentLocation = converter.genLocation(construct.source);
3543
3544 ConstructQueue queue{buildConstructQueue(
3545 converter.getFirOpBuilder().getModule(), semaCtx, eval, construct.source,
3546 llvm::omp::Directive::OMPD_flush, clauses)};
3547 genFlushOp(converter, symTable, semaCtx, eval, currentLocation, objects,
3548 queue, queue.begin());
3549}
3550
3551static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
3552 semantics::SemanticsContext &semaCtx,
3553 lower::pft::Evaluation &eval,
3554 const parser::OpenMPCancelConstruct &cancelConstruct) {
3555 List<Clause> clauses = makeList(cancelConstruct.v.Clauses().v, [&](auto &&s) {
3556 return makeClause(s, semaCtx);
3557 });
3558 mlir::Location loc = converter.genLocation(cancelConstruct.source);
3559
3560 ConstructQueue queue{buildConstructQueue(
3561 converter.getFirOpBuilder().getModule(), semaCtx, eval,
3562 cancelConstruct.source, llvm::omp::Directive::OMPD_cancel, clauses)};
3563 genCancelOp(converter, semaCtx, eval, loc, queue, queue.begin());
3564}
3565
3566static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
3567 semantics::SemanticsContext &semaCtx,
3568 lower::pft::Evaluation &eval,
3569 const parser::OpenMPCancellationPointConstruct
3570 &cancellationPointConstruct) {
3571 List<Clause> clauses =
3572 makeList(cancellationPointConstruct.v.Clauses().v,
3573 [&](auto &&s) { return makeClause(s, semaCtx); });
3574 mlir::Location loc = converter.genLocation(cancellationPointConstruct.source);
3575
3576 ConstructQueue queue{
3577 buildConstructQueue(converter.getFirOpBuilder().getModule(), semaCtx,
3578 eval, cancellationPointConstruct.source,
3579 llvm::omp::Directive::OMPD_cancel, clauses)};
3580 genCancellationPointOp(converter, semaCtx, eval, loc, queue, queue.begin());
3581}
3582
3583static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
3584 semantics::SemanticsContext &semaCtx,
3585 lower::pft::Evaluation &eval,
3586 const parser::OpenMPDepobjConstruct &construct) {
3587 // These values will be ignored until the construct itself is implemented,
3588 // but run them anyway for the sake of testing (via a Todo test).
3589 ObjectList objects = makeObjects(construct.v.Arguments(), semaCtx);
3590 assert(objects.size() == 1);
3591 List<Clause> clauses = makeClauses(construct.v.Clauses(), semaCtx);
3592 assert(clauses.size() == 1);
3593 (void)objects;
3594 (void)clauses;
3595
3596 TODO(converter.getCurrentLocation(), "OpenMPDepobjConstruct");
3597}
3598
3599static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
3600 semantics::SemanticsContext &semaCtx,
3601 lower::pft::Evaluation &eval,
3602 const parser::OpenMPInteropConstruct &interopConstruct) {
3603 TODO(converter.getCurrentLocation(), "OpenMPInteropConstruct");
3604}
3605
3606static void
3607genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
3608 semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
3609 const parser::OpenMPStandaloneConstruct &standaloneConstruct) {
3610 Fortran::common::visit(
3611 [&](auto &&s) { return genOMP(converter, symTable, semaCtx, eval, s); },
3612 standaloneConstruct.u);
3613}
3614
3615static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
3616 semantics::SemanticsContext &semaCtx,
3617 lower::pft::Evaluation &eval,
3618 const parser::OpenMPAllocatorsConstruct &allocsConstruct) {
3619 TODO(converter.getCurrentLocation(), "OpenMPAllocatorsConstruct");
3620}
3621
3622//===----------------------------------------------------------------------===//
3623// OpenMPConstruct visitors
3624//===----------------------------------------------------------------------===//
3625
3626static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
3627 semantics::SemanticsContext &semaCtx,
3628 lower::pft::Evaluation &eval,
3629 const parser::OpenMPAtomicConstruct &construct) {
3630 lowerAtomic(converter, symTable, semaCtx, eval, construct);
3631}
3632
3633static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
3634 semantics::SemanticsContext &semaCtx,
3635 lower::pft::Evaluation &eval,
3636 const parser::OpenMPBlockConstruct &blockConstruct) {
3637 const auto &beginBlockDirective =
3638 std::get<parser::OmpBeginBlockDirective>(blockConstruct.t);
3639 const auto &endBlockDirective =
3640 std::get<parser::OmpEndBlockDirective>(blockConstruct.t);
3641 mlir::Location currentLocation =
3642 converter.genLocation(beginBlockDirective.source);
3643 const auto origDirective =
3644 std::get<parser::OmpBlockDirective>(beginBlockDirective.t).v;
3645 List<Clause> clauses = makeClauses(
3646 std::get<parser::OmpClauseList>(beginBlockDirective.t), semaCtx);
3647 clauses.append(makeClauses(
3648 std::get<parser::OmpClauseList>(endBlockDirective.t), semaCtx));
3649
3650 assert(llvm::omp::blockConstructSet.test(origDirective) &&
3651 "Expected block construct");
3652 (void)origDirective;
3653
3654 for (const Clause &clause : clauses) {
3655 mlir::Location clauseLocation = converter.genLocation(clause.source);
3656 if (!std::holds_alternative<clause::Affinity>(clause.u) &&
3657 !std::holds_alternative<clause::Allocate>(clause.u) &&
3658 !std::holds_alternative<clause::Copyin>(clause.u) &&
3659 !std::holds_alternative<clause::Copyprivate>(clause.u) &&
3660 !std::holds_alternative<clause::Default>(clause.u) &&
3661 !std::holds_alternative<clause::Defaultmap>(clause.u) &&
3662 !std::holds_alternative<clause::Depend>(clause.u) &&
3663 !std::holds_alternative<clause::Filter>(clause.u) &&
3664 !std::holds_alternative<clause::Final>(clause.u) &&
3665 !std::holds_alternative<clause::Firstprivate>(clause.u) &&
3666 !std::holds_alternative<clause::HasDeviceAddr>(clause.u) &&
3667 !std::holds_alternative<clause::If>(clause.u) &&
3668 !std::holds_alternative<clause::IsDevicePtr>(clause.u) &&
3669 !std::holds_alternative<clause::Map>(clause.u) &&
3670 !std::holds_alternative<clause::Nowait>(clause.u) &&
3671 !std::holds_alternative<clause::NumTeams>(clause.u) &&
3672 !std::holds_alternative<clause::NumThreads>(clause.u) &&
3673 !std::holds_alternative<clause::OmpxBare>(clause.u) &&
3674 !std::holds_alternative<clause::Priority>(clause.u) &&
3675 !std::holds_alternative<clause::Private>(clause.u) &&
3676 !std::holds_alternative<clause::ProcBind>(clause.u) &&
3677 !std::holds_alternative<clause::Reduction>(clause.u) &&
3678 !std::holds_alternative<clause::Shared>(clause.u) &&
3679 !std::holds_alternative<clause::Simd>(clause.u) &&
3680 !std::holds_alternative<clause::ThreadLimit>(clause.u) &&
3681 !std::holds_alternative<clause::Threads>(clause.u) &&
3682 !std::holds_alternative<clause::UseDeviceAddr>(clause.u) &&
3683 !std::holds_alternative<clause::UseDevicePtr>(clause.u) &&
3684 !std::holds_alternative<clause::InReduction>(clause.u) &&
3685 !std::holds_alternative<clause::Mergeable>(clause.u) &&
3686 !std::holds_alternative<clause::Untied>(clause.u) &&
3687 !std::holds_alternative<clause::TaskReduction>(clause.u) &&
3688 !std::holds_alternative<clause::Detach>(clause.u)) {
3689 std::string name =
3690 parser::ToUpperCaseLetters(llvm::omp::getOpenMPClauseName(clause.id));
3691 TODO(clauseLocation, name + " clause is not implemented yet");
3692 }
3693 }
3694
3695 llvm::omp::Directive directive =
3696 std::get<parser::OmpBlockDirective>(beginBlockDirective.t).v;
3697 const parser::CharBlock &source =
3698 std::get<parser::OmpBlockDirective>(beginBlockDirective.t).source;
3699 ConstructQueue queue{
3700 buildConstructQueue(converter.getFirOpBuilder().getModule(), semaCtx,
3701 eval, source, directive, clauses)};
3702 genOMPDispatch(converter, symTable, semaCtx, eval, currentLocation, queue,
3703 queue.begin());
3704}
3705
3706static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
3707 semantics::SemanticsContext &semaCtx,
3708 lower::pft::Evaluation &eval,
3709 const parser::OpenMPAssumeConstruct &assumeConstruct) {
3710 mlir::Location clauseLocation = converter.genLocation(assumeConstruct.source);
3711 TODO(clauseLocation, "OpenMP ASSUME construct");
3712}
3713
3714static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
3715 semantics::SemanticsContext &semaCtx,
3716 lower::pft::Evaluation &eval,
3717 const parser::OpenMPCriticalConstruct &criticalConstruct) {
3718 const auto &cd = std::get<parser::OmpCriticalDirective>(criticalConstruct.t);
3719 List<Clause> clauses =
3720 makeClauses(std::get<parser::OmpClauseList>(cd.t), semaCtx);
3721
3722 ConstructQueue queue{buildConstructQueue(
3723 converter.getFirOpBuilder().getModule(), semaCtx, eval, cd.source,
3724 llvm::omp::Directive::OMPD_critical, clauses)};
3725
3726 const auto &name = std::get<std::optional<parser::Name>>(cd.t);
3727 mlir::Location currentLocation = converter.getCurrentLocation();
3728 genCriticalOp(converter, symTable, semaCtx, eval, currentLocation, queue,
3729 queue.begin(), name);
3730}
3731
3732static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
3733 semantics::SemanticsContext &semaCtx,
3734 lower::pft::Evaluation &eval,
3735 const parser::OpenMPUtilityConstruct &) {
3736 TODO(converter.getCurrentLocation(), "OpenMPUtilityConstruct");
3737}
3738
3739static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
3740 semantics::SemanticsContext &semaCtx,
3741 lower::pft::Evaluation &eval,
3742 const parser::OpenMPDispatchConstruct &) {
3743 TODO(converter.getCurrentLocation(), "OpenMPDispatchConstruct");
3744}
3745
3746static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
3747 semantics::SemanticsContext &semaCtx,
3748 lower::pft::Evaluation &eval,
3749 const parser::OpenMPExecutableAllocate &execAllocConstruct) {
3750 TODO(converter.getCurrentLocation(), "OpenMPExecutableAllocate");
3751}
3752
3753static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
3754 semantics::SemanticsContext &semaCtx,
3755 lower::pft::Evaluation &eval,
3756 const parser::OpenMPLoopConstruct &loopConstruct) {
3757 const auto &beginLoopDirective =
3758 std::get<parser::OmpBeginLoopDirective>(loopConstruct.t);
3759 List<Clause> clauses = makeClauses(
3760 std::get<parser::OmpClauseList>(beginLoopDirective.t), semaCtx);
3761 if (auto &endLoopDirective =
3762 std::get<std::optional<parser::OmpEndLoopDirective>>(
3763 loopConstruct.t)) {
3764 clauses.append(makeClauses(
3765 std::get<parser::OmpClauseList>(endLoopDirective->t), semaCtx));
3766 }
3767
3768 mlir::Location currentLocation =
3769 converter.genLocation(beginLoopDirective.source);
3770
3771 auto &optLoopCons =
3772 std::get<std::optional<parser::NestedConstruct>>(loopConstruct.t);
3773 if (optLoopCons.has_value()) {
3774 if (auto *ompNestedLoopCons{
3775 std::get_if<common::Indirection<parser::OpenMPLoopConstruct>>(
3776 &*optLoopCons)}) {
3777 genOMP(converter, symTable, semaCtx, eval, ompNestedLoopCons->value());
3778 }
3779 }
3780
3781 llvm::omp::Directive directive =
3782 std::get<parser::OmpLoopDirective>(beginLoopDirective.t).v;
3783 const parser::CharBlock &source =
3784 std::get<parser::OmpLoopDirective>(beginLoopDirective.t).source;
3785 ConstructQueue queue{
3786 buildConstructQueue(converter.getFirOpBuilder().getModule(), semaCtx,
3787 eval, source, directive, clauses)};
3788 genOMPDispatch(converter, symTable, semaCtx, eval, currentLocation, queue,
3789 queue.begin());
3790}
3791
3792static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
3793 semantics::SemanticsContext &semaCtx,
3794 lower::pft::Evaluation &eval,
3795 const parser::OpenMPSectionConstruct &sectionConstruct) {
3796 // Do nothing here. SECTION is lowered inside of the lowering for Sections
3797}
3798
3799static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
3800 semantics::SemanticsContext &semaCtx,
3801 lower::pft::Evaluation &eval,
3802 const parser::OpenMPSectionsConstruct &sectionsConstruct) {
3803 const auto &beginSectionsDirective =
3804 std::get<parser::OmpBeginSectionsDirective>(sectionsConstruct.t);
3805 List<Clause> clauses = makeClauses(
3806 std::get<parser::OmpClauseList>(beginSectionsDirective.t), semaCtx);
3807 const auto &endSectionsDirective =
3808 std::get<parser::OmpEndSectionsDirective>(sectionsConstruct.t);
3809 clauses.append(makeClauses(
3810 std::get<parser::OmpClauseList>(endSectionsDirective.t), semaCtx));
3811 mlir::Location currentLocation = converter.getCurrentLocation();
3812
3813 llvm::omp::Directive directive =
3814 std::get<parser::OmpSectionsDirective>(beginSectionsDirective.t).v;
3815 const parser::CharBlock &source =
3816 std::get<parser::OmpSectionsDirective>(beginSectionsDirective.t).source;
3817 ConstructQueue queue{
3818 buildConstructQueue(converter.getFirOpBuilder().getModule(), semaCtx,
3819 eval, source, directive, clauses)};
3820
3821 mlir::SaveStateStack<SectionsConstructStackFrame> saveStateStack{
3822 converter.getStateStack(), sectionsConstruct};
3823 genOMPDispatch(converter, symTable, semaCtx, eval, currentLocation, queue,
3824 queue.begin());
3825}
3826
3827static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
3828 semantics::SemanticsContext &semaCtx,
3829 lower::pft::Evaluation &eval,
3830 const parser::OpenMPConstruct &ompConstruct) {
3831 Fortran::common::visit(
3832 [&](auto &&s) { return genOMP(converter, symTable, semaCtx, eval, s); },
3833 ompConstruct.u);
3834}
3835
3836//===----------------------------------------------------------------------===//
3837// Public functions
3838//===----------------------------------------------------------------------===//
3839
3840mlir::Operation *Fortran::lower::genOpenMPTerminator(fir::FirOpBuilder &builder,
3841 mlir::Operation *op,
3842 mlir::Location loc) {
3843 if (mlir::isa<mlir::omp::AtomicUpdateOp, mlir::omp::DeclareReductionOp,
3844 mlir::omp::LoopNestOp>(op))
3845 return builder.create<mlir::omp::YieldOp>(loc);
3846 return builder.create<mlir::omp::TerminatorOp>(loc);
3847}
3848
3849void Fortran::lower::genOpenMPConstruct(lower::AbstractConverter &converter,
3850 lower::SymMap &symTable,
3851 semantics::SemanticsContext &semaCtx,
3852 lower::pft::Evaluation &eval,
3853 const parser::OpenMPConstruct &omp) {
3854 lower::SymMapScope scope(symTable);
3855 genOMP(converter, symTable, semaCtx, eval, omp);
3856}
3857
3858void Fortran::lower::genOpenMPDeclarativeConstruct(
3859 lower::AbstractConverter &converter, lower::SymMap &symTable,
3860 semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
3861 const parser::OpenMPDeclarativeConstruct &omp) {
3862 genOMP(converter, symTable, semaCtx, eval, omp);
3863 genNestedEvaluations(converter, eval);
3864}
3865
3866void Fortran::lower::genOpenMPSymbolProperties(
3867 lower::AbstractConverter &converter, const lower::pft::Variable &var) {
3868 assert(var.hasSymbol() && "Expecting Symbol");
3869 const semantics::Symbol &sym = var.getSymbol();
3870
3871 if (sym.test(semantics::Symbol::Flag::OmpThreadprivate))
3872 lower::genThreadprivateOp(converter, var);
3873
3874 if (sym.test(semantics::Symbol::Flag::OmpDeclareTarget))
3875 lower::genDeclareTargetIntGlobal(converter, var);
3876}
3877
3878int64_t
3879Fortran::lower::getCollapseValue(const parser::OmpClauseList &clauseList) {
3880 for (const parser::OmpClause &clause : clauseList.v) {
3881 if (const auto &collapseClause =
3882 std::get_if<parser::OmpClause::Collapse>(&clause.u)) {
3883 const auto *expr = semantics::GetExpr(collapseClause->v);
3884 return evaluate::ToInt64(*expr).value();
3885 }
3886 }
3887 return 1;
3888}
3889
3890void Fortran::lower::genThreadprivateOp(lower::AbstractConverter &converter,
3891 const lower::pft::Variable &var) {
3892 fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
3893 mlir::Location currentLocation = converter.getCurrentLocation();
3894
3895 const semantics::Symbol &sym = var.getSymbol();
3896 mlir::Value symThreadprivateValue;
3897 if (const semantics::Symbol *common =
3898 semantics::FindCommonBlockContaining(sym.GetUltimate())) {
3899 mlir::Value commonValue = converter.getSymbolAddress(*common);
3900 if (mlir::isa<mlir::omp::ThreadprivateOp>(commonValue.getDefiningOp())) {
3901 // Generate ThreadprivateOp for a common block instead of its members and
3902 // only do it once for a common block.
3903 return;
3904 }
3905 // Generate ThreadprivateOp and rebind the common block.
3906 mlir::Value commonThreadprivateValue =
3907 firOpBuilder.create<mlir::omp::ThreadprivateOp>(
3908 currentLocation, commonValue.getType(), commonValue);
3909 converter.bindSymbol(*common, commonThreadprivateValue);
3910 // Generate the threadprivate value for the common block member.
3911 symThreadprivateValue = genCommonBlockMember(converter, currentLocation,
3912 sym, commonThreadprivateValue);
3913 } else if (!var.isGlobal()) {
3914 // Non-global variable which can be in threadprivate directive must be one
3915 // variable in main program, and it has implicit SAVE attribute. Take it as
3916 // with SAVE attribute, so to create GlobalOp for it to simplify the
3917 // translation to LLVM IR.
3918 // Avoids performing multiple globalInitializations.
3919 fir::GlobalOp global;
3920 auto module = converter.getModuleOp();
3921 std::string globalName = converter.mangleName(sym);
3922 if (module.lookupSymbol<fir::GlobalOp>(globalName))
3923 global = module.lookupSymbol<fir::GlobalOp>(globalName);
3924 else
3925 global = globalInitialization(converter, firOpBuilder, sym, var,
3926 currentLocation);
3927
3928 mlir::Value symValue = firOpBuilder.create<fir::AddrOfOp>(
3929 currentLocation, global.resultType(), global.getSymbol());
3930 symThreadprivateValue = firOpBuilder.create<mlir::omp::ThreadprivateOp>(
3931 currentLocation, symValue.getType(), symValue);
3932 } else {
3933 mlir::Value symValue = converter.getSymbolAddress(sym);
3934
3935 // The symbol may be use-associated multiple times, and nothing needs to be
3936 // done after the original symbol is mapped to the threadprivatized value
3937 // for the first time. Use the threadprivatized value directly.
3938 mlir::Operation *op;
3939 if (auto declOp = symValue.getDefiningOp<hlfir::DeclareOp>())
3940 op = declOp.getMemref().getDefiningOp();
3941 else
3942 op = symValue.getDefiningOp();
3943 if (mlir::isa<mlir::omp::ThreadprivateOp>(op))
3944 return;
3945
3946 symThreadprivateValue = firOpBuilder.create<mlir::omp::ThreadprivateOp>(
3947 currentLocation, symValue.getType(), symValue);
3948 }
3949
3950 fir::ExtendedValue sexv = converter.getSymbolExtendedValue(sym);
3951 fir::ExtendedValue symThreadprivateExv =
3952 getExtendedValue(sexv, symThreadprivateValue);
3953 converter.bindSymbol(sym, symThreadprivateExv);
3954}
3955
3956// This function replicates threadprivate's behaviour of generating
3957// an internal fir.GlobalOp for non-global variables in the main program
3958// that have the implicit SAVE attribute, to simplifiy LLVM-IR and MLIR
3959// generation.
3960void Fortran::lower::genDeclareTargetIntGlobal(
3961 lower::AbstractConverter &converter, const lower::pft::Variable &var) {
3962 if (!var.isGlobal()) {
3963 // A non-global variable which can be in a declare target directive must
3964 // be a variable in the main program, and it has the implicit SAVE
3965 // attribute. We create a GlobalOp for it to simplify the translation to
3966 // LLVM IR.
3967 globalInitialization(converter, converter.getFirOpBuilder(),
3968 var.getSymbol(), var, converter.getCurrentLocation());
3969 }
3970}
3971
3972bool Fortran::lower::isOpenMPTargetConstruct(
3973 const parser::OpenMPConstruct &omp) {
3974 llvm::omp::Directive dir = llvm::omp::Directive::OMPD_unknown;
3975 if (const auto *block = std::get_if<parser::OpenMPBlockConstruct>(&omp.u)) {
3976 const auto &begin = std::get<parser::OmpBeginBlockDirective>(block->t);
3977 dir = std::get<parser::OmpBlockDirective>(begin.t).v;
3978 } else if (const auto *loop =
3979 std::get_if<parser::OpenMPLoopConstruct>(&omp.u)) {
3980 const auto &begin = std::get<parser::OmpBeginLoopDirective>(loop->t);
3981 dir = std::get<parser::OmpLoopDirective>(begin.t).v;
3982 }
3983 return llvm::omp::allTargetSet.test(dir);
3984}
3985
3986void Fortran::lower::gatherOpenMPDeferredDeclareTargets(
3987 lower::AbstractConverter &converter, semantics::SemanticsContext &semaCtx,
3988 lower::pft::Evaluation &eval,
3989 const parser::OpenMPDeclarativeConstruct &ompDecl,
3990 llvm::SmallVectorImpl<OMPDeferredDeclareTargetInfo>
3991 &deferredDeclareTarget) {
3992 Fortran::common::visit(
3993 common::visitors{
3994 [&](const parser::OpenMPDeclareTargetConstruct &ompReq) {
3995 collectDeferredDeclareTargets(converter, semaCtx, eval, ompReq,
3996 deferredDeclareTarget);
3997 },
3998 [&](const auto &) {},
3999 },
4000 ompDecl.u);
4001}
4002
4003bool Fortran::lower::isOpenMPDeviceDeclareTarget(
4004 lower::AbstractConverter &converter, semantics::SemanticsContext &semaCtx,
4005 lower::pft::Evaluation &eval,
4006 const parser::OpenMPDeclarativeConstruct &ompDecl) {
4007 return Fortran::common::visit(
4008 common::visitors{
4009 [&](const parser::OpenMPDeclareTargetConstruct &ompReq) {
4010 mlir::omp::DeclareTargetDeviceType targetType =
4011 getDeclareTargetFunctionDevice(converter, semaCtx, eval, ompReq)
4012 .value_or(mlir::omp::DeclareTargetDeviceType::host);
4013 return targetType != mlir::omp::DeclareTargetDeviceType::host;
4014 },
4015 [&](const auto &) { return false; },
4016 },
4017 ompDecl.u);
4018}
4019
4020// In certain cases such as subroutine or function interfaces which declare
4021// but do not define or directly call the subroutine or function in the same
4022// module, their lowering is delayed until after the declare target construct
4023// itself is processed, so there symbol is not within the table.
4024//
4025// This function will also return true if we encounter any device declare
4026// target cases, to satisfy checking if we require the requires attributes
4027// on the module.
4028bool Fortran::lower::markOpenMPDeferredDeclareTargetFunctions(
4029 mlir::Operation *mod,
4030 llvm::SmallVectorImpl<OMPDeferredDeclareTargetInfo> &deferredDeclareTargets,
4031 AbstractConverter &converter) {
4032 bool deviceCodeFound = false;
4033 auto modOp = llvm::cast<mlir::ModuleOp>(mod);
4034 for (auto declTar : deferredDeclareTargets) {
4035 mlir::Operation *op = modOp.lookupSymbol(converter.mangleName(declTar.sym));
4036
4037 // Due to interfaces being optionally emitted on usage in a module,
4038 // not finding an operation at this point cannot be a hard error, we
4039 // simply ignore it for now.
4040 // TODO: Add semantic checks for detecting cases where an erronous
4041 // (undefined) symbol has been supplied to a declare target clause
4042 if (!op)
4043 continue;
4044
4045 auto devType = declTar.declareTargetDeviceType;
4046 if (!deviceCodeFound && devType != mlir::omp::DeclareTargetDeviceType::host)
4047 deviceCodeFound = true;
4048
4049 markDeclareTarget(op, converter, declTar.declareTargetCaptureClause,
4050 devType);
4051 }
4052
4053 return deviceCodeFound;
4054}
4055
4056void Fortran::lower::genOpenMPRequires(mlir::Operation *mod,
4057 const semantics::Symbol *symbol) {
4058 using MlirRequires = mlir::omp::ClauseRequires;
4059 using SemaRequires = semantics::WithOmpDeclarative::RequiresFlag;
4060
4061 if (auto offloadMod =
4062 llvm::dyn_cast<mlir::omp::OffloadModuleInterface>(mod)) {
4063 semantics::WithOmpDeclarative::RequiresFlags semaFlags;
4064 if (symbol) {
4065 common::visit(
4066 [&](const auto &details) {
4067 if constexpr (std::is_base_of_v<semantics::WithOmpDeclarative,
4068 std::decay_t<decltype(details)>>) {
4069 if (details.has_ompRequires())
4070 semaFlags = *details.ompRequires();
4071 }
4072 },
4073 symbol->details());
4074 }
4075
4076 // Use pre-populated omp.requires module attribute if it was set, so that
4077 // the "-fopenmp-force-usm" compiler option is honored.
4078 MlirRequires mlirFlags = offloadMod.getRequires();
4079 if (semaFlags.test(SemaRequires::ReverseOffload))
4080 mlirFlags = mlirFlags | MlirRequires::reverse_offload;
4081 if (semaFlags.test(SemaRequires::UnifiedAddress))
4082 mlirFlags = mlirFlags | MlirRequires::unified_address;
4083 if (semaFlags.test(SemaRequires::UnifiedSharedMemory))
4084 mlirFlags = mlirFlags | MlirRequires::unified_shared_memory;
4085 if (semaFlags.test(SemaRequires::DynamicAllocators))
4086 mlirFlags = mlirFlags | MlirRequires::dynamic_allocators;
4087
4088 offloadMod.setRequires(mlirFlags);
4089 }
4090}
4091

source code of flang/lib/Lower/OpenMP/OpenMP.cpp