1//===-- DataSharingProcessor.cpp --------------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Coding style: https://mlir.llvm.org/getting_started/DeveloperGuide/
10//
11//===----------------------------------------------------------------------===//
12
13#include "DataSharingProcessor.h"
14
15#include "Utils.h"
16#include "flang/Lower/PFTBuilder.h"
17#include "flang/Lower/SymbolMap.h"
18#include "flang/Optimizer/Builder/Todo.h"
19#include "flang/Semantics/tools.h"
20#include "mlir/Dialect/OpenMP/OpenMPDialect.h"
21
22namespace Fortran {
23namespace lower {
24namespace omp {
25
26void DataSharingProcessor::processStep1(
27 mlir::omp::PrivateClauseOps *clauseOps,
28 llvm::SmallVectorImpl<const Fortran::semantics::Symbol *> *privateSyms) {
29 collectSymbolsForPrivatization();
30 collectDefaultSymbols();
31 privatize(clauseOps, privateSyms);
32 defaultPrivatize(clauseOps, privateSyms);
33 insertBarrier();
34}
35
36void DataSharingProcessor::processStep2(mlir::Operation *op, bool isLoop) {
37 insPt = firOpBuilder.saveInsertionPoint();
38 copyLastPrivatize(op);
39 firOpBuilder.restoreInsertionPoint(insPt);
40
41 if (isLoop) {
42 // push deallocs out of the loop
43 firOpBuilder.setInsertionPointAfter(op);
44 insertDeallocs();
45 } else {
46 // insert dummy instruction to mark the insertion position
47 mlir::Value undefMarker = firOpBuilder.create<fir::UndefOp>(
48 op->getLoc(), firOpBuilder.getIndexType());
49 insertDeallocs();
50 firOpBuilder.setInsertionPointAfter(undefMarker.getDefiningOp());
51 }
52}
53
54void DataSharingProcessor::insertDeallocs() {
55 // TODO Extend delayed privatization to include a `dealloc` region.
56 for (const Fortran::semantics::Symbol *sym : privatizedSymbols)
57 if (Fortran::semantics::IsAllocatable(sym->GetUltimate())) {
58 converter.createHostAssociateVarCloneDealloc(*sym);
59 }
60}
61
62void DataSharingProcessor::cloneSymbol(const Fortran::semantics::Symbol *sym) {
63 // Privatization for symbols which are pre-determined (like loop index
64 // variables) happen separately, for everything else privatize here.
65 if (sym->test(Fortran::semantics::Symbol::Flag::OmpPreDetermined))
66 return;
67 bool success = converter.createHostAssociateVarClone(*sym);
68 (void)success;
69 assert(success && "Privatization failed due to existing binding");
70}
71
72void DataSharingProcessor::copyFirstPrivateSymbol(
73 const Fortran::semantics::Symbol *sym,
74 mlir::OpBuilder::InsertPoint *copyAssignIP) {
75 if (sym->test(Fortran::semantics::Symbol::Flag::OmpFirstPrivate))
76 converter.copyHostAssociateVar(*sym, copyAssignIP);
77}
78
79void DataSharingProcessor::copyLastPrivateSymbol(
80 const Fortran::semantics::Symbol *sym,
81 [[maybe_unused]] mlir::OpBuilder::InsertPoint *lastPrivIP) {
82 if (sym->test(Fortran::semantics::Symbol::Flag::OmpLastPrivate))
83 converter.copyHostAssociateVar(*sym, lastPrivIP);
84}
85
86void DataSharingProcessor::collectOmpObjectListSymbol(
87 const omp::ObjectList &objects,
88 llvm::SetVector<const Fortran::semantics::Symbol *> &symbolSet) {
89 for (const omp::Object &object : objects)
90 symbolSet.insert(object.id());
91}
92
93void DataSharingProcessor::collectSymbolsForPrivatization() {
94 bool hasCollapse = false;
95 for (const omp::Clause &clause : clauses) {
96 if (const auto &privateClause =
97 std::get_if<omp::clause::Private>(&clause.u)) {
98 collectOmpObjectListSymbol(privateClause->v, privatizedSymbols);
99 } else if (const auto &firstPrivateClause =
100 std::get_if<omp::clause::Firstprivate>(&clause.u)) {
101 collectOmpObjectListSymbol(firstPrivateClause->v, privatizedSymbols);
102 } else if (const auto &lastPrivateClause =
103 std::get_if<omp::clause::Lastprivate>(&clause.u)) {
104 const ObjectList &objects = std::get<ObjectList>(lastPrivateClause->t);
105 collectOmpObjectListSymbol(objects, privatizedSymbols);
106 hasLastPrivateOp = true;
107 } else if (std::get_if<omp::clause::Collapse>(&clause.u)) {
108 hasCollapse = true;
109 }
110 }
111
112 if (hasCollapse && hasLastPrivateOp)
113 TODO(converter.getCurrentLocation(), "Collapse clause with lastprivate");
114}
115
116bool DataSharingProcessor::needBarrier() {
117 for (const Fortran::semantics::Symbol *sym : privatizedSymbols) {
118 if (sym->test(Fortran::semantics::Symbol::Flag::OmpFirstPrivate) &&
119 sym->test(Fortran::semantics::Symbol::Flag::OmpLastPrivate))
120 return true;
121 }
122 return false;
123}
124
125void DataSharingProcessor::insertBarrier() {
126 // Emit implicit barrier to synchronize threads and avoid data races on
127 // initialization of firstprivate variables and post-update of lastprivate
128 // variables.
129 // FIXME: Emit barrier for lastprivate clause when 'sections' directive has
130 // 'nowait' clause. Otherwise, emit barrier when 'sections' directive has
131 // both firstprivate and lastprivate clause.
132 // Emit implicit barrier for linear clause. Maybe on somewhere else.
133 if (needBarrier())
134 firOpBuilder.create<mlir::omp::BarrierOp>(converter.getCurrentLocation());
135}
136
137void DataSharingProcessor::insertLastPrivateCompare(mlir::Operation *op) {
138 mlir::omp::LoopNestOp loopOp;
139 if (auto wrapper = mlir::dyn_cast<mlir::omp::LoopWrapperInterface>(op))
140 loopOp = wrapper.isWrapper()
141 ? mlir::cast<mlir::omp::LoopNestOp>(wrapper.getWrappedLoop())
142 : nullptr;
143
144 bool cmpCreated = false;
145 mlir::OpBuilder::InsertionGuard guard(firOpBuilder);
146 for (const omp::Clause &clause : clauses) {
147 if (clause.id != llvm::omp::OMPC_lastprivate)
148 continue;
149 // TODO: Add lastprivate support for simd construct
150 if (mlir::isa<mlir::omp::SectionOp>(op)) {
151 if (&eval == &eval.parentConstruct->getLastNestedEvaluation()) {
152 // For `omp.sections`, lastprivatized variables occur in
153 // lexically final `omp.section` operation. The following FIR
154 // shall be generated for the same:
155 //
156 // omp.sections lastprivate(...) {
157 // omp.section {...}
158 // omp.section {...}
159 // omp.section {
160 // fir.allocate for `private`/`firstprivate`
161 // <More operations here>
162 // fir.if %true {
163 // ^%lpv_update_blk
164 // }
165 // }
166 // }
167 //
168 // To keep code consistency while handling privatization
169 // through this control flow, add a `fir.if` operation
170 // that always evaluates to true, in order to create
171 // a dedicated sub-region in `omp.section` where
172 // lastprivate FIR can reside. Later canonicalizations
173 // will optimize away this operation.
174 if (!eval.lowerAsUnstructured()) {
175 auto ifOp = firOpBuilder.create<fir::IfOp>(
176 op->getLoc(),
177 firOpBuilder.createIntegerConstant(
178 op->getLoc(), firOpBuilder.getIntegerType(1), 0x1),
179 /*else*/ false);
180 firOpBuilder.setInsertionPointToStart(&ifOp.getThenRegion().front());
181
182 const Fortran::parser::OpenMPConstruct *parentOmpConstruct =
183 eval.parentConstruct->getIf<Fortran::parser::OpenMPConstruct>();
184 assert(parentOmpConstruct &&
185 "Expected a valid enclosing OpenMP construct");
186 const Fortran::parser::OpenMPSectionsConstruct *sectionsConstruct =
187 std::get_if<Fortran::parser::OpenMPSectionsConstruct>(
188 &parentOmpConstruct->u);
189 assert(sectionsConstruct &&
190 "Expected an enclosing omp.sections construct");
191 const Fortran::parser::OmpClauseList &sectionsEndClauseList =
192 std::get<Fortran::parser::OmpClauseList>(
193 std::get<Fortran::parser::OmpEndSectionsDirective>(
194 sectionsConstruct->t)
195 .t);
196 for (const Fortran::parser::OmpClause &otherClause :
197 sectionsEndClauseList.v)
198 if (std::get_if<Fortran::parser::OmpClause::Nowait>(&otherClause.u))
199 // Emit implicit barrier to synchronize threads and avoid data
200 // races on post-update of lastprivate variables when `nowait`
201 // clause is present.
202 firOpBuilder.create<mlir::omp::BarrierOp>(
203 converter.getCurrentLocation());
204 firOpBuilder.setInsertionPointToStart(&ifOp.getThenRegion().front());
205 lastPrivIP = firOpBuilder.saveInsertionPoint();
206 firOpBuilder.setInsertionPoint(ifOp);
207 insPt = firOpBuilder.saveInsertionPoint();
208 } else {
209 // Lastprivate operation is inserted at the end
210 // of the lexically last section in the sections
211 // construct
212 mlir::OpBuilder::InsertionGuard unstructuredSectionsGuard(
213 firOpBuilder);
214 mlir::Operation *lastOper = op->getRegion(0).back().getTerminator();
215 firOpBuilder.setInsertionPoint(lastOper);
216 lastPrivIP = firOpBuilder.saveInsertionPoint();
217 }
218 }
219 } else if (mlir::isa<mlir::omp::WsloopOp>(op)) {
220 // Update the original variable just before exiting the worksharing
221 // loop. Conversion as follows:
222 //
223 // omp.wsloop { omp.wsloop {
224 // omp.loop_nest { omp.loop_nest {
225 // ... ...
226 // store ===> store
227 // omp.yield %v = arith.addi %iv, %step
228 // } %cmp = %step < 0 ? %v < %ub : %v > %ub
229 // omp.terminator fir.if %cmp {
230 // } fir.store %v to %loopIV
231 // ^%lpv_update_blk:
232 // }
233 // omp.yield
234 // }
235 // omp.terminator
236 // }
237
238 // Only generate the compare once in presence of multiple LastPrivate
239 // clauses.
240 if (cmpCreated)
241 continue;
242 cmpCreated = true;
243
244 mlir::Location loc = loopOp.getLoc();
245 mlir::Operation *lastOper = loopOp.getRegion().back().getTerminator();
246 firOpBuilder.setInsertionPoint(lastOper);
247
248 mlir::Value iv = loopOp.getIVs()[0];
249 mlir::Value ub = loopOp.getUpperBound()[0];
250 mlir::Value step = loopOp.getStep()[0];
251
252 // v = iv + step
253 // cmp = step < 0 ? v < ub : v > ub
254 mlir::Value v = firOpBuilder.create<mlir::arith::AddIOp>(loc, iv, step);
255 mlir::Value zero =
256 firOpBuilder.createIntegerConstant(loc, step.getType(), 0);
257 mlir::Value negativeStep = firOpBuilder.create<mlir::arith::CmpIOp>(
258 loc, mlir::arith::CmpIPredicate::slt, step, zero);
259 mlir::Value vLT = firOpBuilder.create<mlir::arith::CmpIOp>(
260 loc, mlir::arith::CmpIPredicate::slt, v, ub);
261 mlir::Value vGT = firOpBuilder.create<mlir::arith::CmpIOp>(
262 loc, mlir::arith::CmpIPredicate::sgt, v, ub);
263 mlir::Value cmpOp = firOpBuilder.create<mlir::arith::SelectOp>(
264 loc, negativeStep, vLT, vGT);
265
266 auto ifOp = firOpBuilder.create<fir::IfOp>(loc, cmpOp, /*else*/ false);
267 firOpBuilder.setInsertionPointToStart(&ifOp.getThenRegion().front());
268 assert(loopIV && "loopIV was not set");
269 firOpBuilder.create<fir::StoreOp>(loopOp.getLoc(), v, loopIV);
270 lastPrivIP = firOpBuilder.saveInsertionPoint();
271 } else {
272 TODO(converter.getCurrentLocation(),
273 "lastprivate clause in constructs other than "
274 "simd/worksharing-loop");
275 }
276 }
277}
278
279void DataSharingProcessor::collectSymbols(
280 Fortran::semantics::Symbol::Flag flag) {
281 converter.collectSymbolSet(eval, defaultSymbols, flag,
282 /*collectSymbols=*/true,
283 /*collectHostAssociatedSymbols=*/true);
284 for (Fortran::lower::pft::Evaluation &e : eval.getNestedEvaluations()) {
285 if (e.hasNestedEvaluations())
286 converter.collectSymbolSet(e, symbolsInNestedRegions, flag,
287 /*collectSymbols=*/true,
288 /*collectHostAssociatedSymbols=*/false);
289 else
290 converter.collectSymbolSet(e, symbolsInParentRegions, flag,
291 /*collectSymbols=*/false,
292 /*collectHostAssociatedSymbols=*/true);
293 }
294}
295
296void DataSharingProcessor::collectDefaultSymbols() {
297 using DataSharingAttribute = omp::clause::Default::DataSharingAttribute;
298 for (const omp::Clause &clause : clauses) {
299 if (const auto *defaultClause =
300 std::get_if<omp::clause::Default>(&clause.u)) {
301 if (defaultClause->v == DataSharingAttribute::Private)
302 collectSymbols(Fortran::semantics::Symbol::Flag::OmpPrivate);
303 else if (defaultClause->v == DataSharingAttribute::Firstprivate)
304 collectSymbols(Fortran::semantics::Symbol::Flag::OmpFirstPrivate);
305 }
306 }
307}
308
309void DataSharingProcessor::privatize(
310 mlir::omp::PrivateClauseOps *clauseOps,
311 llvm::SmallVectorImpl<const Fortran::semantics::Symbol *> *privateSyms) {
312 for (const Fortran::semantics::Symbol *sym : privatizedSymbols) {
313 if (const auto *commonDet =
314 sym->detailsIf<Fortran::semantics::CommonBlockDetails>()) {
315 for (const auto &mem : commonDet->objects())
316 doPrivatize(&*mem, clauseOps, privateSyms);
317 } else
318 doPrivatize(sym, clauseOps, privateSyms);
319 }
320}
321
322void DataSharingProcessor::copyLastPrivatize(mlir::Operation *op) {
323 insertLastPrivateCompare(op);
324 for (const Fortran::semantics::Symbol *sym : privatizedSymbols)
325 if (const auto *commonDet =
326 sym->detailsIf<Fortran::semantics::CommonBlockDetails>()) {
327 for (const auto &mem : commonDet->objects()) {
328 copyLastPrivateSymbol(&*mem, &lastPrivIP);
329 }
330 } else {
331 copyLastPrivateSymbol(sym, &lastPrivIP);
332 }
333}
334
335void DataSharingProcessor::defaultPrivatize(
336 mlir::omp::PrivateClauseOps *clauseOps,
337 llvm::SmallVectorImpl<const Fortran::semantics::Symbol *> *privateSyms) {
338 for (const Fortran::semantics::Symbol *sym : defaultSymbols) {
339 if (!Fortran::semantics::IsProcedure(*sym) &&
340 !sym->GetUltimate().has<Fortran::semantics::DerivedTypeDetails>() &&
341 !sym->GetUltimate().has<Fortran::semantics::NamelistDetails>() &&
342 !symbolsInNestedRegions.contains(sym) &&
343 !symbolsInParentRegions.contains(sym) &&
344 !privatizedSymbols.contains(sym))
345 doPrivatize(sym, clauseOps, privateSyms);
346 }
347}
348
349void DataSharingProcessor::doPrivatize(
350 const Fortran::semantics::Symbol *sym,
351 mlir::omp::PrivateClauseOps *clauseOps,
352 llvm::SmallVectorImpl<const Fortran::semantics::Symbol *> *privateSyms) {
353 if (!useDelayedPrivatization) {
354 cloneSymbol(sym);
355 copyFirstPrivateSymbol(sym);
356 return;
357 }
358
359 Fortran::lower::SymbolBox hsb = converter.lookupOneLevelUpSymbol(*sym);
360 assert(hsb && "Host symbol box not found");
361
362 mlir::Type symType = hsb.getAddr().getType();
363 mlir::Location symLoc = hsb.getAddr().getLoc();
364 std::string privatizerName = sym->name().ToString() + ".privatizer";
365 bool isFirstPrivate =
366 sym->test(Fortran::semantics::Symbol::Flag::OmpFirstPrivate);
367
368 mlir::omp::PrivateClauseOp privatizerOp = [&]() {
369 auto moduleOp = firOpBuilder.getModule();
370 auto uniquePrivatizerName = fir::getTypeAsString(
371 symType, converter.getKindMap(),
372 converter.mangleName(*sym) +
373 (isFirstPrivate ? "_firstprivate" : "_private"));
374
375 if (auto existingPrivatizer =
376 moduleOp.lookupSymbol<mlir::omp::PrivateClauseOp>(
377 uniquePrivatizerName))
378 return existingPrivatizer;
379
380 mlir::OpBuilder::InsertionGuard guard(firOpBuilder);
381 firOpBuilder.setInsertionPoint(&moduleOp.getBodyRegion().front(),
382 moduleOp.getBodyRegion().front().begin());
383 auto result = firOpBuilder.create<mlir::omp::PrivateClauseOp>(
384 symLoc, uniquePrivatizerName, symType,
385 isFirstPrivate ? mlir::omp::DataSharingClauseType::FirstPrivate
386 : mlir::omp::DataSharingClauseType::Private);
387 fir::ExtendedValue symExV = converter.getSymbolExtendedValue(*sym);
388
389 symTable->pushScope();
390
391 // Populate the `alloc` region.
392 {
393 mlir::Region &allocRegion = result.getAllocRegion();
394 mlir::Block *allocEntryBlock = firOpBuilder.createBlock(
395 &allocRegion, /*insertPt=*/{}, symType, symLoc);
396
397 firOpBuilder.setInsertionPointToEnd(allocEntryBlock);
398 symTable->addSymbol(*sym,
399 fir::substBase(symExV, allocRegion.getArgument(0)));
400 symTable->pushScope();
401 cloneSymbol(sym);
402 firOpBuilder.create<mlir::omp::YieldOp>(
403 hsb.getAddr().getLoc(),
404 symTable->shallowLookupSymbol(*sym).getAddr());
405 symTable->popScope();
406 }
407
408 // Populate the `copy` region if this is a `firstprivate`.
409 if (isFirstPrivate) {
410 mlir::Region &copyRegion = result.getCopyRegion();
411 // First block argument corresponding to the original/host value while
412 // second block argument corresponding to the privatized value.
413 mlir::Block *copyEntryBlock = firOpBuilder.createBlock(
414 &copyRegion, /*insertPt=*/{}, {symType, symType}, {symLoc, symLoc});
415 firOpBuilder.setInsertionPointToEnd(copyEntryBlock);
416 symTable->addSymbol(*sym,
417 fir::substBase(symExV, copyRegion.getArgument(0)),
418 /*force=*/true);
419 symTable->pushScope();
420 symTable->addSymbol(*sym,
421 fir::substBase(symExV, copyRegion.getArgument(1)));
422 auto ip = firOpBuilder.saveInsertionPoint();
423 copyFirstPrivateSymbol(sym, &ip);
424
425 firOpBuilder.create<mlir::omp::YieldOp>(
426 hsb.getAddr().getLoc(),
427 symTable->shallowLookupSymbol(*sym).getAddr());
428 symTable->popScope();
429 }
430
431 symTable->popScope();
432 return result;
433 }();
434
435 if (clauseOps) {
436 clauseOps->privatizers.push_back(mlir::SymbolRefAttr::get(privatizerOp));
437 clauseOps->privateVars.push_back(hsb.getAddr());
438 }
439
440 if (privateSyms)
441 privateSyms->push_back(Elt: sym);
442}
443
444} // namespace omp
445} // namespace lower
446} // namespace Fortran
447

source code of flang/lib/Lower/OpenMP/DataSharingProcessor.cpp