1 | //===-- DataSharingProcessor.cpp --------------------------------*- C++ -*-===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | // |
9 | // Coding style: https://mlir.llvm.org/getting_started/DeveloperGuide/ |
10 | // |
11 | //===----------------------------------------------------------------------===// |
12 | |
13 | #include "DataSharingProcessor.h" |
14 | |
15 | #include "Utils.h" |
16 | #include "flang/Lower/PFTBuilder.h" |
17 | #include "flang/Lower/SymbolMap.h" |
18 | #include "flang/Optimizer/Builder/Todo.h" |
19 | #include "flang/Semantics/tools.h" |
20 | #include "mlir/Dialect/OpenMP/OpenMPDialect.h" |
21 | |
22 | namespace Fortran { |
23 | namespace lower { |
24 | namespace omp { |
25 | |
26 | void DataSharingProcessor::processStep1( |
27 | mlir::omp::PrivateClauseOps *clauseOps, |
28 | llvm::SmallVectorImpl<const Fortran::semantics::Symbol *> *privateSyms) { |
29 | collectSymbolsForPrivatization(); |
30 | collectDefaultSymbols(); |
31 | privatize(clauseOps, privateSyms); |
32 | defaultPrivatize(clauseOps, privateSyms); |
33 | insertBarrier(); |
34 | } |
35 | |
36 | void DataSharingProcessor::processStep2(mlir::Operation *op, bool isLoop) { |
37 | insPt = firOpBuilder.saveInsertionPoint(); |
38 | copyLastPrivatize(op); |
39 | firOpBuilder.restoreInsertionPoint(insPt); |
40 | |
41 | if (isLoop) { |
42 | // push deallocs out of the loop |
43 | firOpBuilder.setInsertionPointAfter(op); |
44 | insertDeallocs(); |
45 | } else { |
46 | // insert dummy instruction to mark the insertion position |
47 | mlir::Value undefMarker = firOpBuilder.create<fir::UndefOp>( |
48 | op->getLoc(), firOpBuilder.getIndexType()); |
49 | insertDeallocs(); |
50 | firOpBuilder.setInsertionPointAfter(undefMarker.getDefiningOp()); |
51 | } |
52 | } |
53 | |
54 | void DataSharingProcessor::insertDeallocs() { |
55 | // TODO Extend delayed privatization to include a `dealloc` region. |
56 | for (const Fortran::semantics::Symbol *sym : privatizedSymbols) |
57 | if (Fortran::semantics::IsAllocatable(sym->GetUltimate())) { |
58 | converter.createHostAssociateVarCloneDealloc(*sym); |
59 | } |
60 | } |
61 | |
62 | void DataSharingProcessor::cloneSymbol(const Fortran::semantics::Symbol *sym) { |
63 | // Privatization for symbols which are pre-determined (like loop index |
64 | // variables) happen separately, for everything else privatize here. |
65 | if (sym->test(Fortran::semantics::Symbol::Flag::OmpPreDetermined)) |
66 | return; |
67 | bool success = converter.createHostAssociateVarClone(*sym); |
68 | (void)success; |
69 | assert(success && "Privatization failed due to existing binding" ); |
70 | } |
71 | |
72 | void DataSharingProcessor::copyFirstPrivateSymbol( |
73 | const Fortran::semantics::Symbol *sym, |
74 | mlir::OpBuilder::InsertPoint *copyAssignIP) { |
75 | if (sym->test(Fortran::semantics::Symbol::Flag::OmpFirstPrivate)) |
76 | converter.copyHostAssociateVar(*sym, copyAssignIP); |
77 | } |
78 | |
79 | void DataSharingProcessor::copyLastPrivateSymbol( |
80 | const Fortran::semantics::Symbol *sym, |
81 | [[maybe_unused]] mlir::OpBuilder::InsertPoint *lastPrivIP) { |
82 | if (sym->test(Fortran::semantics::Symbol::Flag::OmpLastPrivate)) |
83 | converter.copyHostAssociateVar(*sym, lastPrivIP); |
84 | } |
85 | |
86 | void DataSharingProcessor::collectOmpObjectListSymbol( |
87 | const omp::ObjectList &objects, |
88 | llvm::SetVector<const Fortran::semantics::Symbol *> &symbolSet) { |
89 | for (const omp::Object &object : objects) |
90 | symbolSet.insert(object.id()); |
91 | } |
92 | |
93 | void DataSharingProcessor::collectSymbolsForPrivatization() { |
94 | bool hasCollapse = false; |
95 | for (const omp::Clause &clause : clauses) { |
96 | if (const auto &privateClause = |
97 | std::get_if<omp::clause::Private>(&clause.u)) { |
98 | collectOmpObjectListSymbol(privateClause->v, privatizedSymbols); |
99 | } else if (const auto &firstPrivateClause = |
100 | std::get_if<omp::clause::Firstprivate>(&clause.u)) { |
101 | collectOmpObjectListSymbol(firstPrivateClause->v, privatizedSymbols); |
102 | } else if (const auto &lastPrivateClause = |
103 | std::get_if<omp::clause::Lastprivate>(&clause.u)) { |
104 | const ObjectList &objects = std::get<ObjectList>(lastPrivateClause->t); |
105 | collectOmpObjectListSymbol(objects, privatizedSymbols); |
106 | hasLastPrivateOp = true; |
107 | } else if (std::get_if<omp::clause::Collapse>(&clause.u)) { |
108 | hasCollapse = true; |
109 | } |
110 | } |
111 | |
112 | if (hasCollapse && hasLastPrivateOp) |
113 | TODO(converter.getCurrentLocation(), "Collapse clause with lastprivate" ); |
114 | } |
115 | |
116 | bool DataSharingProcessor::needBarrier() { |
117 | for (const Fortran::semantics::Symbol *sym : privatizedSymbols) { |
118 | if (sym->test(Fortran::semantics::Symbol::Flag::OmpFirstPrivate) && |
119 | sym->test(Fortran::semantics::Symbol::Flag::OmpLastPrivate)) |
120 | return true; |
121 | } |
122 | return false; |
123 | } |
124 | |
125 | void DataSharingProcessor::insertBarrier() { |
126 | // Emit implicit barrier to synchronize threads and avoid data races on |
127 | // initialization of firstprivate variables and post-update of lastprivate |
128 | // variables. |
129 | // FIXME: Emit barrier for lastprivate clause when 'sections' directive has |
130 | // 'nowait' clause. Otherwise, emit barrier when 'sections' directive has |
131 | // both firstprivate and lastprivate clause. |
132 | // Emit implicit barrier for linear clause. Maybe on somewhere else. |
133 | if (needBarrier()) |
134 | firOpBuilder.create<mlir::omp::BarrierOp>(converter.getCurrentLocation()); |
135 | } |
136 | |
137 | void DataSharingProcessor::insertLastPrivateCompare(mlir::Operation *op) { |
138 | mlir::omp::LoopNestOp loopOp; |
139 | if (auto wrapper = mlir::dyn_cast<mlir::omp::LoopWrapperInterface>(op)) |
140 | loopOp = wrapper.isWrapper() |
141 | ? mlir::cast<mlir::omp::LoopNestOp>(wrapper.getWrappedLoop()) |
142 | : nullptr; |
143 | |
144 | bool cmpCreated = false; |
145 | mlir::OpBuilder::InsertionGuard guard(firOpBuilder); |
146 | for (const omp::Clause &clause : clauses) { |
147 | if (clause.id != llvm::omp::OMPC_lastprivate) |
148 | continue; |
149 | // TODO: Add lastprivate support for simd construct |
150 | if (mlir::isa<mlir::omp::SectionOp>(op)) { |
151 | if (&eval == &eval.parentConstruct->getLastNestedEvaluation()) { |
152 | // For `omp.sections`, lastprivatized variables occur in |
153 | // lexically final `omp.section` operation. The following FIR |
154 | // shall be generated for the same: |
155 | // |
156 | // omp.sections lastprivate(...) { |
157 | // omp.section {...} |
158 | // omp.section {...} |
159 | // omp.section { |
160 | // fir.allocate for `private`/`firstprivate` |
161 | // <More operations here> |
162 | // fir.if %true { |
163 | // ^%lpv_update_blk |
164 | // } |
165 | // } |
166 | // } |
167 | // |
168 | // To keep code consistency while handling privatization |
169 | // through this control flow, add a `fir.if` operation |
170 | // that always evaluates to true, in order to create |
171 | // a dedicated sub-region in `omp.section` where |
172 | // lastprivate FIR can reside. Later canonicalizations |
173 | // will optimize away this operation. |
174 | if (!eval.lowerAsUnstructured()) { |
175 | auto ifOp = firOpBuilder.create<fir::IfOp>( |
176 | op->getLoc(), |
177 | firOpBuilder.createIntegerConstant( |
178 | op->getLoc(), firOpBuilder.getIntegerType(1), 0x1), |
179 | /*else*/ false); |
180 | firOpBuilder.setInsertionPointToStart(&ifOp.getThenRegion().front()); |
181 | |
182 | const Fortran::parser::OpenMPConstruct *parentOmpConstruct = |
183 | eval.parentConstruct->getIf<Fortran::parser::OpenMPConstruct>(); |
184 | assert(parentOmpConstruct && |
185 | "Expected a valid enclosing OpenMP construct" ); |
186 | const Fortran::parser::OpenMPSectionsConstruct *sectionsConstruct = |
187 | std::get_if<Fortran::parser::OpenMPSectionsConstruct>( |
188 | &parentOmpConstruct->u); |
189 | assert(sectionsConstruct && |
190 | "Expected an enclosing omp.sections construct" ); |
191 | const Fortran::parser::OmpClauseList §ionsEndClauseList = |
192 | std::get<Fortran::parser::OmpClauseList>( |
193 | std::get<Fortran::parser::OmpEndSectionsDirective>( |
194 | sectionsConstruct->t) |
195 | .t); |
196 | for (const Fortran::parser::OmpClause &otherClause : |
197 | sectionsEndClauseList.v) |
198 | if (std::get_if<Fortran::parser::OmpClause::Nowait>(&otherClause.u)) |
199 | // Emit implicit barrier to synchronize threads and avoid data |
200 | // races on post-update of lastprivate variables when `nowait` |
201 | // clause is present. |
202 | firOpBuilder.create<mlir::omp::BarrierOp>( |
203 | converter.getCurrentLocation()); |
204 | firOpBuilder.setInsertionPointToStart(&ifOp.getThenRegion().front()); |
205 | lastPrivIP = firOpBuilder.saveInsertionPoint(); |
206 | firOpBuilder.setInsertionPoint(ifOp); |
207 | insPt = firOpBuilder.saveInsertionPoint(); |
208 | } else { |
209 | // Lastprivate operation is inserted at the end |
210 | // of the lexically last section in the sections |
211 | // construct |
212 | mlir::OpBuilder::InsertionGuard unstructuredSectionsGuard( |
213 | firOpBuilder); |
214 | mlir::Operation *lastOper = op->getRegion(0).back().getTerminator(); |
215 | firOpBuilder.setInsertionPoint(lastOper); |
216 | lastPrivIP = firOpBuilder.saveInsertionPoint(); |
217 | } |
218 | } |
219 | } else if (mlir::isa<mlir::omp::WsloopOp>(op)) { |
220 | // Update the original variable just before exiting the worksharing |
221 | // loop. Conversion as follows: |
222 | // |
223 | // omp.wsloop { omp.wsloop { |
224 | // omp.loop_nest { omp.loop_nest { |
225 | // ... ... |
226 | // store ===> store |
227 | // omp.yield %v = arith.addi %iv, %step |
228 | // } %cmp = %step < 0 ? %v < %ub : %v > %ub |
229 | // omp.terminator fir.if %cmp { |
230 | // } fir.store %v to %loopIV |
231 | // ^%lpv_update_blk: |
232 | // } |
233 | // omp.yield |
234 | // } |
235 | // omp.terminator |
236 | // } |
237 | |
238 | // Only generate the compare once in presence of multiple LastPrivate |
239 | // clauses. |
240 | if (cmpCreated) |
241 | continue; |
242 | cmpCreated = true; |
243 | |
244 | mlir::Location loc = loopOp.getLoc(); |
245 | mlir::Operation *lastOper = loopOp.getRegion().back().getTerminator(); |
246 | firOpBuilder.setInsertionPoint(lastOper); |
247 | |
248 | mlir::Value iv = loopOp.getIVs()[0]; |
249 | mlir::Value ub = loopOp.getUpperBound()[0]; |
250 | mlir::Value step = loopOp.getStep()[0]; |
251 | |
252 | // v = iv + step |
253 | // cmp = step < 0 ? v < ub : v > ub |
254 | mlir::Value v = firOpBuilder.create<mlir::arith::AddIOp>(loc, iv, step); |
255 | mlir::Value zero = |
256 | firOpBuilder.createIntegerConstant(loc, step.getType(), 0); |
257 | mlir::Value negativeStep = firOpBuilder.create<mlir::arith::CmpIOp>( |
258 | loc, mlir::arith::CmpIPredicate::slt, step, zero); |
259 | mlir::Value vLT = firOpBuilder.create<mlir::arith::CmpIOp>( |
260 | loc, mlir::arith::CmpIPredicate::slt, v, ub); |
261 | mlir::Value vGT = firOpBuilder.create<mlir::arith::CmpIOp>( |
262 | loc, mlir::arith::CmpIPredicate::sgt, v, ub); |
263 | mlir::Value cmpOp = firOpBuilder.create<mlir::arith::SelectOp>( |
264 | loc, negativeStep, vLT, vGT); |
265 | |
266 | auto ifOp = firOpBuilder.create<fir::IfOp>(loc, cmpOp, /*else*/ false); |
267 | firOpBuilder.setInsertionPointToStart(&ifOp.getThenRegion().front()); |
268 | assert(loopIV && "loopIV was not set" ); |
269 | firOpBuilder.create<fir::StoreOp>(loopOp.getLoc(), v, loopIV); |
270 | lastPrivIP = firOpBuilder.saveInsertionPoint(); |
271 | } else { |
272 | TODO(converter.getCurrentLocation(), |
273 | "lastprivate clause in constructs other than " |
274 | "simd/worksharing-loop" ); |
275 | } |
276 | } |
277 | } |
278 | |
279 | void DataSharingProcessor::collectSymbols( |
280 | Fortran::semantics::Symbol::Flag flag) { |
281 | converter.collectSymbolSet(eval, defaultSymbols, flag, |
282 | /*collectSymbols=*/true, |
283 | /*collectHostAssociatedSymbols=*/true); |
284 | for (Fortran::lower::pft::Evaluation &e : eval.getNestedEvaluations()) { |
285 | if (e.hasNestedEvaluations()) |
286 | converter.collectSymbolSet(e, symbolsInNestedRegions, flag, |
287 | /*collectSymbols=*/true, |
288 | /*collectHostAssociatedSymbols=*/false); |
289 | else |
290 | converter.collectSymbolSet(e, symbolsInParentRegions, flag, |
291 | /*collectSymbols=*/false, |
292 | /*collectHostAssociatedSymbols=*/true); |
293 | } |
294 | } |
295 | |
296 | void DataSharingProcessor::collectDefaultSymbols() { |
297 | using DataSharingAttribute = omp::clause::Default::DataSharingAttribute; |
298 | for (const omp::Clause &clause : clauses) { |
299 | if (const auto *defaultClause = |
300 | std::get_if<omp::clause::Default>(&clause.u)) { |
301 | if (defaultClause->v == DataSharingAttribute::Private) |
302 | collectSymbols(Fortran::semantics::Symbol::Flag::OmpPrivate); |
303 | else if (defaultClause->v == DataSharingAttribute::Firstprivate) |
304 | collectSymbols(Fortran::semantics::Symbol::Flag::OmpFirstPrivate); |
305 | } |
306 | } |
307 | } |
308 | |
309 | void DataSharingProcessor::privatize( |
310 | mlir::omp::PrivateClauseOps *clauseOps, |
311 | llvm::SmallVectorImpl<const Fortran::semantics::Symbol *> *privateSyms) { |
312 | for (const Fortran::semantics::Symbol *sym : privatizedSymbols) { |
313 | if (const auto *commonDet = |
314 | sym->detailsIf<Fortran::semantics::CommonBlockDetails>()) { |
315 | for (const auto &mem : commonDet->objects()) |
316 | doPrivatize(&*mem, clauseOps, privateSyms); |
317 | } else |
318 | doPrivatize(sym, clauseOps, privateSyms); |
319 | } |
320 | } |
321 | |
322 | void DataSharingProcessor::copyLastPrivatize(mlir::Operation *op) { |
323 | insertLastPrivateCompare(op); |
324 | for (const Fortran::semantics::Symbol *sym : privatizedSymbols) |
325 | if (const auto *commonDet = |
326 | sym->detailsIf<Fortran::semantics::CommonBlockDetails>()) { |
327 | for (const auto &mem : commonDet->objects()) { |
328 | copyLastPrivateSymbol(&*mem, &lastPrivIP); |
329 | } |
330 | } else { |
331 | copyLastPrivateSymbol(sym, &lastPrivIP); |
332 | } |
333 | } |
334 | |
335 | void DataSharingProcessor::defaultPrivatize( |
336 | mlir::omp::PrivateClauseOps *clauseOps, |
337 | llvm::SmallVectorImpl<const Fortran::semantics::Symbol *> *privateSyms) { |
338 | for (const Fortran::semantics::Symbol *sym : defaultSymbols) { |
339 | if (!Fortran::semantics::IsProcedure(*sym) && |
340 | !sym->GetUltimate().has<Fortran::semantics::DerivedTypeDetails>() && |
341 | !sym->GetUltimate().has<Fortran::semantics::NamelistDetails>() && |
342 | !symbolsInNestedRegions.contains(sym) && |
343 | !symbolsInParentRegions.contains(sym) && |
344 | !privatizedSymbols.contains(sym)) |
345 | doPrivatize(sym, clauseOps, privateSyms); |
346 | } |
347 | } |
348 | |
349 | void DataSharingProcessor::doPrivatize( |
350 | const Fortran::semantics::Symbol *sym, |
351 | mlir::omp::PrivateClauseOps *clauseOps, |
352 | llvm::SmallVectorImpl<const Fortran::semantics::Symbol *> *privateSyms) { |
353 | if (!useDelayedPrivatization) { |
354 | cloneSymbol(sym); |
355 | copyFirstPrivateSymbol(sym); |
356 | return; |
357 | } |
358 | |
359 | Fortran::lower::SymbolBox hsb = converter.lookupOneLevelUpSymbol(*sym); |
360 | assert(hsb && "Host symbol box not found" ); |
361 | |
362 | mlir::Type symType = hsb.getAddr().getType(); |
363 | mlir::Location symLoc = hsb.getAddr().getLoc(); |
364 | std::string privatizerName = sym->name().ToString() + ".privatizer" ; |
365 | bool isFirstPrivate = |
366 | sym->test(Fortran::semantics::Symbol::Flag::OmpFirstPrivate); |
367 | |
368 | mlir::omp::PrivateClauseOp privatizerOp = [&]() { |
369 | auto moduleOp = firOpBuilder.getModule(); |
370 | auto uniquePrivatizerName = fir::getTypeAsString( |
371 | symType, converter.getKindMap(), |
372 | converter.mangleName(*sym) + |
373 | (isFirstPrivate ? "_firstprivate" : "_private" )); |
374 | |
375 | if (auto existingPrivatizer = |
376 | moduleOp.lookupSymbol<mlir::omp::PrivateClauseOp>( |
377 | uniquePrivatizerName)) |
378 | return existingPrivatizer; |
379 | |
380 | mlir::OpBuilder::InsertionGuard guard(firOpBuilder); |
381 | firOpBuilder.setInsertionPoint(&moduleOp.getBodyRegion().front(), |
382 | moduleOp.getBodyRegion().front().begin()); |
383 | auto result = firOpBuilder.create<mlir::omp::PrivateClauseOp>( |
384 | symLoc, uniquePrivatizerName, symType, |
385 | isFirstPrivate ? mlir::omp::DataSharingClauseType::FirstPrivate |
386 | : mlir::omp::DataSharingClauseType::Private); |
387 | fir::ExtendedValue symExV = converter.getSymbolExtendedValue(*sym); |
388 | |
389 | symTable->pushScope(); |
390 | |
391 | // Populate the `alloc` region. |
392 | { |
393 | mlir::Region &allocRegion = result.getAllocRegion(); |
394 | mlir::Block *allocEntryBlock = firOpBuilder.createBlock( |
395 | &allocRegion, /*insertPt=*/{}, symType, symLoc); |
396 | |
397 | firOpBuilder.setInsertionPointToEnd(allocEntryBlock); |
398 | symTable->addSymbol(*sym, |
399 | fir::substBase(symExV, allocRegion.getArgument(0))); |
400 | symTable->pushScope(); |
401 | cloneSymbol(sym); |
402 | firOpBuilder.create<mlir::omp::YieldOp>( |
403 | hsb.getAddr().getLoc(), |
404 | symTable->shallowLookupSymbol(*sym).getAddr()); |
405 | symTable->popScope(); |
406 | } |
407 | |
408 | // Populate the `copy` region if this is a `firstprivate`. |
409 | if (isFirstPrivate) { |
410 | mlir::Region ©Region = result.getCopyRegion(); |
411 | // First block argument corresponding to the original/host value while |
412 | // second block argument corresponding to the privatized value. |
413 | mlir::Block *copyEntryBlock = firOpBuilder.createBlock( |
414 | ©Region, /*insertPt=*/{}, {symType, symType}, {symLoc, symLoc}); |
415 | firOpBuilder.setInsertionPointToEnd(copyEntryBlock); |
416 | symTable->addSymbol(*sym, |
417 | fir::substBase(symExV, copyRegion.getArgument(0)), |
418 | /*force=*/true); |
419 | symTable->pushScope(); |
420 | symTable->addSymbol(*sym, |
421 | fir::substBase(symExV, copyRegion.getArgument(1))); |
422 | auto ip = firOpBuilder.saveInsertionPoint(); |
423 | copyFirstPrivateSymbol(sym, &ip); |
424 | |
425 | firOpBuilder.create<mlir::omp::YieldOp>( |
426 | hsb.getAddr().getLoc(), |
427 | symTable->shallowLookupSymbol(*sym).getAddr()); |
428 | symTable->popScope(); |
429 | } |
430 | |
431 | symTable->popScope(); |
432 | return result; |
433 | }(); |
434 | |
435 | if (clauseOps) { |
436 | clauseOps->privatizers.push_back(mlir::SymbolRefAttr::get(privatizerOp)); |
437 | clauseOps->privateVars.push_back(hsb.getAddr()); |
438 | } |
439 | |
440 | if (privateSyms) |
441 | privateSyms->push_back(Elt: sym); |
442 | } |
443 | |
444 | } // namespace omp |
445 | } // namespace lower |
446 | } // namespace Fortran |
447 | |