| 1 | //===-- DataSharingProcessor.cpp --------------------------------*- C++ -*-===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // Coding style: https://mlir.llvm.org/getting_started/DeveloperGuide/ |
| 10 | // |
| 11 | //===----------------------------------------------------------------------===// |
| 12 | |
| 13 | #include "DataSharingProcessor.h" |
| 14 | |
| 15 | #include "Utils.h" |
| 16 | #include "flang/Lower/ConvertVariable.h" |
| 17 | #include "flang/Lower/PFTBuilder.h" |
| 18 | #include "flang/Lower/Support/PrivateReductionUtils.h" |
| 19 | #include "flang/Lower/Support/Utils.h" |
| 20 | #include "flang/Lower/SymbolMap.h" |
| 21 | #include "flang/Optimizer/Builder/BoxValue.h" |
| 22 | #include "flang/Optimizer/Builder/HLFIRTools.h" |
| 23 | #include "flang/Optimizer/Builder/Todo.h" |
| 24 | #include "flang/Optimizer/Dialect/FIROps.h" |
| 25 | #include "flang/Optimizer/HLFIR/HLFIRDialect.h" |
| 26 | #include "flang/Optimizer/HLFIR/HLFIROps.h" |
| 27 | #include "flang/Semantics/attr.h" |
| 28 | #include "flang/Semantics/tools.h" |
| 29 | |
| 30 | namespace Fortran { |
| 31 | namespace lower { |
| 32 | namespace omp { |
| 33 | bool DataSharingProcessor::OMPConstructSymbolVisitor::isSymbolDefineBy( |
| 34 | const semantics::Symbol *symbol, lower::pft::Evaluation &eval) const { |
| 35 | return eval.visit( |
| 36 | common::visitors{[&](const parser::OpenMPConstruct &functionParserNode) { |
| 37 | return symDefMap.count(symbol) && |
| 38 | symDefMap.at(symbol) == &functionParserNode; |
| 39 | }, |
| 40 | [](const auto &functionParserNode) { return false; }}); |
| 41 | } |
| 42 | |
| 43 | DataSharingProcessor::DataSharingProcessor( |
| 44 | lower::AbstractConverter &converter, semantics::SemanticsContext &semaCtx, |
| 45 | const List<Clause> &clauses, lower::pft::Evaluation &eval, |
| 46 | bool shouldCollectPreDeterminedSymbols, bool useDelayedPrivatization, |
| 47 | lower::SymMap &symTable) |
| 48 | : converter(converter), semaCtx(semaCtx), |
| 49 | firOpBuilder(converter.getFirOpBuilder()), clauses(clauses), eval(eval), |
| 50 | shouldCollectPreDeterminedSymbols(shouldCollectPreDeterminedSymbols), |
| 51 | useDelayedPrivatization(useDelayedPrivatization), symTable(symTable), |
| 52 | visitor() { |
| 53 | eval.visit([&](const auto &functionParserNode) { |
| 54 | parser::Walk(functionParserNode, visitor); |
| 55 | }); |
| 56 | } |
| 57 | |
| 58 | DataSharingProcessor::DataSharingProcessor(lower::AbstractConverter &converter, |
| 59 | semantics::SemanticsContext &semaCtx, |
| 60 | lower::pft::Evaluation &eval, |
| 61 | bool useDelayedPrivatization, |
| 62 | lower::SymMap &symTable) |
| 63 | : DataSharingProcessor(converter, semaCtx, {}, eval, |
| 64 | /*shouldCollectPreDeterminedSymols=*/false, |
| 65 | useDelayedPrivatization, symTable) {} |
| 66 | |
| 67 | void DataSharingProcessor::processStep1( |
| 68 | mlir::omp::PrivateClauseOps *clauseOps) { |
| 69 | collectSymbolsForPrivatization(); |
| 70 | collectDefaultSymbols(); |
| 71 | collectImplicitSymbols(); |
| 72 | collectPreDeterminedSymbols(); |
| 73 | |
| 74 | privatize(clauseOps); |
| 75 | |
| 76 | insertBarrier(clauseOps); |
| 77 | } |
| 78 | |
| 79 | void DataSharingProcessor::processStep2(mlir::Operation *op, bool isLoop) { |
| 80 | // 'sections' lastprivate is handled by genOMP() |
| 81 | if (mlir::isa<mlir::omp::SectionOp>(op)) |
| 82 | return; |
| 83 | if (!mlir::isa<mlir::omp::SectionsOp>(op)) { |
| 84 | mlir::OpBuilder::InsertionGuard guard(firOpBuilder); |
| 85 | copyLastPrivatize(op); |
| 86 | } |
| 87 | |
| 88 | if (isLoop) { |
| 89 | // push deallocs out of the loop |
| 90 | firOpBuilder.setInsertionPointAfter(op); |
| 91 | insertDeallocs(); |
| 92 | } else { |
| 93 | mlir::OpBuilder::InsertionGuard guard(firOpBuilder); |
| 94 | insertDeallocs(); |
| 95 | } |
| 96 | } |
| 97 | |
| 98 | void DataSharingProcessor::insertDeallocs() { |
| 99 | for (const semantics::Symbol *sym : allPrivatizedSymbols) |
| 100 | if (semantics::IsAllocatable(sym->GetUltimate())) { |
| 101 | if (!useDelayedPrivatization) { |
| 102 | converter.createHostAssociateVarCloneDealloc(*sym); |
| 103 | continue; |
| 104 | } |
| 105 | // For delayed privatization deallocs are created by |
| 106 | // populateByRefInitAndCleanupRegions |
| 107 | } |
| 108 | } |
| 109 | |
| 110 | void DataSharingProcessor::cloneSymbol(const semantics::Symbol *sym) { |
| 111 | bool isFirstPrivate = sym->test(semantics::Symbol::Flag::OmpFirstPrivate); |
| 112 | |
| 113 | // If we are doing eager-privatization on a symbol created using delayed |
| 114 | // privatization there could be incompatible types here e.g. |
| 115 | // fir.ref<fir.box<fir.array<>>> |
| 116 | bool success = [&]() -> bool { |
| 117 | const auto *details = |
| 118 | sym->detailsIf<Fortran::semantics::HostAssocDetails>(); |
| 119 | assert(details && "No host-association found" ); |
| 120 | const Fortran::semantics::Symbol &hsym = details->symbol(); |
| 121 | mlir::Value addr = converter.getSymbolAddress(hsym); |
| 122 | |
| 123 | if (auto refTy = mlir::dyn_cast<fir::ReferenceType>(addr.getType())) { |
| 124 | if (auto boxTy = mlir::dyn_cast<fir::BoxType>(refTy.getElementType())) { |
| 125 | if (auto arrayTy = |
| 126 | mlir::dyn_cast<fir::SequenceType>(boxTy.getElementType())) { |
| 127 | // FirConverter/fir::ExtendedValue considers all references to boxes |
| 128 | // as mutable boxes. Outside of OpenMP it doesn't make sense to have a |
| 129 | // mutable box of an array. Work around this here by loading the |
| 130 | // reference so it is a normal boxed array. |
| 131 | fir::FirOpBuilder &builder = converter.getFirOpBuilder(); |
| 132 | mlir::Location loc = converter.genLocation(hsym.name()); |
| 133 | fir::ExtendedValue hexv = converter.getSymbolExtendedValue(hsym); |
| 134 | |
| 135 | llvm::SmallVector<mlir::Value> extents = |
| 136 | fir::factory::getExtents(loc, builder, hexv); |
| 137 | |
| 138 | // TODO: uniqName, name |
| 139 | mlir::Value allocVal = |
| 140 | builder.allocateLocal(loc, arrayTy, /*uniqName=*/"" , |
| 141 | /*name=*/"" , extents, /*typeParams=*/{}, |
| 142 | sym->GetUltimate().attrs().test( |
| 143 | Fortran::semantics::Attr::TARGET)); |
| 144 | mlir::Value shape = builder.genShape(loc, extents); |
| 145 | mlir::Value box = builder.createBox(loc, boxTy, allocVal, shape, |
| 146 | nullptr, {}, nullptr); |
| 147 | |
| 148 | // This can't be a CharArrayBoxValue because otherwise |
| 149 | // boxTy.getElementType() would be a character type. |
| 150 | // Assume the array element type isn't polymorphic because we are |
| 151 | // privatizing. |
| 152 | fir::ExtendedValue newExv = fir::ArrayBoxValue{box, extents}; |
| 153 | |
| 154 | converter.bindSymbol(*sym, newExv); |
| 155 | return true; |
| 156 | } |
| 157 | } |
| 158 | } |
| 159 | |
| 160 | // Normal case: |
| 161 | return converter.createHostAssociateVarClone( |
| 162 | *sym, /*skipDefaultInit=*/isFirstPrivate); |
| 163 | }(); |
| 164 | (void)success; |
| 165 | assert(success && "Privatization failed due to existing binding" ); |
| 166 | |
| 167 | // Initialize clone from original object if it has any allocatable member. |
| 168 | auto needInitClone = [&] { |
| 169 | if (isFirstPrivate) |
| 170 | return false; |
| 171 | |
| 172 | SymbolBox sb = symTable.lookupSymbol(sym); |
| 173 | assert(sb); |
| 174 | mlir::Value addr = sb.getAddr(); |
| 175 | assert(addr); |
| 176 | return !fir::isPointerType(addr.getType()) && |
| 177 | hlfir::mayHaveAllocatableComponent(addr.getType()); |
| 178 | }; |
| 179 | |
| 180 | if (needInitClone()) { |
| 181 | Fortran::lower::initializeCloneAtRuntime(converter, *sym, symTable); |
| 182 | mightHaveReadHostSym.insert(sym); |
| 183 | } |
| 184 | } |
| 185 | |
| 186 | void DataSharingProcessor::copyFirstPrivateSymbol( |
| 187 | const semantics::Symbol *sym, mlir::OpBuilder::InsertPoint *copyAssignIP) { |
| 188 | if (sym->test(semantics::Symbol::Flag::OmpFirstPrivate) || |
| 189 | sym->test(semantics::Symbol::Flag::LocalityLocalInit)) |
| 190 | converter.copyHostAssociateVar(*sym, copyAssignIP); |
| 191 | } |
| 192 | |
| 193 | void DataSharingProcessor::copyLastPrivateSymbol( |
| 194 | const semantics::Symbol *sym, mlir::OpBuilder::InsertPoint *lastPrivIP) { |
| 195 | if (sym->test(semantics::Symbol::Flag::OmpLastPrivate)) |
| 196 | converter.copyHostAssociateVar(*sym, lastPrivIP, /*hostIsSource=*/false); |
| 197 | } |
| 198 | |
| 199 | void DataSharingProcessor::collectOmpObjectListSymbol( |
| 200 | const omp::ObjectList &objects, |
| 201 | llvm::SetVector<const semantics::Symbol *> &symbolSet) { |
| 202 | for (const omp::Object &object : objects) |
| 203 | symbolSet.insert(object.sym()); |
| 204 | } |
| 205 | |
| 206 | void DataSharingProcessor::collectSymbolsForPrivatization() { |
| 207 | for (const omp::Clause &clause : clauses) { |
| 208 | if (const auto &privateClause = |
| 209 | std::get_if<omp::clause::Private>(&clause.u)) { |
| 210 | collectOmpObjectListSymbol(privateClause->v, explicitlyPrivatizedSymbols); |
| 211 | } else if (const auto &firstPrivateClause = |
| 212 | std::get_if<omp::clause::Firstprivate>(&clause.u)) { |
| 213 | collectOmpObjectListSymbol(firstPrivateClause->v, |
| 214 | explicitlyPrivatizedSymbols); |
| 215 | } else if (const auto &lastPrivateClause = |
| 216 | std::get_if<omp::clause::Lastprivate>(&clause.u)) { |
| 217 | lastprivateModifierNotSupported(*lastPrivateClause, |
| 218 | converter.getCurrentLocation()); |
| 219 | const ObjectList &objects = std::get<ObjectList>(lastPrivateClause->t); |
| 220 | collectOmpObjectListSymbol(objects, explicitlyPrivatizedSymbols); |
| 221 | } |
| 222 | } |
| 223 | |
| 224 | // TODO For common blocks, add the underlying objects within the block. Doing |
| 225 | // so, we won't need to explicitely handle block objects (or forget to do |
| 226 | // so). |
| 227 | for (auto *sym : explicitlyPrivatizedSymbols) |
| 228 | if (!sym->test(Fortran::semantics::Symbol::Flag::OmpLinear)) |
| 229 | allPrivatizedSymbols.insert(sym); |
| 230 | } |
| 231 | |
| 232 | bool DataSharingProcessor::needBarrier() { |
| 233 | // Emit implicit barrier to synchronize threads and avoid data races on |
| 234 | // initialization of firstprivate variables and post-update of lastprivate |
| 235 | // variables. |
| 236 | // Emit implicit barrier for linear clause in the OpenMPIRBuilder. |
| 237 | for (const semantics::Symbol *sym : allPrivatizedSymbols) { |
| 238 | if (sym->test(semantics::Symbol::Flag::OmpLastPrivate) && |
| 239 | (sym->test(semantics::Symbol::Flag::OmpFirstPrivate) || |
| 240 | mightHaveReadHostSym.contains(sym))) |
| 241 | return true; |
| 242 | } |
| 243 | return false; |
| 244 | } |
| 245 | |
| 246 | void DataSharingProcessor::insertBarrier( |
| 247 | mlir::omp::PrivateClauseOps *clauseOps) { |
| 248 | if (!needBarrier()) |
| 249 | return; |
| 250 | |
| 251 | if (useDelayedPrivatization) { |
| 252 | if (clauseOps) |
| 253 | clauseOps->privateNeedsBarrier = |
| 254 | mlir::UnitAttr::get(&converter.getMLIRContext()); |
| 255 | } else { |
| 256 | firOpBuilder.create<mlir::omp::BarrierOp>(converter.getCurrentLocation()); |
| 257 | } |
| 258 | } |
| 259 | |
| 260 | void DataSharingProcessor::insertLastPrivateCompare(mlir::Operation *op) { |
| 261 | mlir::omp::LoopNestOp loopOp; |
| 262 | if (auto wrapper = mlir::dyn_cast<mlir::omp::LoopWrapperInterface>(op)) |
| 263 | loopOp = mlir::cast<mlir::omp::LoopNestOp>(wrapper.getWrappedLoop()); |
| 264 | |
| 265 | mlir::OpBuilder::InsertionGuard guard(firOpBuilder); |
| 266 | bool hasLastPrivate = [&]() { |
| 267 | for (const semantics::Symbol *sym : allPrivatizedSymbols) { |
| 268 | if (const auto *commonDet = |
| 269 | sym->detailsIf<semantics::CommonBlockDetails>()) { |
| 270 | for (const auto &mem : commonDet->objects()) |
| 271 | if (mem->test(semantics::Symbol::Flag::OmpLastPrivate)) |
| 272 | return true; |
| 273 | } else if (sym->test(semantics::Symbol::Flag::OmpLastPrivate)) |
| 274 | return true; |
| 275 | } |
| 276 | |
| 277 | return false; |
| 278 | }(); |
| 279 | |
| 280 | if (!hasLastPrivate) |
| 281 | return; |
| 282 | |
| 283 | if (mlir::isa<mlir::omp::WsloopOp>(op) || mlir::isa<mlir::omp::SimdOp>(op)) { |
| 284 | mlir::omp::LoopRelatedClauseOps result; |
| 285 | llvm::SmallVector<const semantics::Symbol *> iv; |
| 286 | collectLoopRelatedInfo(converter, converter.getCurrentLocation(), eval, |
| 287 | clauses, result, iv); |
| 288 | |
| 289 | // Update the original variable just before exiting the worksharing |
| 290 | // loop. Conversion as follows: |
| 291 | // |
| 292 | // omp.wsloop / omp.simd { omp.wsloop / omp.simd { |
| 293 | // omp.loop_nest { omp.loop_nest { |
| 294 | // ... ... |
| 295 | // store ===> store |
| 296 | // omp.yield %v = arith.addi %iv, %step |
| 297 | // } %cmp = %step < 0 ? %v < %ub : %v > %ub |
| 298 | // } fir.if %cmp { |
| 299 | // fir.store %v to %loopIV |
| 300 | // ^%lpv_update_blk: |
| 301 | // } |
| 302 | // omp.yield |
| 303 | // } |
| 304 | // } |
| 305 | mlir::Location loc = loopOp.getLoc(); |
| 306 | mlir::Operation *lastOper = loopOp.getRegion().back().getTerminator(); |
| 307 | firOpBuilder.setInsertionPoint(lastOper); |
| 308 | |
| 309 | mlir::Value cmpOp; |
| 310 | llvm::SmallVector<mlir::Value> vs; |
| 311 | vs.reserve(loopOp.getIVs().size()); |
| 312 | for (auto [iv, ub, step] : llvm::zip_equal( |
| 313 | loopOp.getIVs(), result.loopUpperBounds, result.loopSteps)) { |
| 314 | // v = iv + step |
| 315 | // cmp = step < 0 ? v < ub : v > ub |
| 316 | mlir::Value v = firOpBuilder.create<mlir::arith::AddIOp>(loc, iv, step); |
| 317 | vs.push_back(v); |
| 318 | mlir::Value zero = |
| 319 | firOpBuilder.createIntegerConstant(loc, step.getType(), 0); |
| 320 | mlir::Value negativeStep = firOpBuilder.create<mlir::arith::CmpIOp>( |
| 321 | loc, mlir::arith::CmpIPredicate::slt, step, zero); |
| 322 | mlir::Value vLT = firOpBuilder.create<mlir::arith::CmpIOp>( |
| 323 | loc, mlir::arith::CmpIPredicate::slt, v, ub); |
| 324 | mlir::Value vGT = firOpBuilder.create<mlir::arith::CmpIOp>( |
| 325 | loc, mlir::arith::CmpIPredicate::sgt, v, ub); |
| 326 | mlir::Value icmpOp = firOpBuilder.create<mlir::arith::SelectOp>( |
| 327 | loc, negativeStep, vLT, vGT); |
| 328 | |
| 329 | if (cmpOp) |
| 330 | cmpOp = firOpBuilder.create<mlir::arith::AndIOp>(loc, cmpOp, icmpOp); |
| 331 | else |
| 332 | cmpOp = icmpOp; |
| 333 | } |
| 334 | |
| 335 | auto ifOp = firOpBuilder.create<fir::IfOp>(loc, cmpOp, /*else*/ false); |
| 336 | firOpBuilder.setInsertionPointToStart(&ifOp.getThenRegion().front()); |
| 337 | for (auto [v, loopIV] : llvm::zip_equal(vs, loopIVs)) { |
| 338 | hlfir::Entity loopIVEntity{loopIV}; |
| 339 | loopIVEntity = |
| 340 | hlfir::derefPointersAndAllocatables(loc, firOpBuilder, loopIVEntity); |
| 341 | firOpBuilder.create<hlfir::AssignOp>(loc, v, loopIVEntity); |
| 342 | } |
| 343 | lastPrivIP = firOpBuilder.saveInsertionPoint(); |
| 344 | } else if (mlir::isa<mlir::omp::SectionsOp>(op)) { |
| 345 | // Already handled by genOMP() |
| 346 | } else { |
| 347 | TODO(converter.getCurrentLocation(), |
| 348 | "lastprivate clause in constructs other than " |
| 349 | "simd/worksharing-loop" ); |
| 350 | } |
| 351 | } |
| 352 | |
| 353 | static const parser::CharBlock * |
| 354 | getSource(const semantics::SemanticsContext &semaCtx, |
| 355 | const lower::pft::Evaluation &eval) { |
| 356 | const parser::CharBlock *source = nullptr; |
| 357 | |
| 358 | auto ompConsVisit = [&](const parser::OpenMPConstruct &x) { |
| 359 | std::visit( |
| 360 | common::visitors{ |
| 361 | [&](const parser::OpenMPSectionsConstruct &x) { |
| 362 | source = &std::get<0>(x.t).source; |
| 363 | }, |
| 364 | [&](const parser::OpenMPLoopConstruct &x) { |
| 365 | source = &std::get<0>(x.t).source; |
| 366 | }, |
| 367 | [&](const parser::OpenMPBlockConstruct &x) { |
| 368 | source = &std::get<0>(x.t).source; |
| 369 | }, |
| 370 | [&](const parser::OpenMPCriticalConstruct &x) { |
| 371 | source = &std::get<0>(x.t).source; |
| 372 | }, |
| 373 | [&](const parser::OpenMPAtomicConstruct &x) { |
| 374 | source = &std::get<parser::OmpDirectiveSpecification>(x.t).source; |
| 375 | }, |
| 376 | [&](const auto &x) { source = &x.source; }, |
| 377 | }, |
| 378 | x.u); |
| 379 | }; |
| 380 | |
| 381 | eval.visit(common::visitors{ |
| 382 | [&](const parser::OpenMPConstruct &x) { ompConsVisit(x); }, |
| 383 | [&](const parser::OpenMPDeclarativeConstruct &x) { source = &x.source; }, |
| 384 | [&](const parser::OmpEndLoopDirective &x) { source = &x.source; }, |
| 385 | [&](const auto &x) {}, |
| 386 | }); |
| 387 | |
| 388 | return source; |
| 389 | } |
| 390 | |
| 391 | bool DataSharingProcessor::isOpenMPPrivatizingConstruct( |
| 392 | const parser::OpenMPConstruct &omp) { |
| 393 | return common::visit( |
| 394 | [](auto &&s) { |
| 395 | using BareS = llvm::remove_cvref_t<decltype(s)>; |
| 396 | return std::is_same_v<BareS, parser::OpenMPBlockConstruct> || |
| 397 | std::is_same_v<BareS, parser::OpenMPLoopConstruct> || |
| 398 | std::is_same_v<BareS, parser::OpenMPSectionsConstruct>; |
| 399 | }, |
| 400 | omp.u); |
| 401 | } |
| 402 | |
| 403 | bool DataSharingProcessor::isOpenMPPrivatizingEvaluation( |
| 404 | const pft::Evaluation &eval) const { |
| 405 | return eval.visit([](auto &&s) { |
| 406 | using BareS = llvm::remove_cvref_t<decltype(s)>; |
| 407 | if constexpr (std::is_same_v<BareS, parser::OpenMPConstruct>) { |
| 408 | return isOpenMPPrivatizingConstruct(s); |
| 409 | } else { |
| 410 | return false; |
| 411 | } |
| 412 | }); |
| 413 | } |
| 414 | |
| 415 | void DataSharingProcessor::collectSymbolsInNestedRegions( |
| 416 | lower::pft::Evaluation &eval, semantics::Symbol::Flag flag, |
| 417 | llvm::SetVector<const semantics::Symbol *> &symbolsInNestedRegions) { |
| 418 | if (!eval.hasNestedEvaluations()) |
| 419 | return; |
| 420 | for (pft::Evaluation &nestedEval : eval.getNestedEvaluations()) { |
| 421 | if (isOpenMPPrivatizingEvaluation(nestedEval)) { |
| 422 | converter.collectSymbolSet(nestedEval, symbolsInNestedRegions, flag, |
| 423 | /*collectSymbols=*/true, |
| 424 | /*collectHostAssociatedSymbols=*/false); |
| 425 | } else { |
| 426 | // Recursively look for OpenMP constructs within `nestedEval`'s region |
| 427 | collectSymbolsInNestedRegions(nestedEval, flag, symbolsInNestedRegions); |
| 428 | } |
| 429 | } |
| 430 | } |
| 431 | |
| 432 | // Collect symbols to be default privatized in two steps. |
| 433 | // In step 1, collect all symbols in `eval` that match `flag` into |
| 434 | // `defaultSymbols`. In step 2, for nested constructs (if any), if and only if |
| 435 | // the nested construct is an OpenMP construct, collect those nested |
| 436 | // symbols skipping host associated symbols into `symbolsInNestedRegions`. |
| 437 | // Later, in current context, all symbols in the set |
| 438 | // `defaultSymbols` - `symbolsInNestedRegions` will be privatized. |
| 439 | void DataSharingProcessor::collectSymbols( |
| 440 | semantics::Symbol::Flag flag, |
| 441 | llvm::SetVector<const semantics::Symbol *> &symbols) { |
| 442 | // Collect all scopes associated with 'eval'. |
| 443 | llvm::SetVector<const semantics::Scope *> clauseScopes; |
| 444 | std::function<void(const semantics::Scope *)> collectScopes = |
| 445 | [&](const semantics::Scope *scope) { |
| 446 | clauseScopes.insert(scope); |
| 447 | for (const semantics::Scope &child : scope->children()) |
| 448 | collectScopes(&child); |
| 449 | }; |
| 450 | const parser::CharBlock *source = |
| 451 | clauses.empty() ? getSource(semaCtx, eval) : &clauses.front().source; |
| 452 | const semantics::Scope *curScope = nullptr; |
| 453 | if (source && !source->empty()) { |
| 454 | curScope = &semaCtx.FindScope(*source); |
| 455 | collectScopes(curScope); |
| 456 | } |
| 457 | // Collect all symbols referenced in the evaluation being processed, |
| 458 | // that matches 'flag'. |
| 459 | llvm::SetVector<const semantics::Symbol *> allSymbols; |
| 460 | converter.collectSymbolSet(eval, allSymbols, flag, |
| 461 | /*collectSymbols=*/true, |
| 462 | /*collectHostAssociatedSymbols=*/true); |
| 463 | |
| 464 | llvm::SetVector<const semantics::Symbol *> symbolsInNestedRegions; |
| 465 | collectSymbolsInNestedRegions(eval, flag, symbolsInNestedRegions); |
| 466 | |
| 467 | for (auto *symbol : allSymbols) |
| 468 | if (visitor.isSymbolDefineBy(symbol, eval)) |
| 469 | symbolsInNestedRegions.remove(symbol); |
| 470 | |
| 471 | // Filter-out symbols that must not be privatized. |
| 472 | bool collectImplicit = flag == semantics::Symbol::Flag::OmpImplicit; |
| 473 | bool collectPreDetermined = flag == semantics::Symbol::Flag::OmpPreDetermined; |
| 474 | |
| 475 | auto isPrivatizable = [](const semantics::Symbol &sym) -> bool { |
| 476 | return !semantics::IsProcedure(sym) && |
| 477 | !sym.GetUltimate().has<semantics::DerivedTypeDetails>() && |
| 478 | !sym.GetUltimate().has<semantics::NamelistDetails>() && |
| 479 | !semantics::IsImpliedDoIndex(sym.GetUltimate()) && |
| 480 | !semantics::IsStmtFunction(sym); |
| 481 | }; |
| 482 | |
| 483 | auto shouldCollectSymbol = [&](const semantics::Symbol *sym) { |
| 484 | if (collectImplicit) |
| 485 | return sym->test(semantics::Symbol::Flag::OmpImplicit); |
| 486 | |
| 487 | if (collectPreDetermined) |
| 488 | return sym->test(semantics::Symbol::Flag::OmpPreDetermined); |
| 489 | |
| 490 | return !sym->test(semantics::Symbol::Flag::OmpImplicit) && |
| 491 | !sym->test(semantics::Symbol::Flag::OmpPreDetermined); |
| 492 | }; |
| 493 | |
| 494 | for (const auto *sym : allSymbols) { |
| 495 | assert(curScope && "couldn't find current scope" ); |
| 496 | if (isPrivatizable(*sym) && !symbolsInNestedRegions.contains(sym) && |
| 497 | !explicitlyPrivatizedSymbols.contains(sym) && |
| 498 | shouldCollectSymbol(sym) && clauseScopes.contains(&sym->owner())) { |
| 499 | allPrivatizedSymbols.insert(sym); |
| 500 | symbols.insert(sym); |
| 501 | } |
| 502 | } |
| 503 | } |
| 504 | |
| 505 | void DataSharingProcessor::collectDefaultSymbols() { |
| 506 | using DataSharingAttribute = omp::clause::Default::DataSharingAttribute; |
| 507 | for (const omp::Clause &clause : clauses) { |
| 508 | if (const auto *defaultClause = |
| 509 | std::get_if<omp::clause::Default>(&clause.u)) { |
| 510 | if (defaultClause->v == DataSharingAttribute::Private) |
| 511 | collectSymbols(semantics::Symbol::Flag::OmpPrivate, defaultSymbols); |
| 512 | else if (defaultClause->v == DataSharingAttribute::Firstprivate) |
| 513 | collectSymbols(semantics::Symbol::Flag::OmpFirstPrivate, |
| 514 | defaultSymbols); |
| 515 | } |
| 516 | } |
| 517 | } |
| 518 | |
| 519 | void DataSharingProcessor::collectImplicitSymbols() { |
| 520 | // There will be no implicit symbols when a default clause is present. |
| 521 | if (defaultSymbols.empty()) |
| 522 | collectSymbols(semantics::Symbol::Flag::OmpImplicit, implicitSymbols); |
| 523 | } |
| 524 | |
| 525 | void DataSharingProcessor::collectPreDeterminedSymbols() { |
| 526 | if (shouldCollectPreDeterminedSymbols) |
| 527 | collectSymbols(semantics::Symbol::Flag::OmpPreDetermined, |
| 528 | preDeterminedSymbols); |
| 529 | } |
| 530 | |
| 531 | void DataSharingProcessor::privatize(mlir::omp::PrivateClauseOps *clauseOps) { |
| 532 | for (const semantics::Symbol *sym : allPrivatizedSymbols) { |
| 533 | if (const auto *commonDet = |
| 534 | sym->detailsIf<semantics::CommonBlockDetails>()) { |
| 535 | for (const auto &mem : commonDet->objects()) |
| 536 | privatizeSymbol(&*mem, clauseOps); |
| 537 | } else |
| 538 | privatizeSymbol(sym, clauseOps); |
| 539 | } |
| 540 | } |
| 541 | |
| 542 | void DataSharingProcessor::copyLastPrivatize(mlir::Operation *op) { |
| 543 | insertLastPrivateCompare(op); |
| 544 | for (const semantics::Symbol *sym : allPrivatizedSymbols) |
| 545 | if (const auto *commonDet = |
| 546 | sym->detailsIf<semantics::CommonBlockDetails>()) { |
| 547 | for (const auto &mem : commonDet->objects()) { |
| 548 | copyLastPrivateSymbol(&*mem, &lastPrivIP); |
| 549 | } |
| 550 | } else { |
| 551 | copyLastPrivateSymbol(sym, &lastPrivIP); |
| 552 | } |
| 553 | } |
| 554 | |
| 555 | void DataSharingProcessor::privatizeSymbol( |
| 556 | const semantics::Symbol *symToPrivatize, |
| 557 | mlir::omp::PrivateClauseOps *clauseOps) { |
| 558 | if (!useDelayedPrivatization) { |
| 559 | cloneSymbol(sym: symToPrivatize); |
| 560 | copyFirstPrivateSymbol(symToPrivatize); |
| 561 | return; |
| 562 | } |
| 563 | |
| 564 | Fortran::lower::privatizeSymbol<mlir::omp::PrivateClauseOp, |
| 565 | mlir::omp::PrivateClauseOps>( |
| 566 | converter, firOpBuilder, symTable, allPrivatizedSymbols, |
| 567 | mightHaveReadHostSym, symToPrivatize, clauseOps); |
| 568 | } |
| 569 | } // namespace omp |
| 570 | } // namespace lower |
| 571 | } // namespace Fortran |
| 572 | |