| 1 | //===-- DataSharingProcessor.cpp --------------------------------*- C++ -*-===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // Coding style: https://mlir.llvm.org/getting_started/DeveloperGuide/ |
| 10 | // |
| 11 | //===----------------------------------------------------------------------===// |
| 12 | |
| 13 | #include "DataSharingProcessor.h" |
| 14 | |
| 15 | #include "Utils.h" |
| 16 | #include "flang/Lower/ConvertVariable.h" |
| 17 | #include "flang/Lower/PFTBuilder.h" |
| 18 | #include "flang/Lower/Support/PrivateReductionUtils.h" |
| 19 | #include "flang/Lower/Support/Utils.h" |
| 20 | #include "flang/Lower/SymbolMap.h" |
| 21 | #include "flang/Optimizer/Builder/BoxValue.h" |
| 22 | #include "flang/Optimizer/Builder/HLFIRTools.h" |
| 23 | #include "flang/Optimizer/Builder/Todo.h" |
| 24 | #include "flang/Optimizer/Dialect/FIROps.h" |
| 25 | #include "flang/Optimizer/HLFIR/HLFIRDialect.h" |
| 26 | #include "flang/Optimizer/HLFIR/HLFIROps.h" |
| 27 | #include "flang/Semantics/attr.h" |
| 28 | #include "flang/Semantics/tools.h" |
| 29 | |
| 30 | namespace Fortran { |
| 31 | namespace lower { |
| 32 | namespace omp { |
| 33 | bool DataSharingProcessor::OMPConstructSymbolVisitor::isSymbolDefineBy( |
| 34 | const semantics::Symbol *symbol, lower::pft::Evaluation &eval) const { |
| 35 | return eval.visit( |
| 36 | common::visitors{[&](const parser::OpenMPConstruct &functionParserNode) { |
| 37 | return symDefMap.count(symbol) && |
| 38 | symDefMap.at(symbol) == &functionParserNode; |
| 39 | }, |
| 40 | [](const auto &functionParserNode) { return false; }}); |
| 41 | } |
| 42 | |
| 43 | DataSharingProcessor::DataSharingProcessor( |
| 44 | lower::AbstractConverter &converter, semantics::SemanticsContext &semaCtx, |
| 45 | const List<Clause> &clauses, lower::pft::Evaluation &eval, |
| 46 | bool shouldCollectPreDeterminedSymbols, bool useDelayedPrivatization, |
| 47 | lower::SymMap &symTable) |
| 48 | : converter(converter), semaCtx(semaCtx), |
| 49 | firOpBuilder(converter.getFirOpBuilder()), clauses(clauses), eval(eval), |
| 50 | shouldCollectPreDeterminedSymbols(shouldCollectPreDeterminedSymbols), |
| 51 | useDelayedPrivatization(useDelayedPrivatization), symTable(symTable), |
| 52 | visitor() { |
| 53 | eval.visit([&](const auto &functionParserNode) { |
| 54 | parser::Walk(functionParserNode, visitor); |
| 55 | }); |
| 56 | } |
| 57 | |
| 58 | DataSharingProcessor::DataSharingProcessor(lower::AbstractConverter &converter, |
| 59 | semantics::SemanticsContext &semaCtx, |
| 60 | lower::pft::Evaluation &eval, |
| 61 | bool useDelayedPrivatization, |
| 62 | lower::SymMap &symTable) |
| 63 | : DataSharingProcessor(converter, semaCtx, {}, eval, |
| 64 | /*shouldCollectPreDeterminedSymols=*/false, |
| 65 | useDelayedPrivatization, symTable) {} |
| 66 | |
| 67 | void DataSharingProcessor::processStep1( |
| 68 | mlir::omp::PrivateClauseOps *clauseOps) { |
| 69 | collectSymbolsForPrivatization(); |
| 70 | collectDefaultSymbols(); |
| 71 | collectImplicitSymbols(); |
| 72 | collectPreDeterminedSymbols(); |
| 73 | |
| 74 | privatize(clauseOps); |
| 75 | |
| 76 | insertBarrier(clauseOps); |
| 77 | } |
| 78 | |
| 79 | void DataSharingProcessor::processStep2(mlir::Operation *op, bool isLoop) { |
| 80 | // 'sections' lastprivate is handled by genOMP() |
| 81 | if (mlir::isa<mlir::omp::SectionOp>(op)) |
| 82 | return; |
| 83 | if (!mlir::isa<mlir::omp::SectionsOp>(op)) { |
| 84 | mlir::OpBuilder::InsertionGuard guard(firOpBuilder); |
| 85 | copyLastPrivatize(op); |
| 86 | } |
| 87 | |
| 88 | if (isLoop) { |
| 89 | // push deallocs out of the loop |
| 90 | firOpBuilder.setInsertionPointAfter(op); |
| 91 | insertDeallocs(); |
| 92 | } else { |
| 93 | mlir::OpBuilder::InsertionGuard guard(firOpBuilder); |
| 94 | insertDeallocs(); |
| 95 | } |
| 96 | } |
| 97 | |
| 98 | void DataSharingProcessor::insertDeallocs() { |
| 99 | for (const semantics::Symbol *sym : allPrivatizedSymbols) |
| 100 | if (semantics::IsAllocatable(sym->GetUltimate())) { |
| 101 | if (!useDelayedPrivatization) { |
| 102 | converter.createHostAssociateVarCloneDealloc(*sym); |
| 103 | continue; |
| 104 | } |
| 105 | // For delayed privatization deallocs are created by |
| 106 | // populateByRefInitAndCleanupRegions |
| 107 | } |
| 108 | } |
| 109 | |
| 110 | void DataSharingProcessor::cloneSymbol(const semantics::Symbol *sym) { |
| 111 | bool isFirstPrivate = sym->test(semantics::Symbol::Flag::OmpFirstPrivate); |
| 112 | |
| 113 | // If we are doing eager-privatization on a symbol created using delayed |
| 114 | // privatization there could be incompatible types here e.g. |
| 115 | // fir.ref<fir.box<fir.array<>>> |
| 116 | bool success = [&]() -> bool { |
| 117 | const auto *details = |
| 118 | sym->detailsIf<Fortran::semantics::HostAssocDetails>(); |
| 119 | assert(details && "No host-association found" ); |
| 120 | const Fortran::semantics::Symbol &hsym = details->symbol(); |
| 121 | mlir::Value addr = converter.getSymbolAddress(hsym); |
| 122 | |
| 123 | if (auto refTy = mlir::dyn_cast<fir::ReferenceType>(addr.getType())) { |
| 124 | if (auto boxTy = mlir::dyn_cast<fir::BoxType>(refTy.getElementType())) { |
| 125 | if (auto arrayTy = |
| 126 | mlir::dyn_cast<fir::SequenceType>(boxTy.getElementType())) { |
| 127 | // FirConverter/fir::ExtendedValue considers all references to boxes |
| 128 | // as mutable boxes. Outside of OpenMP it doesn't make sense to have a |
| 129 | // mutable box of an array. Work around this here by loading the |
| 130 | // reference so it is a normal boxed array. |
| 131 | fir::FirOpBuilder &builder = converter.getFirOpBuilder(); |
| 132 | mlir::Location loc = converter.genLocation(hsym.name()); |
| 133 | fir::ExtendedValue hexv = converter.getSymbolExtendedValue(hsym); |
| 134 | |
| 135 | llvm::SmallVector<mlir::Value> extents = |
| 136 | fir::factory::getExtents(loc, builder, hexv); |
| 137 | |
| 138 | // TODO: uniqName, name |
| 139 | mlir::Value allocVal = |
| 140 | builder.allocateLocal(loc, arrayTy, /*uniqName=*/"" , |
| 141 | /*name=*/"" , extents, /*typeParams=*/{}, |
| 142 | sym->GetUltimate().attrs().test( |
| 143 | Fortran::semantics::Attr::TARGET)); |
| 144 | mlir::Value shape = builder.genShape(loc, extents); |
| 145 | mlir::Value box = builder.createBox(loc, boxTy, allocVal, shape, |
| 146 | nullptr, {}, nullptr); |
| 147 | |
| 148 | // This can't be a CharArrayBoxValue because otherwise |
| 149 | // boxTy.getElementType() would be a character type. |
| 150 | // Assume the array element type isn't polymorphic because we are |
| 151 | // privatizing. |
| 152 | fir::ExtendedValue newExv = fir::ArrayBoxValue{box, extents}; |
| 153 | |
| 154 | converter.bindSymbol(*sym, newExv); |
| 155 | return true; |
| 156 | } |
| 157 | } |
| 158 | } |
| 159 | |
| 160 | // Normal case: |
| 161 | return converter.createHostAssociateVarClone( |
| 162 | *sym, /*skipDefaultInit=*/isFirstPrivate); |
| 163 | }(); |
| 164 | (void)success; |
| 165 | assert(success && "Privatization failed due to existing binding" ); |
| 166 | |
| 167 | // Initialize clone from original object if it has any allocatable member. |
| 168 | auto needInitClone = [&] { |
| 169 | if (isFirstPrivate) |
| 170 | return false; |
| 171 | |
| 172 | SymbolBox sb = symTable.lookupSymbol(sym); |
| 173 | assert(sb); |
| 174 | mlir::Value addr = sb.getAddr(); |
| 175 | assert(addr); |
| 176 | return !fir::isPointerType(addr.getType()) && |
| 177 | hlfir::mayHaveAllocatableComponent(addr.getType()); |
| 178 | }; |
| 179 | |
| 180 | if (needInitClone()) { |
| 181 | Fortran::lower::initializeCloneAtRuntime(converter, *sym, symTable); |
| 182 | mightHaveReadHostSym.insert(sym); |
| 183 | } |
| 184 | } |
| 185 | |
| 186 | void DataSharingProcessor::copyFirstPrivateSymbol( |
| 187 | const semantics::Symbol *sym, mlir::OpBuilder::InsertPoint *copyAssignIP) { |
| 188 | if (sym->test(semantics::Symbol::Flag::OmpFirstPrivate) || |
| 189 | sym->test(semantics::Symbol::Flag::LocalityLocalInit)) |
| 190 | converter.copyHostAssociateVar(*sym, copyAssignIP); |
| 191 | } |
| 192 | |
| 193 | void DataSharingProcessor::copyLastPrivateSymbol( |
| 194 | const semantics::Symbol *sym, mlir::OpBuilder::InsertPoint *lastPrivIP) { |
| 195 | if (sym->test(semantics::Symbol::Flag::OmpLastPrivate)) |
| 196 | converter.copyHostAssociateVar(*sym, lastPrivIP, /*hostIsSource=*/false); |
| 197 | } |
| 198 | |
| 199 | void DataSharingProcessor::collectOmpObjectListSymbol( |
| 200 | const omp::ObjectList &objects, |
| 201 | llvm::SetVector<const semantics::Symbol *> &symbolSet) { |
| 202 | for (const omp::Object &object : objects) |
| 203 | symbolSet.insert(object.sym()); |
| 204 | } |
| 205 | |
| 206 | void DataSharingProcessor::collectSymbolsForPrivatization() { |
| 207 | // Add checks here for exceptional cases where privatization is not |
| 208 | // needed and be deferred to a later phase (like OpenMP IRBuilder). |
| 209 | // Such cases are suggested to be clearly documented and explained |
| 210 | // instead of being silently skipped |
| 211 | auto isException = [&](const Fortran::semantics::Symbol *sym) -> bool { |
| 212 | // `OmpPreDetermined` symbols cannot be exceptions since |
| 213 | // their privatized symbols are heavily used in FIR. |
| 214 | if (sym->test(Fortran::semantics::Symbol::Flag::OmpPreDetermined)) |
| 215 | return false; |
| 216 | |
| 217 | // The handling of linear clause is deferred to the OpenMP |
| 218 | // IRBuilder which is responsible for all its aspects, |
| 219 | // including privatization. Privatizing linear variables at this point would |
| 220 | // cause the following structure: |
| 221 | // |
| 222 | // omp.op linear(%linear = %step : !fir.ref<type>) { |
| 223 | // Use %linear in this BB |
| 224 | // } |
| 225 | // |
| 226 | // to be changed to the following: |
| 227 | // |
| 228 | // omp. op linear(%linear = %step : !fir.ref<type>) |
| 229 | // private(%linear -> %arg0 : !fir.ref<i32>) { |
| 230 | // Declare and use %arg0 in this BB |
| 231 | // } |
| 232 | // |
| 233 | // The OpenMP IRBuilder needs to map the linear MLIR value |
| 234 | // (i.e. %linear) to its `uses` in the BB to correctly |
| 235 | // implement the functionalities of linear clause. However, |
| 236 | // privatizing here disallows the IRBuilder to |
| 237 | // draw a relation between %linear and %arg0. Hence skip. |
| 238 | if (sym->test(Fortran::semantics::Symbol::Flag::OmpLinear)) |
| 239 | return true; |
| 240 | return false; |
| 241 | }; |
| 242 | |
| 243 | for (const omp::Clause &clause : clauses) { |
| 244 | if (const auto &privateClause = |
| 245 | std::get_if<omp::clause::Private>(&clause.u)) { |
| 246 | collectOmpObjectListSymbol(privateClause->v, explicitlyPrivatizedSymbols); |
| 247 | } else if (const auto &firstPrivateClause = |
| 248 | std::get_if<omp::clause::Firstprivate>(&clause.u)) { |
| 249 | collectOmpObjectListSymbol(firstPrivateClause->v, |
| 250 | explicitlyPrivatizedSymbols); |
| 251 | } else if (const auto &lastPrivateClause = |
| 252 | std::get_if<omp::clause::Lastprivate>(&clause.u)) { |
| 253 | lastprivateModifierNotSupported(*lastPrivateClause, |
| 254 | converter.getCurrentLocation()); |
| 255 | const ObjectList &objects = std::get<ObjectList>(lastPrivateClause->t); |
| 256 | collectOmpObjectListSymbol(objects, explicitlyPrivatizedSymbols); |
| 257 | } |
| 258 | } |
| 259 | |
| 260 | // TODO For common blocks, add the underlying objects within the block. Doing |
| 261 | // so, we won't need to explicitly handle block objects (or forget to do |
| 262 | // so). |
| 263 | for (auto *sym : explicitlyPrivatizedSymbols) |
| 264 | if (!isException(sym)) |
| 265 | allPrivatizedSymbols.insert(sym); |
| 266 | } |
| 267 | |
| 268 | bool DataSharingProcessor::needBarrier() { |
| 269 | // Emit implicit barrier to synchronize threads and avoid data races on |
| 270 | // initialization of firstprivate variables and post-update of lastprivate |
| 271 | // variables. |
| 272 | // Emit implicit barrier for linear clause in the OpenMPIRBuilder. |
| 273 | for (const semantics::Symbol *sym : allPrivatizedSymbols) { |
| 274 | if (sym->test(semantics::Symbol::Flag::OmpLastPrivate) && |
| 275 | (sym->test(semantics::Symbol::Flag::OmpFirstPrivate) || |
| 276 | mightHaveReadHostSym.contains(sym))) |
| 277 | return true; |
| 278 | } |
| 279 | return false; |
| 280 | } |
| 281 | |
| 282 | void DataSharingProcessor::insertBarrier( |
| 283 | mlir::omp::PrivateClauseOps *clauseOps) { |
| 284 | if (!needBarrier()) |
| 285 | return; |
| 286 | |
| 287 | if (useDelayedPrivatization) { |
| 288 | if (clauseOps) |
| 289 | clauseOps->privateNeedsBarrier = |
| 290 | mlir::UnitAttr::get(&converter.getMLIRContext()); |
| 291 | } else { |
| 292 | firOpBuilder.create<mlir::omp::BarrierOp>(converter.getCurrentLocation()); |
| 293 | } |
| 294 | } |
| 295 | |
| 296 | void DataSharingProcessor::insertLastPrivateCompare(mlir::Operation *op) { |
| 297 | mlir::omp::LoopNestOp loopOp; |
| 298 | if (auto wrapper = mlir::dyn_cast<mlir::omp::LoopWrapperInterface>(op)) |
| 299 | loopOp = mlir::cast<mlir::omp::LoopNestOp>(wrapper.getWrappedLoop()); |
| 300 | |
| 301 | mlir::OpBuilder::InsertionGuard guard(firOpBuilder); |
| 302 | bool hasLastPrivate = [&]() { |
| 303 | for (const semantics::Symbol *sym : allPrivatizedSymbols) { |
| 304 | if (const auto *commonDet = |
| 305 | sym->detailsIf<semantics::CommonBlockDetails>()) { |
| 306 | for (const auto &mem : commonDet->objects()) |
| 307 | if (mem->test(semantics::Symbol::Flag::OmpLastPrivate)) |
| 308 | return true; |
| 309 | } else if (sym->test(semantics::Symbol::Flag::OmpLastPrivate)) |
| 310 | return true; |
| 311 | } |
| 312 | |
| 313 | return false; |
| 314 | }(); |
| 315 | |
| 316 | if (!hasLastPrivate) |
| 317 | return; |
| 318 | |
| 319 | if (mlir::isa<mlir::omp::WsloopOp>(op) || mlir::isa<mlir::omp::SimdOp>(op)) { |
| 320 | mlir::omp::LoopRelatedClauseOps result; |
| 321 | llvm::SmallVector<const semantics::Symbol *> iv; |
| 322 | collectLoopRelatedInfo(converter, converter.getCurrentLocation(), eval, |
| 323 | clauses, result, iv); |
| 324 | |
| 325 | // Update the original variable just before exiting the worksharing |
| 326 | // loop. Conversion as follows: |
| 327 | // |
| 328 | // omp.wsloop / omp.simd { omp.wsloop / omp.simd { |
| 329 | // omp.loop_nest { omp.loop_nest { |
| 330 | // ... ... |
| 331 | // store ===> store |
| 332 | // omp.yield %v = arith.addi %iv, %step |
| 333 | // } %cmp = %step < 0 ? %v < %ub : %v > %ub |
| 334 | // } fir.if %cmp { |
| 335 | // fir.store %v to %loopIV |
| 336 | // ^%lpv_update_blk: |
| 337 | // } |
| 338 | // omp.yield |
| 339 | // } |
| 340 | // } |
| 341 | mlir::Location loc = loopOp.getLoc(); |
| 342 | mlir::Operation *lastOper = loopOp.getRegion().back().getTerminator(); |
| 343 | firOpBuilder.setInsertionPoint(lastOper); |
| 344 | |
| 345 | mlir::Value cmpOp; |
| 346 | llvm::SmallVector<mlir::Value> vs; |
| 347 | vs.reserve(loopOp.getIVs().size()); |
| 348 | for (auto [iv, ub, step] : llvm::zip_equal( |
| 349 | loopOp.getIVs(), result.loopUpperBounds, result.loopSteps)) { |
| 350 | // v = iv + step |
| 351 | // cmp = step < 0 ? v < ub : v > ub |
| 352 | mlir::Value v = firOpBuilder.create<mlir::arith::AddIOp>(loc, iv, step); |
| 353 | vs.push_back(v); |
| 354 | mlir::Value zero = |
| 355 | firOpBuilder.createIntegerConstant(loc, step.getType(), 0); |
| 356 | mlir::Value negativeStep = firOpBuilder.create<mlir::arith::CmpIOp>( |
| 357 | loc, mlir::arith::CmpIPredicate::slt, step, zero); |
| 358 | mlir::Value vLT = firOpBuilder.create<mlir::arith::CmpIOp>( |
| 359 | loc, mlir::arith::CmpIPredicate::slt, v, ub); |
| 360 | mlir::Value vGT = firOpBuilder.create<mlir::arith::CmpIOp>( |
| 361 | loc, mlir::arith::CmpIPredicate::sgt, v, ub); |
| 362 | mlir::Value icmpOp = firOpBuilder.create<mlir::arith::SelectOp>( |
| 363 | loc, negativeStep, vLT, vGT); |
| 364 | |
| 365 | if (cmpOp) |
| 366 | cmpOp = firOpBuilder.create<mlir::arith::AndIOp>(loc, cmpOp, icmpOp); |
| 367 | else |
| 368 | cmpOp = icmpOp; |
| 369 | } |
| 370 | |
| 371 | auto ifOp = firOpBuilder.create<fir::IfOp>(loc, cmpOp, /*else*/ false); |
| 372 | firOpBuilder.setInsertionPointToStart(&ifOp.getThenRegion().front()); |
| 373 | for (auto [v, loopIV] : llvm::zip_equal(vs, loopIVs)) { |
| 374 | hlfir::Entity loopIVEntity{loopIV}; |
| 375 | loopIVEntity = |
| 376 | hlfir::derefPointersAndAllocatables(loc, firOpBuilder, loopIVEntity); |
| 377 | firOpBuilder.create<hlfir::AssignOp>(loc, v, loopIVEntity); |
| 378 | } |
| 379 | lastPrivIP = firOpBuilder.saveInsertionPoint(); |
| 380 | } else if (mlir::isa<mlir::omp::SectionsOp>(op)) { |
| 381 | // Already handled by genOMP() |
| 382 | } else { |
| 383 | TODO(converter.getCurrentLocation(), |
| 384 | "lastprivate clause in constructs other than " |
| 385 | "simd/worksharing-loop" ); |
| 386 | } |
| 387 | } |
| 388 | |
| 389 | static const parser::CharBlock * |
| 390 | getSource(const semantics::SemanticsContext &semaCtx, |
| 391 | const lower::pft::Evaluation &eval) { |
| 392 | const parser::CharBlock *source = nullptr; |
| 393 | |
| 394 | auto ompConsVisit = [&](const parser::OpenMPConstruct &x) { |
| 395 | std::visit( |
| 396 | common::visitors{ |
| 397 | [&](const parser::OpenMPSectionsConstruct &x) { |
| 398 | source = &std::get<0>(x.t).source; |
| 399 | }, |
| 400 | [&](const parser::OpenMPLoopConstruct &x) { |
| 401 | source = &std::get<0>(x.t).source; |
| 402 | }, |
| 403 | [&](const parser::OpenMPBlockConstruct &x) { |
| 404 | source = &std::get<0>(x.t).source; |
| 405 | }, |
| 406 | [&](const parser::OpenMPCriticalConstruct &x) { |
| 407 | source = &std::get<0>(x.t).source; |
| 408 | }, |
| 409 | [&](const parser::OpenMPAtomicConstruct &x) { |
| 410 | source = &std::get<parser::OmpDirectiveSpecification>(x.t).source; |
| 411 | }, |
| 412 | [&](const auto &x) { source = &x.source; }, |
| 413 | }, |
| 414 | x.u); |
| 415 | }; |
| 416 | |
| 417 | eval.visit(common::visitors{ |
| 418 | [&](const parser::OpenMPConstruct &x) { ompConsVisit(x); }, |
| 419 | [&](const parser::OpenMPDeclarativeConstruct &x) { source = &x.source; }, |
| 420 | [&](const parser::OmpEndLoopDirective &x) { source = &x.source; }, |
| 421 | [&](const auto &x) {}, |
| 422 | }); |
| 423 | |
| 424 | return source; |
| 425 | } |
| 426 | |
| 427 | bool DataSharingProcessor::isOpenMPPrivatizingConstruct( |
| 428 | const parser::OpenMPConstruct &omp) { |
| 429 | return common::visit( |
| 430 | [](auto &&s) { |
| 431 | using BareS = llvm::remove_cvref_t<decltype(s)>; |
| 432 | return std::is_same_v<BareS, parser::OpenMPBlockConstruct> || |
| 433 | std::is_same_v<BareS, parser::OpenMPLoopConstruct> || |
| 434 | std::is_same_v<BareS, parser::OpenMPSectionsConstruct>; |
| 435 | }, |
| 436 | omp.u); |
| 437 | } |
| 438 | |
| 439 | bool DataSharingProcessor::isOpenMPPrivatizingEvaluation( |
| 440 | const pft::Evaluation &eval) const { |
| 441 | return eval.visit([](auto &&s) { |
| 442 | using BareS = llvm::remove_cvref_t<decltype(s)>; |
| 443 | if constexpr (std::is_same_v<BareS, parser::OpenMPConstruct>) { |
| 444 | return isOpenMPPrivatizingConstruct(s); |
| 445 | } else { |
| 446 | return false; |
| 447 | } |
| 448 | }); |
| 449 | } |
| 450 | |
| 451 | void DataSharingProcessor::collectSymbolsInNestedRegions( |
| 452 | lower::pft::Evaluation &eval, semantics::Symbol::Flag flag, |
| 453 | llvm::SetVector<const semantics::Symbol *> &symbolsInNestedRegions) { |
| 454 | if (!eval.hasNestedEvaluations()) |
| 455 | return; |
| 456 | for (pft::Evaluation &nestedEval : eval.getNestedEvaluations()) { |
| 457 | if (isOpenMPPrivatizingEvaluation(nestedEval)) { |
| 458 | converter.collectSymbolSet(nestedEval, symbolsInNestedRegions, flag, |
| 459 | /*collectSymbols=*/true, |
| 460 | /*collectHostAssociatedSymbols=*/false); |
| 461 | } else { |
| 462 | // Recursively look for OpenMP constructs within `nestedEval`'s region |
| 463 | collectSymbolsInNestedRegions(nestedEval, flag, symbolsInNestedRegions); |
| 464 | } |
| 465 | } |
| 466 | } |
| 467 | |
| 468 | // Collect symbols to be default privatized in two steps. |
| 469 | // In step 1, collect all symbols in `eval` that match `flag` into |
| 470 | // `defaultSymbols`. In step 2, for nested constructs (if any), if and only if |
| 471 | // the nested construct is an OpenMP construct, collect those nested |
| 472 | // symbols skipping host associated symbols into `symbolsInNestedRegions`. |
| 473 | // Later, in current context, all symbols in the set |
| 474 | // `defaultSymbols` - `symbolsInNestedRegions` will be privatized. |
| 475 | void DataSharingProcessor::collectSymbols( |
| 476 | semantics::Symbol::Flag flag, |
| 477 | llvm::SetVector<const semantics::Symbol *> &symbols) { |
| 478 | // Collect all scopes associated with 'eval'. |
| 479 | llvm::SetVector<const semantics::Scope *> clauseScopes; |
| 480 | std::function<void(const semantics::Scope *)> collectScopes = |
| 481 | [&](const semantics::Scope *scope) { |
| 482 | clauseScopes.insert(scope); |
| 483 | for (const semantics::Scope &child : scope->children()) |
| 484 | collectScopes(&child); |
| 485 | }; |
| 486 | const parser::CharBlock *source = |
| 487 | clauses.empty() ? getSource(semaCtx, eval) : &clauses.front().source; |
| 488 | const semantics::Scope *curScope = nullptr; |
| 489 | if (source && !source->empty()) { |
| 490 | curScope = &semaCtx.FindScope(*source); |
| 491 | collectScopes(curScope); |
| 492 | } |
| 493 | // Collect all symbols referenced in the evaluation being processed, |
| 494 | // that matches 'flag'. |
| 495 | llvm::SetVector<const semantics::Symbol *> allSymbols; |
| 496 | converter.collectSymbolSet(eval, allSymbols, flag, |
| 497 | /*collectSymbols=*/true, |
| 498 | /*collectHostAssociatedSymbols=*/true); |
| 499 | |
| 500 | llvm::SetVector<const semantics::Symbol *> symbolsInNestedRegions; |
| 501 | collectSymbolsInNestedRegions(eval, flag, symbolsInNestedRegions); |
| 502 | |
| 503 | for (auto *symbol : allSymbols) |
| 504 | if (visitor.isSymbolDefineBy(symbol, eval)) |
| 505 | symbolsInNestedRegions.remove(symbol); |
| 506 | |
| 507 | // Filter-out symbols that must not be privatized. |
| 508 | bool collectImplicit = flag == semantics::Symbol::Flag::OmpImplicit; |
| 509 | bool collectPreDetermined = flag == semantics::Symbol::Flag::OmpPreDetermined; |
| 510 | |
| 511 | auto isPrivatizable = [](const semantics::Symbol &sym) -> bool { |
| 512 | return !semantics::IsProcedure(sym) && |
| 513 | !sym.GetUltimate().has<semantics::DerivedTypeDetails>() && |
| 514 | !sym.GetUltimate().has<semantics::NamelistDetails>() && |
| 515 | !semantics::IsImpliedDoIndex(sym.GetUltimate()) && |
| 516 | !semantics::IsStmtFunction(sym); |
| 517 | }; |
| 518 | |
| 519 | auto shouldCollectSymbol = [&](const semantics::Symbol *sym) { |
| 520 | if (collectImplicit) |
| 521 | return sym->test(semantics::Symbol::Flag::OmpImplicit); |
| 522 | |
| 523 | if (collectPreDetermined) |
| 524 | return sym->test(semantics::Symbol::Flag::OmpPreDetermined); |
| 525 | |
| 526 | return !sym->test(semantics::Symbol::Flag::OmpImplicit) && |
| 527 | !sym->test(semantics::Symbol::Flag::OmpPreDetermined); |
| 528 | }; |
| 529 | |
| 530 | for (const auto *sym : allSymbols) { |
| 531 | assert(curScope && "couldn't find current scope" ); |
| 532 | if (isPrivatizable(*sym) && !symbolsInNestedRegions.contains(sym) && |
| 533 | !explicitlyPrivatizedSymbols.contains(sym) && |
| 534 | shouldCollectSymbol(sym) && clauseScopes.contains(&sym->owner())) { |
| 535 | allPrivatizedSymbols.insert(sym); |
| 536 | symbols.insert(sym); |
| 537 | } |
| 538 | } |
| 539 | } |
| 540 | |
| 541 | void DataSharingProcessor::collectDefaultSymbols() { |
| 542 | using DataSharingAttribute = omp::clause::Default::DataSharingAttribute; |
| 543 | for (const omp::Clause &clause : clauses) { |
| 544 | if (const auto *defaultClause = |
| 545 | std::get_if<omp::clause::Default>(&clause.u)) { |
| 546 | if (defaultClause->v == DataSharingAttribute::Private) |
| 547 | collectSymbols(semantics::Symbol::Flag::OmpPrivate, defaultSymbols); |
| 548 | else if (defaultClause->v == DataSharingAttribute::Firstprivate) |
| 549 | collectSymbols(semantics::Symbol::Flag::OmpFirstPrivate, |
| 550 | defaultSymbols); |
| 551 | } |
| 552 | } |
| 553 | } |
| 554 | |
| 555 | void DataSharingProcessor::collectImplicitSymbols() { |
| 556 | // There will be no implicit symbols when a default clause is present. |
| 557 | if (defaultSymbols.empty()) |
| 558 | collectSymbols(semantics::Symbol::Flag::OmpImplicit, implicitSymbols); |
| 559 | } |
| 560 | |
| 561 | void DataSharingProcessor::collectPreDeterminedSymbols() { |
| 562 | if (shouldCollectPreDeterminedSymbols) |
| 563 | collectSymbols(semantics::Symbol::Flag::OmpPreDetermined, |
| 564 | preDeterminedSymbols); |
| 565 | } |
| 566 | |
| 567 | void DataSharingProcessor::privatize(mlir::omp::PrivateClauseOps *clauseOps) { |
| 568 | for (const semantics::Symbol *sym : allPrivatizedSymbols) { |
| 569 | if (const auto *commonDet = |
| 570 | sym->detailsIf<semantics::CommonBlockDetails>()) { |
| 571 | for (const auto &mem : commonDet->objects()) |
| 572 | privatizeSymbol(&*mem, clauseOps); |
| 573 | } else |
| 574 | privatizeSymbol(sym, clauseOps); |
| 575 | } |
| 576 | } |
| 577 | |
| 578 | void DataSharingProcessor::copyLastPrivatize(mlir::Operation *op) { |
| 579 | insertLastPrivateCompare(op); |
| 580 | for (const semantics::Symbol *sym : allPrivatizedSymbols) |
| 581 | if (const auto *commonDet = |
| 582 | sym->detailsIf<semantics::CommonBlockDetails>()) { |
| 583 | for (const auto &mem : commonDet->objects()) { |
| 584 | copyLastPrivateSymbol(&*mem, &lastPrivIP); |
| 585 | } |
| 586 | } else { |
| 587 | copyLastPrivateSymbol(sym, &lastPrivIP); |
| 588 | } |
| 589 | } |
| 590 | |
| 591 | void DataSharingProcessor::privatizeSymbol( |
| 592 | const semantics::Symbol *symToPrivatize, |
| 593 | mlir::omp::PrivateClauseOps *clauseOps) { |
| 594 | if (!useDelayedPrivatization) { |
| 595 | cloneSymbol(symToPrivatize); |
| 596 | copyFirstPrivateSymbol(symToPrivatize); |
| 597 | return; |
| 598 | } |
| 599 | |
| 600 | Fortran::lower::privatizeSymbol<mlir::omp::PrivateClauseOp, |
| 601 | mlir::omp::PrivateClauseOps>( |
| 602 | converter, firOpBuilder, symTable, allPrivatizedSymbols, |
| 603 | mightHaveReadHostSym, symToPrivatize, clauseOps); |
| 604 | } |
| 605 | } // namespace omp |
| 606 | } // namespace lower |
| 607 | } // namespace Fortran |
| 608 | |