1 | //===- BufferizeHLFIR.cpp - Bufferize HLFIR ------------------------------===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | // This file defines a pass that bufferize hlfir.expr. It translates operations |
9 | // producing or consuming hlfir.expr into operations operating on memory. |
10 | // An hlfir.expr is translated to a tuple<variable address, cleanupflag> |
11 | // where cleanupflag is set to true if storage for the expression was allocated |
12 | // on the heap. |
13 | //===----------------------------------------------------------------------===// |
14 | |
15 | #include "flang/Optimizer/Builder/Character.h" |
16 | #include "flang/Optimizer/Builder/FIRBuilder.h" |
17 | #include "flang/Optimizer/Builder/HLFIRTools.h" |
18 | #include "flang/Optimizer/Builder/MutableBox.h" |
19 | #include "flang/Optimizer/Builder/Runtime/Allocatable.h" |
20 | #include "flang/Optimizer/Builder/Runtime/Derived.h" |
21 | #include "flang/Optimizer/Builder/Todo.h" |
22 | #include "flang/Optimizer/Dialect/FIRDialect.h" |
23 | #include "flang/Optimizer/Dialect/FIROps.h" |
24 | #include "flang/Optimizer/Dialect/FIRType.h" |
25 | #include "flang/Optimizer/Dialect/Support/FIRContext.h" |
26 | #include "flang/Optimizer/HLFIR/HLFIRDialect.h" |
27 | #include "flang/Optimizer/HLFIR/HLFIROps.h" |
28 | #include "flang/Optimizer/HLFIR/Passes.h" |
29 | #include "flang/Optimizer/OpenMP/Passes.h" |
30 | #include "mlir/Dialect/OpenMP/OpenMPDialect.h" |
31 | #include "mlir/IR/Dominance.h" |
32 | #include "mlir/IR/PatternMatch.h" |
33 | #include "mlir/Pass/Pass.h" |
34 | #include "mlir/Pass/PassManager.h" |
35 | #include "mlir/Transforms/DialectConversion.h" |
36 | #include "llvm/ADT/TypeSwitch.h" |
37 | |
38 | namespace hlfir { |
39 | #define GEN_PASS_DEF_BUFFERIZEHLFIR |
40 | #include "flang/Optimizer/HLFIR/Passes.h.inc" |
41 | } // namespace hlfir |
42 | |
43 | namespace { |
44 | |
45 | /// Helper to create tuple from a bufferized expr storage and clean up |
46 | /// instruction flag. The storage is an HLFIR variable so that it can |
47 | /// be manipulated as a variable later (all shape and length information |
48 | /// cam be retrieved from it). |
49 | static mlir::Value packageBufferizedExpr(mlir::Location loc, |
50 | fir::FirOpBuilder &builder, |
51 | hlfir::Entity storage, |
52 | mlir::Value mustFree) { |
53 | auto tupleType = mlir::TupleType::get( |
54 | builder.getContext(), |
55 | mlir::TypeRange{storage.getType(), mustFree.getType()}); |
56 | auto undef = builder.create<fir::UndefOp>(loc, tupleType); |
57 | auto insert = builder.create<fir::InsertValueOp>( |
58 | loc, tupleType, undef, mustFree, |
59 | builder.getArrayAttr( |
60 | {builder.getIntegerAttr(builder.getIndexType(), 1)})); |
61 | return builder.create<fir::InsertValueOp>( |
62 | loc, tupleType, insert, storage, |
63 | builder.getArrayAttr( |
64 | {builder.getIntegerAttr(builder.getIndexType(), 0)})); |
65 | } |
66 | |
67 | /// Helper to create tuple from a bufferized expr storage and constant |
68 | /// boolean clean-up flag. |
69 | static mlir::Value packageBufferizedExpr(mlir::Location loc, |
70 | fir::FirOpBuilder &builder, |
71 | hlfir::Entity storage, bool mustFree) { |
72 | mlir::Value mustFreeValue = builder.createBool(loc, mustFree); |
73 | return packageBufferizedExpr(loc, builder, storage, mustFreeValue); |
74 | } |
75 | |
76 | /// Helper to extract the storage from a tuple created by packageBufferizedExpr. |
77 | /// It assumes no tuples are used as HLFIR operation operands, which is |
78 | /// currently enforced by the verifiers that only accept HLFIR value or |
79 | /// variable types which do not include tuples. |
80 | static hlfir::Entity getBufferizedExprStorage(mlir::Value bufferizedExpr) { |
81 | auto tupleType = mlir::dyn_cast<mlir::TupleType>(bufferizedExpr.getType()); |
82 | if (!tupleType) |
83 | return hlfir::Entity{bufferizedExpr}; |
84 | assert(tupleType.size() == 2 && "unexpected tuple type" ); |
85 | if (auto insert = bufferizedExpr.getDefiningOp<fir::InsertValueOp>()) |
86 | if (insert.getVal().getType() == tupleType.getType(0)) |
87 | return hlfir::Entity{insert.getVal()}; |
88 | TODO(bufferizedExpr.getLoc(), "general extract storage case" ); |
89 | } |
90 | |
91 | /// Helper to extract the clean-up flag from a tuple created by |
92 | /// packageBufferizedExpr. |
93 | static mlir::Value getBufferizedExprMustFreeFlag(mlir::Value bufferizedExpr) { |
94 | auto tupleType = mlir::dyn_cast<mlir::TupleType>(bufferizedExpr.getType()); |
95 | if (!tupleType) |
96 | return bufferizedExpr; |
97 | assert(tupleType.size() == 2 && "unexpected tuple type" ); |
98 | if (auto insert = bufferizedExpr.getDefiningOp<fir::InsertValueOp>()) |
99 | if (auto insert0 = insert.getAdt().getDefiningOp<fir::InsertValueOp>()) |
100 | if (insert0.getVal().getType() == tupleType.getType(1)) |
101 | return insert0.getVal(); |
102 | TODO(bufferizedExpr.getLoc(), "general extract storage case" ); |
103 | } |
104 | |
105 | static std::pair<hlfir::Entity, mlir::Value> |
106 | createArrayTemp(mlir::Location loc, fir::FirOpBuilder &builder, |
107 | mlir::Type exprType, mlir::Value shape, |
108 | llvm::ArrayRef<mlir::Value> extents, |
109 | llvm::ArrayRef<mlir::Value> lenParams, |
110 | std::optional<hlfir::Entity> polymorphicMold) { |
111 | auto sequenceType = mlir::cast<fir::SequenceType>( |
112 | hlfir::getFortranElementOrSequenceType(exprType)); |
113 | |
114 | auto genTempDeclareOp = |
115 | [](fir::FirOpBuilder &builder, mlir::Location loc, mlir::Value memref, |
116 | llvm::StringRef name, mlir::Value shape, |
117 | llvm::ArrayRef<mlir::Value> typeParams, |
118 | fir::FortranVariableFlagsAttr attrs) -> mlir::Value { |
119 | auto declareOp = |
120 | builder.create<hlfir::DeclareOp>(loc, memref, name, shape, typeParams, |
121 | /*dummy_scope=*/nullptr, attrs); |
122 | return declareOp.getBase(); |
123 | }; |
124 | |
125 | auto [base, isHeapAlloc] = builder.createArrayTemp( |
126 | loc, sequenceType, shape, extents, lenParams, genTempDeclareOp, |
127 | polymorphicMold ? polymorphicMold->getFirBase() : nullptr); |
128 | hlfir::Entity temp = hlfir::Entity{base}; |
129 | assert(!temp.isAllocatable() && "temp must have been allocated" ); |
130 | return {temp, builder.createBool(loc, isHeapAlloc)}; |
131 | } |
132 | |
133 | /// Copy \p source into a new temporary and package the temporary into a |
134 | /// <temp,cleanup> tuple. The temporary may be heap or stack allocated. |
135 | static mlir::Value copyInTempAndPackage(mlir::Location loc, |
136 | fir::FirOpBuilder &builder, |
137 | hlfir::Entity source) { |
138 | auto [temp, cleanup] = hlfir::createTempFromMold(loc, builder, source); |
139 | assert(!temp.isAllocatable() && "expect temp to already be allocated" ); |
140 | builder.create<hlfir::AssignOp>(loc, source, temp, /*realloc=*/false, |
141 | /*keep_lhs_length_if_realloc=*/false, |
142 | /*temporary_lhs=*/true); |
143 | return packageBufferizedExpr(loc, builder, temp, cleanup); |
144 | } |
145 | |
146 | struct AsExprOpConversion : public mlir::OpConversionPattern<hlfir::AsExprOp> { |
147 | using mlir::OpConversionPattern<hlfir::AsExprOp>::OpConversionPattern; |
148 | explicit AsExprOpConversion(mlir::MLIRContext *ctx) |
149 | : mlir::OpConversionPattern<hlfir::AsExprOp>{ctx} {} |
150 | llvm::LogicalResult |
151 | matchAndRewrite(hlfir::AsExprOp asExpr, OpAdaptor adaptor, |
152 | mlir::ConversionPatternRewriter &rewriter) const override { |
153 | mlir::Location loc = asExpr->getLoc(); |
154 | auto module = asExpr->getParentOfType<mlir::ModuleOp>(); |
155 | fir::FirOpBuilder builder(rewriter, module); |
156 | if (asExpr.isMove()) { |
157 | // Move variable storage for the hlfir.expr buffer. |
158 | mlir::Value bufferizedExpr = packageBufferizedExpr( |
159 | loc, builder, hlfir::Entity{adaptor.getVar()}, adaptor.getMustFree()); |
160 | rewriter.replaceOp(asExpr, bufferizedExpr); |
161 | return mlir::success(); |
162 | } |
163 | // Otherwise, create a copy in a new buffer. |
164 | hlfir::Entity source = hlfir::Entity{adaptor.getVar()}; |
165 | mlir::Value bufferizedExpr = copyInTempAndPackage(loc, builder, source); |
166 | rewriter.replaceOp(asExpr, bufferizedExpr); |
167 | return mlir::success(); |
168 | } |
169 | }; |
170 | |
171 | struct ShapeOfOpConversion |
172 | : public mlir::OpConversionPattern<hlfir::ShapeOfOp> { |
173 | using mlir::OpConversionPattern<hlfir::ShapeOfOp>::OpConversionPattern; |
174 | |
175 | llvm::LogicalResult |
176 | matchAndRewrite(hlfir::ShapeOfOp shapeOf, OpAdaptor adaptor, |
177 | mlir::ConversionPatternRewriter &rewriter) const override { |
178 | mlir::Location loc = shapeOf.getLoc(); |
179 | mlir::ModuleOp mod = shapeOf->getParentOfType<mlir::ModuleOp>(); |
180 | fir::FirOpBuilder builder(rewriter, mod); |
181 | |
182 | mlir::Value shape; |
183 | hlfir::Entity bufferizedExpr{getBufferizedExprStorage(adaptor.getExpr())}; |
184 | if (bufferizedExpr.isVariable()) { |
185 | shape = hlfir::genShape(loc, builder, bufferizedExpr); |
186 | } else { |
187 | // everything else failed so try to create a shape from static type info |
188 | hlfir::ExprType exprTy = |
189 | mlir::dyn_cast_or_null<hlfir::ExprType>(adaptor.getExpr().getType()); |
190 | if (exprTy) |
191 | shape = hlfir::genExprShape(builder, loc, exprTy); |
192 | } |
193 | // expected to never happen |
194 | if (!shape) |
195 | return emitError(loc, |
196 | "Unresolvable hlfir.shape_of where extents are unknown" ); |
197 | |
198 | rewriter.replaceOp(shapeOf, shape); |
199 | return mlir::success(); |
200 | } |
201 | }; |
202 | |
203 | struct ApplyOpConversion : public mlir::OpConversionPattern<hlfir::ApplyOp> { |
204 | using mlir::OpConversionPattern<hlfir::ApplyOp>::OpConversionPattern; |
205 | explicit ApplyOpConversion(mlir::MLIRContext *ctx) |
206 | : mlir::OpConversionPattern<hlfir::ApplyOp>{ctx} {} |
207 | llvm::LogicalResult |
208 | matchAndRewrite(hlfir::ApplyOp apply, OpAdaptor adaptor, |
209 | mlir::ConversionPatternRewriter &rewriter) const override { |
210 | mlir::Location loc = apply->getLoc(); |
211 | hlfir::Entity bufferizedExpr = getBufferizedExprStorage(adaptor.getExpr()); |
212 | mlir::Type resultType = hlfir::getVariableElementType(bufferizedExpr); |
213 | mlir::Value result = rewriter.create<hlfir::DesignateOp>( |
214 | loc, resultType, bufferizedExpr, adaptor.getIndices(), |
215 | adaptor.getTypeparams()); |
216 | if (fir::isa_trivial(apply.getType())) { |
217 | result = rewriter.create<fir::LoadOp>(loc, result); |
218 | } else { |
219 | fir::FirOpBuilder builder(rewriter, apply.getOperation()); |
220 | result = |
221 | packageBufferizedExpr(loc, builder, hlfir::Entity{result}, false); |
222 | } |
223 | rewriter.replaceOp(apply, result); |
224 | return mlir::success(); |
225 | } |
226 | }; |
227 | |
228 | struct AssignOpConversion : public mlir::OpConversionPattern<hlfir::AssignOp> { |
229 | using mlir::OpConversionPattern<hlfir::AssignOp>::OpConversionPattern; |
230 | explicit AssignOpConversion(mlir::MLIRContext *ctx) |
231 | : mlir::OpConversionPattern<hlfir::AssignOp>{ctx} {} |
232 | llvm::LogicalResult |
233 | matchAndRewrite(hlfir::AssignOp assign, OpAdaptor adaptor, |
234 | mlir::ConversionPatternRewriter &rewriter) const override { |
235 | llvm::SmallVector<mlir::Value> newOperands; |
236 | for (mlir::Value operand : adaptor.getOperands()) |
237 | newOperands.push_back(getBufferizedExprStorage(operand)); |
238 | rewriter.startOpModification(assign); |
239 | assign->setOperands(newOperands); |
240 | rewriter.finalizeOpModification(assign); |
241 | return mlir::success(); |
242 | } |
243 | }; |
244 | |
245 | struct ConcatOpConversion : public mlir::OpConversionPattern<hlfir::ConcatOp> { |
246 | using mlir::OpConversionPattern<hlfir::ConcatOp>::OpConversionPattern; |
247 | explicit ConcatOpConversion(mlir::MLIRContext *ctx) |
248 | : mlir::OpConversionPattern<hlfir::ConcatOp>{ctx} {} |
249 | llvm::LogicalResult |
250 | matchAndRewrite(hlfir::ConcatOp concat, OpAdaptor adaptor, |
251 | mlir::ConversionPatternRewriter &rewriter) const override { |
252 | mlir::Location loc = concat->getLoc(); |
253 | fir::FirOpBuilder builder(rewriter, concat.getOperation()); |
254 | assert(adaptor.getStrings().size() >= 2 && |
255 | "must have at least two strings operands" ); |
256 | if (adaptor.getStrings().size() > 2) |
257 | TODO(loc, "codegen of optimized chained concatenation of more than two " |
258 | "strings" ); |
259 | hlfir::Entity lhs = getBufferizedExprStorage(adaptor.getStrings()[0]); |
260 | hlfir::Entity rhs = getBufferizedExprStorage(adaptor.getStrings()[1]); |
261 | auto [lhsExv, c1] = hlfir::translateToExtendedValue(loc, builder, lhs); |
262 | auto [rhsExv, c2] = hlfir::translateToExtendedValue(loc, builder, rhs); |
263 | assert(!c1 && !c2 && "expected variables" ); |
264 | fir::ExtendedValue res = |
265 | fir::factory::CharacterExprHelper{builder, loc}.createConcatenate( |
266 | *lhsExv.getCharBox(), *rhsExv.getCharBox()); |
267 | // Ensure the memory type is the same as the result type. |
268 | mlir::Type addrType = fir::ReferenceType::get( |
269 | hlfir::getFortranElementType(concat.getResult().getType())); |
270 | mlir::Value cast = builder.createConvert(loc, addrType, fir::getBase(res)); |
271 | res = fir::substBase(res, cast); |
272 | hlfir::Entity hlfirTempRes = |
273 | hlfir::Entity{hlfir::genDeclare(loc, builder, res, "tmp" , |
274 | fir::FortranVariableFlagsAttr{}) |
275 | .getBase()}; |
276 | mlir::Value bufferizedExpr = |
277 | packageBufferizedExpr(loc, builder, hlfirTempRes, false); |
278 | rewriter.replaceOp(concat, bufferizedExpr); |
279 | return mlir::success(); |
280 | } |
281 | }; |
282 | |
283 | struct SetLengthOpConversion |
284 | : public mlir::OpConversionPattern<hlfir::SetLengthOp> { |
285 | using mlir::OpConversionPattern<hlfir::SetLengthOp>::OpConversionPattern; |
286 | explicit SetLengthOpConversion(mlir::MLIRContext *ctx) |
287 | : mlir::OpConversionPattern<hlfir::SetLengthOp>{ctx} {} |
288 | llvm::LogicalResult |
289 | matchAndRewrite(hlfir::SetLengthOp setLength, OpAdaptor adaptor, |
290 | mlir::ConversionPatternRewriter &rewriter) const override { |
291 | mlir::Location loc = setLength->getLoc(); |
292 | fir::FirOpBuilder builder(rewriter, setLength.getOperation()); |
293 | // Create a temp with the new length. |
294 | hlfir::Entity string = getBufferizedExprStorage(adaptor.getString()); |
295 | auto charType = hlfir::getFortranElementType(setLength.getType()); |
296 | llvm::StringRef tmpName{".tmp" }; |
297 | llvm::SmallVector<mlir::Value, 1> lenParams{adaptor.getLength()}; |
298 | auto alloca = builder.createTemporary(loc, charType, tmpName, |
299 | /*shape=*/std::nullopt, lenParams); |
300 | auto declareOp = builder.create<hlfir::DeclareOp>( |
301 | loc, alloca, tmpName, /*shape=*/mlir::Value{}, lenParams, |
302 | /*dummy_scope=*/nullptr, fir::FortranVariableFlagsAttr{}); |
303 | hlfir::Entity temp{declareOp.getBase()}; |
304 | // Assign string value to the created temp. |
305 | builder.create<hlfir::AssignOp>(loc, string, temp, |
306 | /*realloc=*/false, |
307 | /*keep_lhs_length_if_realloc=*/false, |
308 | /*temporary_lhs=*/true); |
309 | mlir::Value bufferizedExpr = |
310 | packageBufferizedExpr(loc, builder, temp, false); |
311 | rewriter.replaceOp(setLength, bufferizedExpr); |
312 | return mlir::success(); |
313 | } |
314 | }; |
315 | |
316 | struct GetLengthOpConversion |
317 | : public mlir::OpConversionPattern<hlfir::GetLengthOp> { |
318 | using mlir::OpConversionPattern<hlfir::GetLengthOp>::OpConversionPattern; |
319 | explicit GetLengthOpConversion(mlir::MLIRContext *ctx) |
320 | : mlir::OpConversionPattern<hlfir::GetLengthOp>{ctx} {} |
321 | llvm::LogicalResult |
322 | matchAndRewrite(hlfir::GetLengthOp getLength, OpAdaptor adaptor, |
323 | mlir::ConversionPatternRewriter &rewriter) const override { |
324 | mlir::Location loc = getLength->getLoc(); |
325 | fir::FirOpBuilder builder(rewriter, getLength.getOperation()); |
326 | hlfir::Entity bufferizedExpr = getBufferizedExprStorage(adaptor.getExpr()); |
327 | mlir::Value length = hlfir::genCharLength(loc, builder, bufferizedExpr); |
328 | if (!length) |
329 | return rewriter.notifyMatchFailure( |
330 | getLength, "could not deduce length from GetLengthOp operand" ); |
331 | length = builder.createConvert(loc, builder.getIndexType(), length); |
332 | rewriter.replaceOp(getLength, length); |
333 | return mlir::success(); |
334 | } |
335 | }; |
336 | |
337 | /// The current hlfir.associate lowering does not handle multiple uses of a |
338 | /// non-trivial expression value because it generates the cleanup for the |
339 | /// expression bufferization at hlfir.end_associate. If there was more than one |
340 | /// hlfir.end_associate, it would be cleaned up multiple times, perhaps before |
341 | /// one of the other uses. |
342 | /// Note that we have to be careful about expressions used by a single |
343 | /// hlfir.end_associate that may be executed more times than the producer |
344 | /// of the expression value. This may also cause multiple clean-ups |
345 | /// for the same memory (e.g. cause double-free errors). For example, |
346 | /// hlfir.end_associate inside hlfir.elemental may cause such issues |
347 | /// for expressions produced outside of hlfir.elemental. |
348 | static bool allOtherUsesAreSafeForAssociate(mlir::Value value, |
349 | mlir::Operation *currentUse, |
350 | mlir::Operation *endAssociate) { |
351 | // If value producer is from a different region than |
352 | // hlfir.associate/end_associate, then conservatively assume |
353 | // that the hlfir.end_associate may execute more times than |
354 | // the value producer. |
355 | // TODO: this may be improved for operations that cannot |
356 | // result in multiple executions (e.g. ifOp). |
357 | if (value.getParentRegion() != currentUse->getParentRegion() || |
358 | (endAssociate && |
359 | value.getParentRegion() != endAssociate->getParentRegion())) |
360 | return false; |
361 | |
362 | for (mlir::Operation *useOp : value.getUsers()) { |
363 | // Ignore DestroyOp's that do not imply finalization. |
364 | // If finalization is implied, then we must delegate |
365 | // the finalization to the correspoding EndAssociateOp, |
366 | // but we currently do not; so we disable the buffer |
367 | // reuse in this case. |
368 | if (auto destroy = mlir::dyn_cast<hlfir::DestroyOp>(useOp)) { |
369 | if (destroy.mustFinalizeExpr()) |
370 | return false; |
371 | else |
372 | continue; |
373 | } |
374 | |
375 | if (useOp != currentUse) { |
376 | // hlfir.shape_of and hlfir.get_length will not disrupt cleanup so it is |
377 | // safe for hlfir.associate. These operations might read from the box and |
378 | // so they need to come before the hflir.end_associate (which may |
379 | // deallocate). |
380 | if (mlir::isa<hlfir::ShapeOfOp>(useOp) || |
381 | mlir::isa<hlfir::GetLengthOp>(useOp)) { |
382 | if (!endAssociate) |
383 | continue; |
384 | // If useOp dominates the endAssociate, then it is definitely safe. |
385 | if (useOp->getBlock() != endAssociate->getBlock()) |
386 | if (mlir::DominanceInfo{}.dominates(useOp, endAssociate)) |
387 | continue; |
388 | if (useOp->isBeforeInBlock(endAssociate)) |
389 | continue; |
390 | } |
391 | return false; |
392 | } |
393 | } |
394 | return true; |
395 | } |
396 | |
397 | static void eraseAllUsesInDestroys(mlir::Value value, |
398 | mlir::ConversionPatternRewriter &rewriter) { |
399 | for (mlir::Operation *useOp : value.getUsers()) |
400 | if (auto destroy = mlir::dyn_cast<hlfir::DestroyOp>(useOp)) { |
401 | assert(!destroy.mustFinalizeExpr() && |
402 | "deleting DestroyOp with finalize attribute" ); |
403 | rewriter.eraseOp(destroy); |
404 | } |
405 | } |
406 | |
407 | struct AssociateOpConversion |
408 | : public mlir::OpConversionPattern<hlfir::AssociateOp> { |
409 | using mlir::OpConversionPattern<hlfir::AssociateOp>::OpConversionPattern; |
410 | explicit AssociateOpConversion(mlir::MLIRContext *ctx) |
411 | : mlir::OpConversionPattern<hlfir::AssociateOp>{ctx} {} |
412 | llvm::LogicalResult |
413 | matchAndRewrite(hlfir::AssociateOp associate, OpAdaptor adaptor, |
414 | mlir::ConversionPatternRewriter &rewriter) const override { |
415 | mlir::Location loc = associate->getLoc(); |
416 | fir::FirOpBuilder builder(rewriter, associate.getOperation()); |
417 | mlir::Value bufferizedExpr = getBufferizedExprStorage(adaptor.getSource()); |
418 | const bool isTrivialValue = fir::isa_trivial(bufferizedExpr.getType()); |
419 | |
420 | auto getEndAssociate = |
421 | [](hlfir::AssociateOp associate) -> mlir::Operation * { |
422 | for (mlir::Operation *useOp : associate->getUsers()) |
423 | if (mlir::isa<hlfir::EndAssociateOp>(useOp)) |
424 | return useOp; |
425 | // happens in some hand coded mlir in tests |
426 | return nullptr; |
427 | }; |
428 | |
429 | auto replaceWith = [&](mlir::Value hlfirVar, mlir::Value firVar, |
430 | mlir::Value flag) { |
431 | // 0-dim variables may need special handling: |
432 | // %0 = hlfir.as_expr %x move %true : |
433 | // (!fir.box<!fir.heap<!fir.type<_T{y:i32}>>>, i1) -> |
434 | // !hlfir.expr<!fir.type<_T{y:i32}>> |
435 | // %1:3 = hlfir.associate %0 {adapt.valuebyref} : |
436 | // (!hlfir.expr<!fir.type<_T{y:i32}>>) -> |
437 | // (!fir.ref<!fir.type<_T{y:i32}>>, |
438 | // !fir.ref<!fir.type<_T{y:i32}>>, |
439 | // i1) |
440 | // |
441 | // !fir.box<!fir.heap<!fir.type<_T{y:i32}>>> value must be |
442 | // propagated as the box address !fir.ref<!fir.type<_T{y:i32}>>. |
443 | auto adjustVar = [&](mlir::Value sourceVar, mlir::Type assocType) { |
444 | if ((mlir::isa<fir::BaseBoxType>(sourceVar.getType()) && |
445 | !mlir::isa<fir::BaseBoxType>(assocType)) || |
446 | ((mlir::isa<fir::BoxCharType>(sourceVar.getType()) && |
447 | !mlir::isa<fir::BoxCharType>(assocType)))) { |
448 | sourceVar = builder.create<fir::BoxAddrOp>(loc, assocType, sourceVar); |
449 | } else { |
450 | sourceVar = builder.createConvert(loc, assocType, sourceVar); |
451 | } |
452 | return sourceVar; |
453 | }; |
454 | |
455 | mlir::Type associateHlfirVarType = associate.getResultTypes()[0]; |
456 | hlfirVar = adjustVar(hlfirVar, associateHlfirVarType); |
457 | associate.getResult(0).replaceAllUsesWith(hlfirVar); |
458 | |
459 | mlir::Type associateFirVarType = associate.getResultTypes()[1]; |
460 | firVar = adjustVar(firVar, associateFirVarType); |
461 | associate.getResult(1).replaceAllUsesWith(firVar); |
462 | associate.getResult(2).replaceAllUsesWith(flag); |
463 | // FIXME: note that the AssociateOp that is being erased |
464 | // here will continue to be a user of the original Source |
465 | // operand (e.g. a result of hlfir.elemental), because |
466 | // the erasure is not immediate in the rewriter. |
467 | // In case there are multiple uses of the Source operand, |
468 | // the allOtherUsesAreSafeForAssociate() below will always |
469 | // see them, so there is no way to reuse the buffer. |
470 | // I think we have to run this analysis before doing |
471 | // the conversions, so that we can analyze HLFIR in its |
472 | // original form and decide which of the AssociateOp |
473 | // users of hlfir.expr can reuse the buffer (if it can). |
474 | rewriter.eraseOp(associate); |
475 | }; |
476 | |
477 | // If this is the last use of the expression value and this is an hlfir.expr |
478 | // that was bufferized, re-use the storage. |
479 | // Otherwise, create a temp and assign the storage to it. |
480 | // |
481 | // WARNING: it is important to use the original Source operand |
482 | // of the AssociateOp to look for the users, because its replacement |
483 | // has zero materialized users at this point. |
484 | // So allOtherUsesAreSafeForAssociate() may incorrectly return |
485 | // true here. |
486 | if (!isTrivialValue && allOtherUsesAreSafeForAssociate( |
487 | associate.getSource(), associate.getOperation(), |
488 | getEndAssociate(associate))) { |
489 | // Re-use hlfir.expr buffer if this is the only use of the hlfir.expr |
490 | // outside of the hlfir.destroy. Take on the cleaning-up responsibility |
491 | // for the related hlfir.end_associate, and erase the hlfir.destroy (if |
492 | // any). |
493 | mlir::Value mustFree = getBufferizedExprMustFreeFlag(adaptor.getSource()); |
494 | mlir::Value firBase = hlfir::Entity{bufferizedExpr}.getFirBase(); |
495 | replaceWith(bufferizedExpr, firBase, mustFree); |
496 | eraseAllUsesInDestroys(associate.getSource(), rewriter); |
497 | // Make sure to erase the hlfir.destroy if there is an indirection through |
498 | // a hlfir.no_reassoc operation. |
499 | if (auto noReassoc = mlir::dyn_cast_or_null<hlfir::NoReassocOp>( |
500 | associate.getSource().getDefiningOp())) |
501 | eraseAllUsesInDestroys(noReassoc.getVal(), rewriter); |
502 | return mlir::success(); |
503 | } |
504 | if (isTrivialValue) { |
505 | llvm::SmallVector<mlir::NamedAttribute, 1> attrs; |
506 | if (associate->hasAttr(fir::getAdaptToByRefAttrName())) { |
507 | attrs.push_back(fir::getAdaptToByRefAttr(builder)); |
508 | } |
509 | llvm::StringRef name = "" ; |
510 | if (associate.getUniqName()) |
511 | name = *associate.getUniqName(); |
512 | auto temp = |
513 | builder.createTemporary(loc, bufferizedExpr.getType(), name, attrs); |
514 | builder.create<fir::StoreOp>(loc, bufferizedExpr, temp); |
515 | mlir::Value mustFree = builder.createBool(loc, false); |
516 | replaceWith(temp, temp, mustFree); |
517 | return mlir::success(); |
518 | } |
519 | // non-trivial value with more than one use. We will have to make a copy and |
520 | // use that |
521 | hlfir::Entity source = hlfir::Entity{bufferizedExpr}; |
522 | mlir::Value bufferTuple = copyInTempAndPackage(loc, builder, source); |
523 | bufferizedExpr = getBufferizedExprStorage(bufferTuple); |
524 | replaceWith(bufferizedExpr, hlfir::Entity{bufferizedExpr}.getFirBase(), |
525 | getBufferizedExprMustFreeFlag(bufferTuple)); |
526 | return mlir::success(); |
527 | } |
528 | }; |
529 | |
530 | static void genBufferDestruction(mlir::Location loc, fir::FirOpBuilder &builder, |
531 | mlir::Value var, mlir::Value mustFree, |
532 | bool mustFinalize) { |
533 | auto genFreeOrFinalize = [&](bool doFree, bool deallocComponents, |
534 | bool doFinalize) { |
535 | if (!doFree && !deallocComponents && !doFinalize) |
536 | return; |
537 | |
538 | mlir::Value addr = var; |
539 | |
540 | // fir::FreeMemOp operand type must be a fir::HeapType. |
541 | mlir::Type heapType = fir::HeapType::get( |
542 | hlfir::getFortranElementOrSequenceType(var.getType())); |
543 | if (mlir::isa<fir::BaseBoxType, fir::BoxCharType>(var.getType())) { |
544 | if (mustFinalize && !mlir::isa<fir::BaseBoxType>(var.getType())) |
545 | fir::emitFatalError(loc, "non-finalizable variable" ); |
546 | |
547 | addr = builder.create<fir::BoxAddrOp>(loc, heapType, var); |
548 | } else { |
549 | if (!mlir::isa<fir::HeapType>(var.getType())) |
550 | addr = builder.create<fir::ConvertOp>(loc, heapType, var); |
551 | |
552 | if (mustFinalize || deallocComponents) { |
553 | // Embox the raw pointer using proper shape and type params |
554 | // (note that the shape might be visible via the array finalization |
555 | // routines). |
556 | if (!hlfir::isFortranEntity(var)) |
557 | TODO(loc, "need a Fortran entity to create a box" ); |
558 | |
559 | hlfir::Entity entity{var}; |
560 | llvm::SmallVector<mlir::Value> lenParams; |
561 | hlfir::genLengthParameters(loc, builder, entity, lenParams); |
562 | mlir::Value shape; |
563 | if (entity.isArray()) |
564 | shape = hlfir::genShape(loc, builder, entity); |
565 | mlir::Type boxType = fir::BoxType::get(heapType); |
566 | var = builder.createBox(loc, boxType, addr, shape, /*slice=*/nullptr, |
567 | lenParams, /*tdesc=*/nullptr); |
568 | } |
569 | } |
570 | |
571 | if (mustFinalize) |
572 | fir::runtime::genDerivedTypeFinalize(builder, loc, var); |
573 | |
574 | // If there are allocatable components, they need to be deallocated |
575 | // (regardless of the mustFree and mustFinalize settings). |
576 | if (deallocComponents) |
577 | fir::runtime::genDerivedTypeDestroyWithoutFinalization(builder, loc, var); |
578 | |
579 | if (doFree) |
580 | builder.create<fir::FreeMemOp>(loc, addr); |
581 | }; |
582 | bool deallocComponents = hlfir::mayHaveAllocatableComponent(var.getType()); |
583 | |
584 | auto genFree = [&]() { |
585 | genFreeOrFinalize(/*doFree=*/true, /*deallocComponents=*/false, |
586 | /*doFinalize=*/false); |
587 | }; |
588 | if (auto cstMustFree = fir::getIntIfConstant(mustFree)) { |
589 | genFreeOrFinalize(*cstMustFree != 0 ? true : false, deallocComponents, |
590 | mustFinalize); |
591 | return; |
592 | } |
593 | |
594 | // If mustFree is dynamic, first, deallocate any allocatable |
595 | // components and finalize. |
596 | genFreeOrFinalize(/*doFree=*/false, deallocComponents, |
597 | /*doFinalize=*/mustFinalize); |
598 | // Conditionally free the memory. |
599 | builder.genIfThen(loc, mustFree).genThen(genFree).end(); |
600 | } |
601 | |
602 | struct EndAssociateOpConversion |
603 | : public mlir::OpConversionPattern<hlfir::EndAssociateOp> { |
604 | using mlir::OpConversionPattern<hlfir::EndAssociateOp>::OpConversionPattern; |
605 | explicit EndAssociateOpConversion(mlir::MLIRContext *ctx) |
606 | : mlir::OpConversionPattern<hlfir::EndAssociateOp>{ctx} {} |
607 | llvm::LogicalResult |
608 | matchAndRewrite(hlfir::EndAssociateOp endAssociate, OpAdaptor adaptor, |
609 | mlir::ConversionPatternRewriter &rewriter) const override { |
610 | mlir::Location loc = endAssociate->getLoc(); |
611 | fir::FirOpBuilder builder(rewriter, endAssociate.getOperation()); |
612 | genBufferDestruction(loc, builder, adaptor.getVar(), adaptor.getMustFree(), |
613 | /*mustFinalize=*/false); |
614 | rewriter.eraseOp(endAssociate); |
615 | return mlir::success(); |
616 | } |
617 | }; |
618 | |
619 | struct DestroyOpConversion |
620 | : public mlir::OpConversionPattern<hlfir::DestroyOp> { |
621 | using mlir::OpConversionPattern<hlfir::DestroyOp>::OpConversionPattern; |
622 | explicit DestroyOpConversion(mlir::MLIRContext *ctx) |
623 | : mlir::OpConversionPattern<hlfir::DestroyOp>{ctx} {} |
624 | llvm::LogicalResult |
625 | matchAndRewrite(hlfir::DestroyOp destroy, OpAdaptor adaptor, |
626 | mlir::ConversionPatternRewriter &rewriter) const override { |
627 | // If expr was bufferized on the heap, now is time to deallocate the buffer. |
628 | mlir::Location loc = destroy->getLoc(); |
629 | hlfir::Entity bufferizedExpr = getBufferizedExprStorage(adaptor.getExpr()); |
630 | if (!fir::isa_trivial(bufferizedExpr.getType())) { |
631 | fir::FirOpBuilder builder(rewriter, destroy.getOperation()); |
632 | mlir::Value mustFree = getBufferizedExprMustFreeFlag(adaptor.getExpr()); |
633 | // Passing FIR base might be enough for cases when |
634 | // component deallocation and finalization are not required. |
635 | // If extra BoxAddr operations become a performance problem, |
636 | // we may pass both bases and let genBufferDestruction decide |
637 | // which one to use. |
638 | mlir::Value base = bufferizedExpr.getBase(); |
639 | genBufferDestruction(loc, builder, base, mustFree, |
640 | destroy.mustFinalizeExpr()); |
641 | } |
642 | |
643 | rewriter.eraseOp(destroy); |
644 | return mlir::success(); |
645 | } |
646 | }; |
647 | |
648 | struct NoReassocOpConversion |
649 | : public mlir::OpConversionPattern<hlfir::NoReassocOp> { |
650 | using mlir::OpConversionPattern<hlfir::NoReassocOp>::OpConversionPattern; |
651 | explicit NoReassocOpConversion(mlir::MLIRContext *ctx) |
652 | : mlir::OpConversionPattern<hlfir::NoReassocOp>{ctx} {} |
653 | llvm::LogicalResult |
654 | matchAndRewrite(hlfir::NoReassocOp noreassoc, OpAdaptor adaptor, |
655 | mlir::ConversionPatternRewriter &rewriter) const override { |
656 | mlir::Location loc = noreassoc->getLoc(); |
657 | fir::FirOpBuilder builder(rewriter, noreassoc.getOperation()); |
658 | mlir::Value bufferizedExpr = getBufferizedExprStorage(adaptor.getVal()); |
659 | mlir::Value result = |
660 | builder.create<hlfir::NoReassocOp>(loc, bufferizedExpr); |
661 | |
662 | if (!fir::isa_trivial(bufferizedExpr.getType())) { |
663 | // NoReassocOp should not be needed on the mustFree path. |
664 | mlir::Value mustFree = getBufferizedExprMustFreeFlag(adaptor.getVal()); |
665 | result = |
666 | packageBufferizedExpr(loc, builder, hlfir::Entity{result}, mustFree); |
667 | } |
668 | rewriter.replaceOp(noreassoc, result); |
669 | return mlir::success(); |
670 | } |
671 | }; |
672 | |
673 | /// Was \p value created in the mlir block where \p builder is currently set ? |
674 | static bool wasCreatedInCurrentBlock(mlir::Value value, |
675 | fir::FirOpBuilder &builder) { |
676 | if (mlir::Operation *op = value.getDefiningOp()) |
677 | return op->getBlock() == builder.getBlock(); |
678 | return false; |
679 | } |
680 | |
681 | /// This Listener allows setting both the builder and the rewriter as |
682 | /// listeners. This is required when a pattern uses a firBuilder helper that |
683 | /// may create illegal operations that will need to be translated and requires |
684 | /// notifying the rewriter. |
685 | struct HLFIRListener : public mlir::OpBuilder::Listener { |
686 | HLFIRListener(fir::FirOpBuilder &builder, |
687 | mlir::ConversionPatternRewriter &rewriter) |
688 | : builder{builder}, rewriter{rewriter} {} |
689 | void notifyOperationInserted(mlir::Operation *op, |
690 | mlir::OpBuilder::InsertPoint previous) override { |
691 | builder.notifyOperationInserted(op, previous); |
692 | rewriter.getListener()->notifyOperationInserted(op, previous); |
693 | } |
694 | virtual void notifyBlockInserted(mlir::Block *block, mlir::Region *previous, |
695 | mlir::Region::iterator previousIt) override { |
696 | builder.notifyBlockInserted(block, previous, previousIt); |
697 | rewriter.getListener()->notifyBlockInserted(block, previous, previousIt); |
698 | } |
699 | fir::FirOpBuilder &builder; |
700 | mlir::ConversionPatternRewriter &rewriter; |
701 | }; |
702 | |
703 | struct ElementalOpConversion |
704 | : public mlir::OpConversionPattern<hlfir::ElementalOp> { |
705 | using mlir::OpConversionPattern<hlfir::ElementalOp>::OpConversionPattern; |
706 | explicit ElementalOpConversion(mlir::MLIRContext *ctx, |
707 | bool optimizeEmptyElementals = false) |
708 | : mlir::OpConversionPattern<hlfir::ElementalOp>{ctx}, |
709 | optimizeEmptyElementals(optimizeEmptyElementals) { |
710 | // This pattern recursively converts nested ElementalOp's |
711 | // by cloning and then converting them, so we have to allow |
712 | // for recursive pattern application. The recursion is bounded |
713 | // by the nesting level of ElementalOp's. |
714 | setHasBoundedRewriteRecursion(); |
715 | } |
716 | llvm::LogicalResult |
717 | matchAndRewrite(hlfir::ElementalOp elemental, OpAdaptor adaptor, |
718 | mlir::ConversionPatternRewriter &rewriter) const override { |
719 | mlir::Location loc = elemental->getLoc(); |
720 | fir::FirOpBuilder builder(rewriter, elemental.getOperation()); |
721 | // The body of the elemental op may contain operation that will require |
722 | // to be translated. Notify the rewriter about the cloned operations. |
723 | HLFIRListener listener{builder, rewriter}; |
724 | builder.setListener(&listener); |
725 | |
726 | mlir::Value shape = adaptor.getShape(); |
727 | std::optional<hlfir::Entity> mold; |
728 | if (adaptor.getMold()) |
729 | mold = getBufferizedExprStorage(adaptor.getMold()); |
730 | auto extents = hlfir::getIndexExtents(loc, builder, shape); |
731 | llvm::SmallVector<mlir::Value> typeParams(adaptor.getTypeparams().begin(), |
732 | adaptor.getTypeparams().end()); |
733 | auto [temp, cleanup] = createArrayTemp(loc, builder, elemental.getType(), |
734 | shape, extents, typeParams, mold); |
735 | |
736 | if (optimizeEmptyElementals) |
737 | extents = fir::factory::updateRuntimeExtentsForEmptyArrays(builder, loc, |
738 | extents); |
739 | |
740 | // Generate a loop nest looping around the fir.elemental shape and clone |
741 | // fir.elemental region inside the inner loop. |
742 | hlfir::LoopNest loopNest = |
743 | hlfir::genLoopNest(loc, builder, extents, !elemental.isOrdered(), |
744 | flangomp::shouldUseWorkshareLowering(elemental)); |
745 | auto insPt = builder.saveInsertionPoint(); |
746 | builder.setInsertionPointToStart(loopNest.body); |
747 | auto yield = hlfir::inlineElementalOp(loc, builder, elemental, |
748 | loopNest.oneBasedIndices); |
749 | hlfir::Entity elementValue(yield.getElementValue()); |
750 | // Skip final AsExpr if any. It would create an element temporary, |
751 | // which is no needed since the element will be assigned right away in |
752 | // the array temporary. An hlfir.as_expr may have been added if the |
753 | // elemental is a "view" over a variable (e.g parentheses or transpose). |
754 | if (auto asExpr = elementValue.getDefiningOp<hlfir::AsExprOp>()) { |
755 | if (asExpr->hasOneUse() && !asExpr.isMove()) { |
756 | // Check that the asExpr is the final operation before the yield, |
757 | // otherwise, clean-ups could impact the memory being re-used. |
758 | if (asExpr->getNextNode() == yield.getOperation()) { |
759 | elementValue = hlfir::Entity{asExpr.getVar()}; |
760 | rewriter.eraseOp(asExpr); |
761 | } |
762 | } |
763 | } |
764 | rewriter.eraseOp(yield); |
765 | // Assign the element value to the temp element for this iteration. |
766 | auto tempElement = |
767 | hlfir::getElementAt(loc, builder, temp, loopNest.oneBasedIndices); |
768 | // If the elemental result is a temporary of a derived type, |
769 | // we can avoid the deep copy implied by the AssignOp and just |
770 | // do the shallow copy with load/store. This helps avoiding the overhead |
771 | // of deallocating allocatable components of the temporary (if any) |
772 | // on each iteration of the elemental operation. |
773 | auto asExpr = elementValue.getDefiningOp<hlfir::AsExprOp>(); |
774 | auto elemType = hlfir::getFortranElementType(elementValue.getType()); |
775 | if (asExpr && asExpr.isMove() && mlir::isa<fir::RecordType>(elemType) && |
776 | hlfir::mayHaveAllocatableComponent(elemType) && |
777 | wasCreatedInCurrentBlock(elementValue, builder)) { |
778 | auto load = builder.create<fir::LoadOp>(loc, asExpr.getVar()); |
779 | builder.create<fir::StoreOp>(loc, load, tempElement); |
780 | } else { |
781 | builder.create<hlfir::AssignOp>(loc, elementValue, tempElement, |
782 | /*realloc=*/false, |
783 | /*keep_lhs_length_if_realloc=*/false, |
784 | /*temporary_lhs=*/true); |
785 | |
786 | // hlfir.yield_element implicitly marks the end-of-life its operand if |
787 | // it is an expression created in the hlfir.elemental (since it is its |
788 | // last use and an hlfir.destroy could not be created afterwards) |
789 | // Now that this node has been removed and the expression has been used in |
790 | // the assign, insert an hlfir.destroy to mark the expression end-of-life. |
791 | // If the expression creation allocated a buffer on the heap inside the |
792 | // loop, this will ensure the buffer properly deallocated. |
793 | if (mlir::isa<hlfir::ExprType>(elementValue.getType()) && |
794 | wasCreatedInCurrentBlock(elementValue, builder)) |
795 | builder.create<hlfir::DestroyOp>(loc, elementValue); |
796 | } |
797 | builder.restoreInsertionPoint(insPt); |
798 | |
799 | mlir::Value bufferizedExpr = |
800 | packageBufferizedExpr(loc, builder, temp, cleanup); |
801 | // Explicitly delete the body of the elemental to get rid |
802 | // of any users of hlfir.expr values inside the body as early |
803 | // as possible. |
804 | rewriter.startOpModification(elemental); |
805 | rewriter.eraseBlock(elemental.getBody()); |
806 | rewriter.finalizeOpModification(elemental); |
807 | rewriter.replaceOp(elemental, bufferizedExpr); |
808 | return mlir::success(); |
809 | } |
810 | |
811 | private: |
812 | bool optimizeEmptyElementals = false; |
813 | }; |
814 | struct CharExtremumOpConversion |
815 | : public mlir::OpConversionPattern<hlfir::CharExtremumOp> { |
816 | using mlir::OpConversionPattern<hlfir::CharExtremumOp>::OpConversionPattern; |
817 | explicit CharExtremumOpConversion(mlir::MLIRContext *ctx) |
818 | : mlir::OpConversionPattern<hlfir::CharExtremumOp>{ctx} {} |
819 | llvm::LogicalResult |
820 | matchAndRewrite(hlfir::CharExtremumOp char_extremum, OpAdaptor adaptor, |
821 | mlir::ConversionPatternRewriter &rewriter) const override { |
822 | mlir::Location loc = char_extremum->getLoc(); |
823 | auto predicate = char_extremum.getPredicate(); |
824 | bool predIsMin = |
825 | predicate == hlfir::CharExtremumPredicate::min ? true : false; |
826 | fir::FirOpBuilder builder(rewriter, char_extremum.getOperation()); |
827 | assert(adaptor.getStrings().size() >= 2 && |
828 | "must have at least two strings operands" ); |
829 | auto numOperands = adaptor.getStrings().size(); |
830 | |
831 | std::vector<hlfir::Entity> chars; |
832 | std::vector< |
833 | std::pair<fir::ExtendedValue, std::optional<hlfir::CleanupFunction>>> |
834 | pairs; |
835 | llvm::SmallVector<fir::CharBoxValue> opCBVs; |
836 | for (size_t i = 0; i < numOperands; ++i) { |
837 | chars.emplace_back(getBufferizedExprStorage(adaptor.getStrings()[i])); |
838 | pairs.emplace_back( |
839 | hlfir::translateToExtendedValue(loc, builder, chars[i])); |
840 | assert(!pairs[i].second && "expected variables" ); |
841 | opCBVs.emplace_back(*pairs[i].first.getCharBox()); |
842 | } |
843 | |
844 | fir::ExtendedValue res = |
845 | fir::factory::CharacterExprHelper{builder, loc}.createCharExtremum( |
846 | predIsMin, opCBVs); |
847 | mlir::Type addrType = fir::ReferenceType::get( |
848 | hlfir::getFortranElementType(char_extremum.getResult().getType())); |
849 | mlir::Value cast = builder.createConvert(loc, addrType, fir::getBase(res)); |
850 | res = fir::substBase(res, cast); |
851 | hlfir::Entity hlfirTempRes = |
852 | hlfir::Entity{hlfir::genDeclare(loc, builder, res, ".tmp.char_extremum" , |
853 | fir::FortranVariableFlagsAttr{}) |
854 | .getBase()}; |
855 | mlir::Value bufferizedExpr = |
856 | packageBufferizedExpr(loc, builder, hlfirTempRes, false); |
857 | rewriter.replaceOp(char_extremum, bufferizedExpr); |
858 | return mlir::success(); |
859 | } |
860 | }; |
861 | |
862 | struct EvaluateInMemoryOpConversion |
863 | : public mlir::OpConversionPattern<hlfir::EvaluateInMemoryOp> { |
864 | using mlir::OpConversionPattern< |
865 | hlfir::EvaluateInMemoryOp>::OpConversionPattern; |
866 | explicit EvaluateInMemoryOpConversion(mlir::MLIRContext *ctx) |
867 | : mlir::OpConversionPattern<hlfir::EvaluateInMemoryOp>{ctx} {} |
868 | llvm::LogicalResult |
869 | matchAndRewrite(hlfir::EvaluateInMemoryOp evalInMemOp, OpAdaptor adaptor, |
870 | mlir::ConversionPatternRewriter &rewriter) const override { |
871 | mlir::Location loc = evalInMemOp->getLoc(); |
872 | fir::FirOpBuilder builder(rewriter, evalInMemOp.getOperation()); |
873 | auto [temp, isHeapAlloc] = hlfir::computeEvaluateOpInNewTemp( |
874 | loc, builder, evalInMemOp, adaptor.getShape(), adaptor.getTypeparams()); |
875 | mlir::Value bufferizedExpr = |
876 | packageBufferizedExpr(loc, builder, temp, isHeapAlloc); |
877 | rewriter.replaceOp(evalInMemOp, bufferizedExpr); |
878 | return mlir::success(); |
879 | } |
880 | }; |
881 | |
882 | class BufferizeHLFIR : public hlfir::impl::BufferizeHLFIRBase<BufferizeHLFIR> { |
883 | public: |
884 | using BufferizeHLFIRBase<BufferizeHLFIR>::BufferizeHLFIRBase; |
885 | |
886 | void runOnOperation() override { |
887 | // TODO: make this a pass operating on FuncOp. The issue is that |
888 | // FirOpBuilder helpers may generate new FuncOp because of runtime/llvm |
889 | // intrinsics calls creation. This may create race conflict if the pass is |
890 | // scheduled on FuncOp. A solution could be to provide an optional mutex |
891 | // when building a FirOpBuilder and locking around FuncOp and GlobalOp |
892 | // creation, but this needs a bit more thinking, so at this point the pass |
893 | // is scheduled on the moduleOp. |
894 | auto module = this->getOperation(); |
895 | auto *context = &getContext(); |
896 | mlir::RewritePatternSet patterns(context); |
897 | patterns.insert<ApplyOpConversion, AsExprOpConversion, AssignOpConversion, |
898 | AssociateOpConversion, CharExtremumOpConversion, |
899 | ConcatOpConversion, DestroyOpConversion, |
900 | EndAssociateOpConversion, EvaluateInMemoryOpConversion, |
901 | NoReassocOpConversion, SetLengthOpConversion, |
902 | ShapeOfOpConversion, GetLengthOpConversion>(context); |
903 | patterns.insert<ElementalOpConversion>(context, optimizeEmptyElementals); |
904 | mlir::ConversionTarget target(*context); |
905 | // Note that YieldElementOp is not marked as an illegal operation. |
906 | // It must be erased by its parent converter and there is no explicit |
907 | // conversion pattern to YieldElementOp itself. If any YieldElementOp |
908 | // survives this pass, the verifier will detect it because it has to be |
909 | // a child of ElementalOp and ElementalOp's are explicitly illegal. |
910 | target.addIllegalOp<hlfir::ApplyOp, hlfir::AssociateOp, hlfir::ElementalOp, |
911 | hlfir::EndAssociateOp, hlfir::SetLengthOp>(); |
912 | |
913 | target.markUnknownOpDynamicallyLegal([](mlir::Operation *op) { |
914 | return llvm::all_of(op->getResultTypes(), |
915 | [](mlir::Type ty) { |
916 | return !mlir::isa<hlfir::ExprType>(ty); |
917 | }) && |
918 | llvm::all_of(op->getOperandTypes(), [](mlir::Type ty) { |
919 | return !mlir::isa<hlfir::ExprType>(ty); |
920 | }); |
921 | }); |
922 | if (mlir::failed( |
923 | mlir::applyFullConversion(module, target, std::move(patterns)))) { |
924 | mlir::emitError(mlir::UnknownLoc::get(context), |
925 | "failure in HLFIR bufferization pass" ); |
926 | signalPassFailure(); |
927 | } |
928 | } |
929 | }; |
930 | } // namespace |
931 | |