1//===- BufferizeHLFIR.cpp - Bufferize HLFIR ------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8// This file defines a pass that bufferize hlfir.expr. It translates operations
9// producing or consuming hlfir.expr into operations operating on memory.
10// An hlfir.expr is translated to a tuple<variable address, cleanupflag>
11// where cleanupflag is set to true if storage for the expression was allocated
12// on the heap.
13//===----------------------------------------------------------------------===//
14
15#include "flang/Optimizer/Builder/Character.h"
16#include "flang/Optimizer/Builder/FIRBuilder.h"
17#include "flang/Optimizer/Builder/HLFIRTools.h"
18#include "flang/Optimizer/Builder/MutableBox.h"
19#include "flang/Optimizer/Builder/Runtime/Allocatable.h"
20#include "flang/Optimizer/Builder/Runtime/Derived.h"
21#include "flang/Optimizer/Builder/Todo.h"
22#include "flang/Optimizer/Dialect/FIRDialect.h"
23#include "flang/Optimizer/Dialect/FIROps.h"
24#include "flang/Optimizer/Dialect/FIRType.h"
25#include "flang/Optimizer/Dialect/Support/FIRContext.h"
26#include "flang/Optimizer/HLFIR/HLFIRDialect.h"
27#include "flang/Optimizer/HLFIR/HLFIROps.h"
28#include "flang/Optimizer/HLFIR/Passes.h"
29#include "flang/Optimizer/OpenMP/Passes.h"
30#include "mlir/Dialect/OpenMP/OpenMPDialect.h"
31#include "mlir/IR/Dominance.h"
32#include "mlir/IR/PatternMatch.h"
33#include "mlir/Pass/Pass.h"
34#include "mlir/Pass/PassManager.h"
35#include "mlir/Transforms/DialectConversion.h"
36#include "llvm/ADT/TypeSwitch.h"
37
38namespace hlfir {
39#define GEN_PASS_DEF_BUFFERIZEHLFIR
40#include "flang/Optimizer/HLFIR/Passes.h.inc"
41} // namespace hlfir
42
43namespace {
44
45/// Helper to create tuple from a bufferized expr storage and clean up
46/// instruction flag. The storage is an HLFIR variable so that it can
47/// be manipulated as a variable later (all shape and length information
48/// cam be retrieved from it).
49static mlir::Value packageBufferizedExpr(mlir::Location loc,
50 fir::FirOpBuilder &builder,
51 hlfir::Entity storage,
52 mlir::Value mustFree) {
53 auto tupleType = mlir::TupleType::get(
54 builder.getContext(),
55 mlir::TypeRange{storage.getType(), mustFree.getType()});
56 auto undef = builder.create<fir::UndefOp>(loc, tupleType);
57 auto insert = builder.create<fir::InsertValueOp>(
58 loc, tupleType, undef, mustFree,
59 builder.getArrayAttr(
60 {builder.getIntegerAttr(builder.getIndexType(), 1)}));
61 return builder.create<fir::InsertValueOp>(
62 loc, tupleType, insert, storage,
63 builder.getArrayAttr(
64 {builder.getIntegerAttr(builder.getIndexType(), 0)}));
65}
66
67/// Helper to create tuple from a bufferized expr storage and constant
68/// boolean clean-up flag.
69static mlir::Value packageBufferizedExpr(mlir::Location loc,
70 fir::FirOpBuilder &builder,
71 hlfir::Entity storage, bool mustFree) {
72 mlir::Value mustFreeValue = builder.createBool(loc, mustFree);
73 return packageBufferizedExpr(loc, builder, storage, mustFreeValue);
74}
75
76/// Helper to extract the storage from a tuple created by packageBufferizedExpr.
77/// It assumes no tuples are used as HLFIR operation operands, which is
78/// currently enforced by the verifiers that only accept HLFIR value or
79/// variable types which do not include tuples.
80static hlfir::Entity getBufferizedExprStorage(mlir::Value bufferizedExpr) {
81 auto tupleType = mlir::dyn_cast<mlir::TupleType>(bufferizedExpr.getType());
82 if (!tupleType)
83 return hlfir::Entity{bufferizedExpr};
84 assert(tupleType.size() == 2 && "unexpected tuple type");
85 if (auto insert = bufferizedExpr.getDefiningOp<fir::InsertValueOp>())
86 if (insert.getVal().getType() == tupleType.getType(0))
87 return hlfir::Entity{insert.getVal()};
88 TODO(bufferizedExpr.getLoc(), "general extract storage case");
89}
90
91/// Helper to extract the clean-up flag from a tuple created by
92/// packageBufferizedExpr.
93static mlir::Value getBufferizedExprMustFreeFlag(mlir::Value bufferizedExpr) {
94 auto tupleType = mlir::dyn_cast<mlir::TupleType>(bufferizedExpr.getType());
95 if (!tupleType)
96 return bufferizedExpr;
97 assert(tupleType.size() == 2 && "unexpected tuple type");
98 if (auto insert = bufferizedExpr.getDefiningOp<fir::InsertValueOp>())
99 if (auto insert0 = insert.getAdt().getDefiningOp<fir::InsertValueOp>())
100 if (insert0.getVal().getType() == tupleType.getType(1))
101 return insert0.getVal();
102 TODO(bufferizedExpr.getLoc(), "general extract storage case");
103}
104
105static std::pair<hlfir::Entity, mlir::Value>
106createArrayTemp(mlir::Location loc, fir::FirOpBuilder &builder,
107 mlir::Type exprType, mlir::Value shape,
108 llvm::ArrayRef<mlir::Value> extents,
109 llvm::ArrayRef<mlir::Value> lenParams,
110 std::optional<hlfir::Entity> polymorphicMold) {
111 auto sequenceType = mlir::cast<fir::SequenceType>(
112 hlfir::getFortranElementOrSequenceType(exprType));
113
114 auto genTempDeclareOp =
115 [](fir::FirOpBuilder &builder, mlir::Location loc, mlir::Value memref,
116 llvm::StringRef name, mlir::Value shape,
117 llvm::ArrayRef<mlir::Value> typeParams,
118 fir::FortranVariableFlagsAttr attrs) -> mlir::Value {
119 auto declareOp =
120 builder.create<hlfir::DeclareOp>(loc, memref, name, shape, typeParams,
121 /*dummy_scope=*/nullptr, attrs);
122 return declareOp.getBase();
123 };
124
125 auto [base, isHeapAlloc] = builder.createArrayTemp(
126 loc, sequenceType, shape, extents, lenParams, genTempDeclareOp,
127 polymorphicMold ? polymorphicMold->getFirBase() : nullptr);
128 hlfir::Entity temp = hlfir::Entity{base};
129 assert(!temp.isAllocatable() && "temp must have been allocated");
130 return {temp, builder.createBool(loc, isHeapAlloc)};
131}
132
133/// Copy \p source into a new temporary and package the temporary into a
134/// <temp,cleanup> tuple. The temporary may be heap or stack allocated.
135static mlir::Value copyInTempAndPackage(mlir::Location loc,
136 fir::FirOpBuilder &builder,
137 hlfir::Entity source) {
138 auto [temp, cleanup] = hlfir::createTempFromMold(loc, builder, source);
139 assert(!temp.isAllocatable() && "expect temp to already be allocated");
140 builder.create<hlfir::AssignOp>(loc, source, temp, /*realloc=*/false,
141 /*keep_lhs_length_if_realloc=*/false,
142 /*temporary_lhs=*/true);
143 return packageBufferizedExpr(loc, builder, temp, cleanup);
144}
145
146struct AsExprOpConversion : public mlir::OpConversionPattern<hlfir::AsExprOp> {
147 using mlir::OpConversionPattern<hlfir::AsExprOp>::OpConversionPattern;
148 explicit AsExprOpConversion(mlir::MLIRContext *ctx)
149 : mlir::OpConversionPattern<hlfir::AsExprOp>{ctx} {}
150 llvm::LogicalResult
151 matchAndRewrite(hlfir::AsExprOp asExpr, OpAdaptor adaptor,
152 mlir::ConversionPatternRewriter &rewriter) const override {
153 mlir::Location loc = asExpr->getLoc();
154 auto module = asExpr->getParentOfType<mlir::ModuleOp>();
155 fir::FirOpBuilder builder(rewriter, module);
156 if (asExpr.isMove()) {
157 // Move variable storage for the hlfir.expr buffer.
158 mlir::Value bufferizedExpr = packageBufferizedExpr(
159 loc, builder, hlfir::Entity{adaptor.getVar()}, adaptor.getMustFree());
160 rewriter.replaceOp(asExpr, bufferizedExpr);
161 return mlir::success();
162 }
163 // Otherwise, create a copy in a new buffer.
164 hlfir::Entity source = hlfir::Entity{adaptor.getVar()};
165 mlir::Value bufferizedExpr = copyInTempAndPackage(loc, builder, source);
166 rewriter.replaceOp(asExpr, bufferizedExpr);
167 return mlir::success();
168 }
169};
170
171struct ShapeOfOpConversion
172 : public mlir::OpConversionPattern<hlfir::ShapeOfOp> {
173 using mlir::OpConversionPattern<hlfir::ShapeOfOp>::OpConversionPattern;
174
175 llvm::LogicalResult
176 matchAndRewrite(hlfir::ShapeOfOp shapeOf, OpAdaptor adaptor,
177 mlir::ConversionPatternRewriter &rewriter) const override {
178 mlir::Location loc = shapeOf.getLoc();
179 mlir::ModuleOp mod = shapeOf->getParentOfType<mlir::ModuleOp>();
180 fir::FirOpBuilder builder(rewriter, mod);
181
182 mlir::Value shape;
183 hlfir::Entity bufferizedExpr{getBufferizedExprStorage(adaptor.getExpr())};
184 if (bufferizedExpr.isVariable()) {
185 shape = hlfir::genShape(loc, builder, bufferizedExpr);
186 } else {
187 // everything else failed so try to create a shape from static type info
188 hlfir::ExprType exprTy =
189 mlir::dyn_cast_or_null<hlfir::ExprType>(adaptor.getExpr().getType());
190 if (exprTy)
191 shape = hlfir::genExprShape(builder, loc, exprTy);
192 }
193 // expected to never happen
194 if (!shape)
195 return emitError(loc,
196 "Unresolvable hlfir.shape_of where extents are unknown");
197
198 rewriter.replaceOp(shapeOf, shape);
199 return mlir::success();
200 }
201};
202
203struct ApplyOpConversion : public mlir::OpConversionPattern<hlfir::ApplyOp> {
204 using mlir::OpConversionPattern<hlfir::ApplyOp>::OpConversionPattern;
205 explicit ApplyOpConversion(mlir::MLIRContext *ctx)
206 : mlir::OpConversionPattern<hlfir::ApplyOp>{ctx} {}
207 llvm::LogicalResult
208 matchAndRewrite(hlfir::ApplyOp apply, OpAdaptor adaptor,
209 mlir::ConversionPatternRewriter &rewriter) const override {
210 mlir::Location loc = apply->getLoc();
211 hlfir::Entity bufferizedExpr = getBufferizedExprStorage(adaptor.getExpr());
212 mlir::Type resultType = hlfir::getVariableElementType(bufferizedExpr);
213 mlir::Value result = rewriter.create<hlfir::DesignateOp>(
214 loc, resultType, bufferizedExpr, adaptor.getIndices(),
215 adaptor.getTypeparams());
216 if (fir::isa_trivial(apply.getType())) {
217 result = rewriter.create<fir::LoadOp>(loc, result);
218 } else {
219 fir::FirOpBuilder builder(rewriter, apply.getOperation());
220 result =
221 packageBufferizedExpr(loc, builder, hlfir::Entity{result}, false);
222 }
223 rewriter.replaceOp(apply, result);
224 return mlir::success();
225 }
226};
227
228struct AssignOpConversion : public mlir::OpConversionPattern<hlfir::AssignOp> {
229 using mlir::OpConversionPattern<hlfir::AssignOp>::OpConversionPattern;
230 explicit AssignOpConversion(mlir::MLIRContext *ctx)
231 : mlir::OpConversionPattern<hlfir::AssignOp>{ctx} {}
232 llvm::LogicalResult
233 matchAndRewrite(hlfir::AssignOp assign, OpAdaptor adaptor,
234 mlir::ConversionPatternRewriter &rewriter) const override {
235 llvm::SmallVector<mlir::Value> newOperands;
236 for (mlir::Value operand : adaptor.getOperands())
237 newOperands.push_back(getBufferizedExprStorage(operand));
238 rewriter.startOpModification(assign);
239 assign->setOperands(newOperands);
240 rewriter.finalizeOpModification(assign);
241 return mlir::success();
242 }
243};
244
245struct ConcatOpConversion : public mlir::OpConversionPattern<hlfir::ConcatOp> {
246 using mlir::OpConversionPattern<hlfir::ConcatOp>::OpConversionPattern;
247 explicit ConcatOpConversion(mlir::MLIRContext *ctx)
248 : mlir::OpConversionPattern<hlfir::ConcatOp>{ctx} {}
249 llvm::LogicalResult
250 matchAndRewrite(hlfir::ConcatOp concat, OpAdaptor adaptor,
251 mlir::ConversionPatternRewriter &rewriter) const override {
252 mlir::Location loc = concat->getLoc();
253 fir::FirOpBuilder builder(rewriter, concat.getOperation());
254 assert(adaptor.getStrings().size() >= 2 &&
255 "must have at least two strings operands");
256 if (adaptor.getStrings().size() > 2)
257 TODO(loc, "codegen of optimized chained concatenation of more than two "
258 "strings");
259 hlfir::Entity lhs = getBufferizedExprStorage(adaptor.getStrings()[0]);
260 hlfir::Entity rhs = getBufferizedExprStorage(adaptor.getStrings()[1]);
261 auto [lhsExv, c1] = hlfir::translateToExtendedValue(loc, builder, lhs);
262 auto [rhsExv, c2] = hlfir::translateToExtendedValue(loc, builder, rhs);
263 assert(!c1 && !c2 && "expected variables");
264 fir::ExtendedValue res =
265 fir::factory::CharacterExprHelper{builder, loc}.createConcatenate(
266 *lhsExv.getCharBox(), *rhsExv.getCharBox());
267 // Ensure the memory type is the same as the result type.
268 mlir::Type addrType = fir::ReferenceType::get(
269 hlfir::getFortranElementType(concat.getResult().getType()));
270 mlir::Value cast = builder.createConvert(loc, addrType, fir::getBase(res));
271 res = fir::substBase(res, cast);
272 hlfir::Entity hlfirTempRes =
273 hlfir::Entity{hlfir::genDeclare(loc, builder, res, "tmp",
274 fir::FortranVariableFlagsAttr{})
275 .getBase()};
276 mlir::Value bufferizedExpr =
277 packageBufferizedExpr(loc, builder, hlfirTempRes, false);
278 rewriter.replaceOp(concat, bufferizedExpr);
279 return mlir::success();
280 }
281};
282
283struct SetLengthOpConversion
284 : public mlir::OpConversionPattern<hlfir::SetLengthOp> {
285 using mlir::OpConversionPattern<hlfir::SetLengthOp>::OpConversionPattern;
286 explicit SetLengthOpConversion(mlir::MLIRContext *ctx)
287 : mlir::OpConversionPattern<hlfir::SetLengthOp>{ctx} {}
288 llvm::LogicalResult
289 matchAndRewrite(hlfir::SetLengthOp setLength, OpAdaptor adaptor,
290 mlir::ConversionPatternRewriter &rewriter) const override {
291 mlir::Location loc = setLength->getLoc();
292 fir::FirOpBuilder builder(rewriter, setLength.getOperation());
293 // Create a temp with the new length.
294 hlfir::Entity string = getBufferizedExprStorage(adaptor.getString());
295 auto charType = hlfir::getFortranElementType(setLength.getType());
296 llvm::StringRef tmpName{".tmp"};
297 llvm::SmallVector<mlir::Value, 1> lenParams{adaptor.getLength()};
298 auto alloca = builder.createTemporary(loc, charType, tmpName,
299 /*shape=*/std::nullopt, lenParams);
300 auto declareOp = builder.create<hlfir::DeclareOp>(
301 loc, alloca, tmpName, /*shape=*/mlir::Value{}, lenParams,
302 /*dummy_scope=*/nullptr, fir::FortranVariableFlagsAttr{});
303 hlfir::Entity temp{declareOp.getBase()};
304 // Assign string value to the created temp.
305 builder.create<hlfir::AssignOp>(loc, string, temp,
306 /*realloc=*/false,
307 /*keep_lhs_length_if_realloc=*/false,
308 /*temporary_lhs=*/true);
309 mlir::Value bufferizedExpr =
310 packageBufferizedExpr(loc, builder, temp, false);
311 rewriter.replaceOp(setLength, bufferizedExpr);
312 return mlir::success();
313 }
314};
315
316struct GetLengthOpConversion
317 : public mlir::OpConversionPattern<hlfir::GetLengthOp> {
318 using mlir::OpConversionPattern<hlfir::GetLengthOp>::OpConversionPattern;
319 explicit GetLengthOpConversion(mlir::MLIRContext *ctx)
320 : mlir::OpConversionPattern<hlfir::GetLengthOp>{ctx} {}
321 llvm::LogicalResult
322 matchAndRewrite(hlfir::GetLengthOp getLength, OpAdaptor adaptor,
323 mlir::ConversionPatternRewriter &rewriter) const override {
324 mlir::Location loc = getLength->getLoc();
325 fir::FirOpBuilder builder(rewriter, getLength.getOperation());
326 hlfir::Entity bufferizedExpr = getBufferizedExprStorage(adaptor.getExpr());
327 mlir::Value length = hlfir::genCharLength(loc, builder, bufferizedExpr);
328 if (!length)
329 return rewriter.notifyMatchFailure(
330 getLength, "could not deduce length from GetLengthOp operand");
331 length = builder.createConvert(loc, builder.getIndexType(), length);
332 rewriter.replaceOp(getLength, length);
333 return mlir::success();
334 }
335};
336
337/// The current hlfir.associate lowering does not handle multiple uses of a
338/// non-trivial expression value because it generates the cleanup for the
339/// expression bufferization at hlfir.end_associate. If there was more than one
340/// hlfir.end_associate, it would be cleaned up multiple times, perhaps before
341/// one of the other uses.
342/// Note that we have to be careful about expressions used by a single
343/// hlfir.end_associate that may be executed more times than the producer
344/// of the expression value. This may also cause multiple clean-ups
345/// for the same memory (e.g. cause double-free errors). For example,
346/// hlfir.end_associate inside hlfir.elemental may cause such issues
347/// for expressions produced outside of hlfir.elemental.
348static bool allOtherUsesAreSafeForAssociate(mlir::Value value,
349 mlir::Operation *currentUse,
350 mlir::Operation *endAssociate) {
351 // If value producer is from a different region than
352 // hlfir.associate/end_associate, then conservatively assume
353 // that the hlfir.end_associate may execute more times than
354 // the value producer.
355 // TODO: this may be improved for operations that cannot
356 // result in multiple executions (e.g. ifOp).
357 if (value.getParentRegion() != currentUse->getParentRegion() ||
358 (endAssociate &&
359 value.getParentRegion() != endAssociate->getParentRegion()))
360 return false;
361
362 for (mlir::Operation *useOp : value.getUsers()) {
363 // Ignore DestroyOp's that do not imply finalization.
364 // If finalization is implied, then we must delegate
365 // the finalization to the correspoding EndAssociateOp,
366 // but we currently do not; so we disable the buffer
367 // reuse in this case.
368 if (auto destroy = mlir::dyn_cast<hlfir::DestroyOp>(useOp)) {
369 if (destroy.mustFinalizeExpr())
370 return false;
371 else
372 continue;
373 }
374
375 if (useOp != currentUse) {
376 // hlfir.shape_of and hlfir.get_length will not disrupt cleanup so it is
377 // safe for hlfir.associate. These operations might read from the box and
378 // so they need to come before the hflir.end_associate (which may
379 // deallocate).
380 if (mlir::isa<hlfir::ShapeOfOp>(useOp) ||
381 mlir::isa<hlfir::GetLengthOp>(useOp)) {
382 if (!endAssociate)
383 continue;
384 // If useOp dominates the endAssociate, then it is definitely safe.
385 if (useOp->getBlock() != endAssociate->getBlock())
386 if (mlir::DominanceInfo{}.dominates(useOp, endAssociate))
387 continue;
388 if (useOp->isBeforeInBlock(endAssociate))
389 continue;
390 }
391 return false;
392 }
393 }
394 return true;
395}
396
397static void eraseAllUsesInDestroys(mlir::Value value,
398 mlir::ConversionPatternRewriter &rewriter) {
399 for (mlir::Operation *useOp : value.getUsers())
400 if (auto destroy = mlir::dyn_cast<hlfir::DestroyOp>(useOp)) {
401 assert(!destroy.mustFinalizeExpr() &&
402 "deleting DestroyOp with finalize attribute");
403 rewriter.eraseOp(destroy);
404 }
405}
406
407struct AssociateOpConversion
408 : public mlir::OpConversionPattern<hlfir::AssociateOp> {
409 using mlir::OpConversionPattern<hlfir::AssociateOp>::OpConversionPattern;
410 explicit AssociateOpConversion(mlir::MLIRContext *ctx)
411 : mlir::OpConversionPattern<hlfir::AssociateOp>{ctx} {}
412 llvm::LogicalResult
413 matchAndRewrite(hlfir::AssociateOp associate, OpAdaptor adaptor,
414 mlir::ConversionPatternRewriter &rewriter) const override {
415 mlir::Location loc = associate->getLoc();
416 fir::FirOpBuilder builder(rewriter, associate.getOperation());
417 mlir::Value bufferizedExpr = getBufferizedExprStorage(adaptor.getSource());
418 const bool isTrivialValue = fir::isa_trivial(bufferizedExpr.getType());
419
420 auto getEndAssociate =
421 [](hlfir::AssociateOp associate) -> mlir::Operation * {
422 for (mlir::Operation *useOp : associate->getUsers())
423 if (mlir::isa<hlfir::EndAssociateOp>(useOp))
424 return useOp;
425 // happens in some hand coded mlir in tests
426 return nullptr;
427 };
428
429 auto replaceWith = [&](mlir::Value hlfirVar, mlir::Value firVar,
430 mlir::Value flag) {
431 // 0-dim variables may need special handling:
432 // %0 = hlfir.as_expr %x move %true :
433 // (!fir.box<!fir.heap<!fir.type<_T{y:i32}>>>, i1) ->
434 // !hlfir.expr<!fir.type<_T{y:i32}>>
435 // %1:3 = hlfir.associate %0 {adapt.valuebyref} :
436 // (!hlfir.expr<!fir.type<_T{y:i32}>>) ->
437 // (!fir.ref<!fir.type<_T{y:i32}>>,
438 // !fir.ref<!fir.type<_T{y:i32}>>,
439 // i1)
440 //
441 // !fir.box<!fir.heap<!fir.type<_T{y:i32}>>> value must be
442 // propagated as the box address !fir.ref<!fir.type<_T{y:i32}>>.
443 auto adjustVar = [&](mlir::Value sourceVar, mlir::Type assocType) {
444 if ((mlir::isa<fir::BaseBoxType>(sourceVar.getType()) &&
445 !mlir::isa<fir::BaseBoxType>(assocType)) ||
446 ((mlir::isa<fir::BoxCharType>(sourceVar.getType()) &&
447 !mlir::isa<fir::BoxCharType>(assocType)))) {
448 sourceVar = builder.create<fir::BoxAddrOp>(loc, assocType, sourceVar);
449 } else {
450 sourceVar = builder.createConvert(loc, assocType, sourceVar);
451 }
452 return sourceVar;
453 };
454
455 mlir::Type associateHlfirVarType = associate.getResultTypes()[0];
456 hlfirVar = adjustVar(hlfirVar, associateHlfirVarType);
457 associate.getResult(0).replaceAllUsesWith(hlfirVar);
458
459 mlir::Type associateFirVarType = associate.getResultTypes()[1];
460 firVar = adjustVar(firVar, associateFirVarType);
461 associate.getResult(1).replaceAllUsesWith(firVar);
462 associate.getResult(2).replaceAllUsesWith(flag);
463 // FIXME: note that the AssociateOp that is being erased
464 // here will continue to be a user of the original Source
465 // operand (e.g. a result of hlfir.elemental), because
466 // the erasure is not immediate in the rewriter.
467 // In case there are multiple uses of the Source operand,
468 // the allOtherUsesAreSafeForAssociate() below will always
469 // see them, so there is no way to reuse the buffer.
470 // I think we have to run this analysis before doing
471 // the conversions, so that we can analyze HLFIR in its
472 // original form and decide which of the AssociateOp
473 // users of hlfir.expr can reuse the buffer (if it can).
474 rewriter.eraseOp(associate);
475 };
476
477 // If this is the last use of the expression value and this is an hlfir.expr
478 // that was bufferized, re-use the storage.
479 // Otherwise, create a temp and assign the storage to it.
480 //
481 // WARNING: it is important to use the original Source operand
482 // of the AssociateOp to look for the users, because its replacement
483 // has zero materialized users at this point.
484 // So allOtherUsesAreSafeForAssociate() may incorrectly return
485 // true here.
486 if (!isTrivialValue && allOtherUsesAreSafeForAssociate(
487 associate.getSource(), associate.getOperation(),
488 getEndAssociate(associate))) {
489 // Re-use hlfir.expr buffer if this is the only use of the hlfir.expr
490 // outside of the hlfir.destroy. Take on the cleaning-up responsibility
491 // for the related hlfir.end_associate, and erase the hlfir.destroy (if
492 // any).
493 mlir::Value mustFree = getBufferizedExprMustFreeFlag(adaptor.getSource());
494 mlir::Value firBase = hlfir::Entity{bufferizedExpr}.getFirBase();
495 replaceWith(bufferizedExpr, firBase, mustFree);
496 eraseAllUsesInDestroys(associate.getSource(), rewriter);
497 // Make sure to erase the hlfir.destroy if there is an indirection through
498 // a hlfir.no_reassoc operation.
499 if (auto noReassoc = mlir::dyn_cast_or_null<hlfir::NoReassocOp>(
500 associate.getSource().getDefiningOp()))
501 eraseAllUsesInDestroys(noReassoc.getVal(), rewriter);
502 return mlir::success();
503 }
504 if (isTrivialValue) {
505 llvm::SmallVector<mlir::NamedAttribute, 1> attrs;
506 if (associate->hasAttr(fir::getAdaptToByRefAttrName())) {
507 attrs.push_back(fir::getAdaptToByRefAttr(builder));
508 }
509 llvm::StringRef name = "";
510 if (associate.getUniqName())
511 name = *associate.getUniqName();
512 auto temp =
513 builder.createTemporary(loc, bufferizedExpr.getType(), name, attrs);
514 builder.create<fir::StoreOp>(loc, bufferizedExpr, temp);
515 mlir::Value mustFree = builder.createBool(loc, false);
516 replaceWith(temp, temp, mustFree);
517 return mlir::success();
518 }
519 // non-trivial value with more than one use. We will have to make a copy and
520 // use that
521 hlfir::Entity source = hlfir::Entity{bufferizedExpr};
522 mlir::Value bufferTuple = copyInTempAndPackage(loc, builder, source);
523 bufferizedExpr = getBufferizedExprStorage(bufferTuple);
524 replaceWith(bufferizedExpr, hlfir::Entity{bufferizedExpr}.getFirBase(),
525 getBufferizedExprMustFreeFlag(bufferTuple));
526 return mlir::success();
527 }
528};
529
530static void genBufferDestruction(mlir::Location loc, fir::FirOpBuilder &builder,
531 mlir::Value var, mlir::Value mustFree,
532 bool mustFinalize) {
533 auto genFreeOrFinalize = [&](bool doFree, bool deallocComponents,
534 bool doFinalize) {
535 if (!doFree && !deallocComponents && !doFinalize)
536 return;
537
538 mlir::Value addr = var;
539
540 // fir::FreeMemOp operand type must be a fir::HeapType.
541 mlir::Type heapType = fir::HeapType::get(
542 hlfir::getFortranElementOrSequenceType(var.getType()));
543 if (mlir::isa<fir::BaseBoxType, fir::BoxCharType>(var.getType())) {
544 if (mustFinalize && !mlir::isa<fir::BaseBoxType>(var.getType()))
545 fir::emitFatalError(loc, "non-finalizable variable");
546
547 addr = builder.create<fir::BoxAddrOp>(loc, heapType, var);
548 } else {
549 if (!mlir::isa<fir::HeapType>(var.getType()))
550 addr = builder.create<fir::ConvertOp>(loc, heapType, var);
551
552 if (mustFinalize || deallocComponents) {
553 // Embox the raw pointer using proper shape and type params
554 // (note that the shape might be visible via the array finalization
555 // routines).
556 if (!hlfir::isFortranEntity(var))
557 TODO(loc, "need a Fortran entity to create a box");
558
559 hlfir::Entity entity{var};
560 llvm::SmallVector<mlir::Value> lenParams;
561 hlfir::genLengthParameters(loc, builder, entity, lenParams);
562 mlir::Value shape;
563 if (entity.isArray())
564 shape = hlfir::genShape(loc, builder, entity);
565 mlir::Type boxType = fir::BoxType::get(heapType);
566 var = builder.createBox(loc, boxType, addr, shape, /*slice=*/nullptr,
567 lenParams, /*tdesc=*/nullptr);
568 }
569 }
570
571 if (mustFinalize)
572 fir::runtime::genDerivedTypeFinalize(builder, loc, var);
573
574 // If there are allocatable components, they need to be deallocated
575 // (regardless of the mustFree and mustFinalize settings).
576 if (deallocComponents)
577 fir::runtime::genDerivedTypeDestroyWithoutFinalization(builder, loc, var);
578
579 if (doFree)
580 builder.create<fir::FreeMemOp>(loc, addr);
581 };
582 bool deallocComponents = hlfir::mayHaveAllocatableComponent(var.getType());
583
584 auto genFree = [&]() {
585 genFreeOrFinalize(/*doFree=*/true, /*deallocComponents=*/false,
586 /*doFinalize=*/false);
587 };
588 if (auto cstMustFree = fir::getIntIfConstant(mustFree)) {
589 genFreeOrFinalize(*cstMustFree != 0 ? true : false, deallocComponents,
590 mustFinalize);
591 return;
592 }
593
594 // If mustFree is dynamic, first, deallocate any allocatable
595 // components and finalize.
596 genFreeOrFinalize(/*doFree=*/false, deallocComponents,
597 /*doFinalize=*/mustFinalize);
598 // Conditionally free the memory.
599 builder.genIfThen(loc, mustFree).genThen(genFree).end();
600}
601
602struct EndAssociateOpConversion
603 : public mlir::OpConversionPattern<hlfir::EndAssociateOp> {
604 using mlir::OpConversionPattern<hlfir::EndAssociateOp>::OpConversionPattern;
605 explicit EndAssociateOpConversion(mlir::MLIRContext *ctx)
606 : mlir::OpConversionPattern<hlfir::EndAssociateOp>{ctx} {}
607 llvm::LogicalResult
608 matchAndRewrite(hlfir::EndAssociateOp endAssociate, OpAdaptor adaptor,
609 mlir::ConversionPatternRewriter &rewriter) const override {
610 mlir::Location loc = endAssociate->getLoc();
611 fir::FirOpBuilder builder(rewriter, endAssociate.getOperation());
612 genBufferDestruction(loc, builder, adaptor.getVar(), adaptor.getMustFree(),
613 /*mustFinalize=*/false);
614 rewriter.eraseOp(endAssociate);
615 return mlir::success();
616 }
617};
618
619struct DestroyOpConversion
620 : public mlir::OpConversionPattern<hlfir::DestroyOp> {
621 using mlir::OpConversionPattern<hlfir::DestroyOp>::OpConversionPattern;
622 explicit DestroyOpConversion(mlir::MLIRContext *ctx)
623 : mlir::OpConversionPattern<hlfir::DestroyOp>{ctx} {}
624 llvm::LogicalResult
625 matchAndRewrite(hlfir::DestroyOp destroy, OpAdaptor adaptor,
626 mlir::ConversionPatternRewriter &rewriter) const override {
627 // If expr was bufferized on the heap, now is time to deallocate the buffer.
628 mlir::Location loc = destroy->getLoc();
629 hlfir::Entity bufferizedExpr = getBufferizedExprStorage(adaptor.getExpr());
630 if (!fir::isa_trivial(bufferizedExpr.getType())) {
631 fir::FirOpBuilder builder(rewriter, destroy.getOperation());
632 mlir::Value mustFree = getBufferizedExprMustFreeFlag(adaptor.getExpr());
633 // Passing FIR base might be enough for cases when
634 // component deallocation and finalization are not required.
635 // If extra BoxAddr operations become a performance problem,
636 // we may pass both bases and let genBufferDestruction decide
637 // which one to use.
638 mlir::Value base = bufferizedExpr.getBase();
639 genBufferDestruction(loc, builder, base, mustFree,
640 destroy.mustFinalizeExpr());
641 }
642
643 rewriter.eraseOp(destroy);
644 return mlir::success();
645 }
646};
647
648struct NoReassocOpConversion
649 : public mlir::OpConversionPattern<hlfir::NoReassocOp> {
650 using mlir::OpConversionPattern<hlfir::NoReassocOp>::OpConversionPattern;
651 explicit NoReassocOpConversion(mlir::MLIRContext *ctx)
652 : mlir::OpConversionPattern<hlfir::NoReassocOp>{ctx} {}
653 llvm::LogicalResult
654 matchAndRewrite(hlfir::NoReassocOp noreassoc, OpAdaptor adaptor,
655 mlir::ConversionPatternRewriter &rewriter) const override {
656 mlir::Location loc = noreassoc->getLoc();
657 fir::FirOpBuilder builder(rewriter, noreassoc.getOperation());
658 mlir::Value bufferizedExpr = getBufferizedExprStorage(adaptor.getVal());
659 mlir::Value result =
660 builder.create<hlfir::NoReassocOp>(loc, bufferizedExpr);
661
662 if (!fir::isa_trivial(bufferizedExpr.getType())) {
663 // NoReassocOp should not be needed on the mustFree path.
664 mlir::Value mustFree = getBufferizedExprMustFreeFlag(adaptor.getVal());
665 result =
666 packageBufferizedExpr(loc, builder, hlfir::Entity{result}, mustFree);
667 }
668 rewriter.replaceOp(noreassoc, result);
669 return mlir::success();
670 }
671};
672
673/// Was \p value created in the mlir block where \p builder is currently set ?
674static bool wasCreatedInCurrentBlock(mlir::Value value,
675 fir::FirOpBuilder &builder) {
676 if (mlir::Operation *op = value.getDefiningOp())
677 return op->getBlock() == builder.getBlock();
678 return false;
679}
680
681/// This Listener allows setting both the builder and the rewriter as
682/// listeners. This is required when a pattern uses a firBuilder helper that
683/// may create illegal operations that will need to be translated and requires
684/// notifying the rewriter.
685struct HLFIRListener : public mlir::OpBuilder::Listener {
686 HLFIRListener(fir::FirOpBuilder &builder,
687 mlir::ConversionPatternRewriter &rewriter)
688 : builder{builder}, rewriter{rewriter} {}
689 void notifyOperationInserted(mlir::Operation *op,
690 mlir::OpBuilder::InsertPoint previous) override {
691 builder.notifyOperationInserted(op, previous);
692 rewriter.getListener()->notifyOperationInserted(op, previous);
693 }
694 virtual void notifyBlockInserted(mlir::Block *block, mlir::Region *previous,
695 mlir::Region::iterator previousIt) override {
696 builder.notifyBlockInserted(block, previous, previousIt);
697 rewriter.getListener()->notifyBlockInserted(block, previous, previousIt);
698 }
699 fir::FirOpBuilder &builder;
700 mlir::ConversionPatternRewriter &rewriter;
701};
702
703struct ElementalOpConversion
704 : public mlir::OpConversionPattern<hlfir::ElementalOp> {
705 using mlir::OpConversionPattern<hlfir::ElementalOp>::OpConversionPattern;
706 explicit ElementalOpConversion(mlir::MLIRContext *ctx,
707 bool optimizeEmptyElementals = false)
708 : mlir::OpConversionPattern<hlfir::ElementalOp>{ctx},
709 optimizeEmptyElementals(optimizeEmptyElementals) {
710 // This pattern recursively converts nested ElementalOp's
711 // by cloning and then converting them, so we have to allow
712 // for recursive pattern application. The recursion is bounded
713 // by the nesting level of ElementalOp's.
714 setHasBoundedRewriteRecursion();
715 }
716 llvm::LogicalResult
717 matchAndRewrite(hlfir::ElementalOp elemental, OpAdaptor adaptor,
718 mlir::ConversionPatternRewriter &rewriter) const override {
719 mlir::Location loc = elemental->getLoc();
720 fir::FirOpBuilder builder(rewriter, elemental.getOperation());
721 // The body of the elemental op may contain operation that will require
722 // to be translated. Notify the rewriter about the cloned operations.
723 HLFIRListener listener{builder, rewriter};
724 builder.setListener(&listener);
725
726 mlir::Value shape = adaptor.getShape();
727 std::optional<hlfir::Entity> mold;
728 if (adaptor.getMold())
729 mold = getBufferizedExprStorage(adaptor.getMold());
730 auto extents = hlfir::getIndexExtents(loc, builder, shape);
731 llvm::SmallVector<mlir::Value> typeParams(adaptor.getTypeparams().begin(),
732 adaptor.getTypeparams().end());
733 auto [temp, cleanup] = createArrayTemp(loc, builder, elemental.getType(),
734 shape, extents, typeParams, mold);
735
736 if (optimizeEmptyElementals)
737 extents = fir::factory::updateRuntimeExtentsForEmptyArrays(builder, loc,
738 extents);
739
740 // Generate a loop nest looping around the fir.elemental shape and clone
741 // fir.elemental region inside the inner loop.
742 hlfir::LoopNest loopNest =
743 hlfir::genLoopNest(loc, builder, extents, !elemental.isOrdered(),
744 flangomp::shouldUseWorkshareLowering(elemental));
745 auto insPt = builder.saveInsertionPoint();
746 builder.setInsertionPointToStart(loopNest.body);
747 auto yield = hlfir::inlineElementalOp(loc, builder, elemental,
748 loopNest.oneBasedIndices);
749 hlfir::Entity elementValue(yield.getElementValue());
750 // Skip final AsExpr if any. It would create an element temporary,
751 // which is no needed since the element will be assigned right away in
752 // the array temporary. An hlfir.as_expr may have been added if the
753 // elemental is a "view" over a variable (e.g parentheses or transpose).
754 if (auto asExpr = elementValue.getDefiningOp<hlfir::AsExprOp>()) {
755 if (asExpr->hasOneUse() && !asExpr.isMove()) {
756 // Check that the asExpr is the final operation before the yield,
757 // otherwise, clean-ups could impact the memory being re-used.
758 if (asExpr->getNextNode() == yield.getOperation()) {
759 elementValue = hlfir::Entity{asExpr.getVar()};
760 rewriter.eraseOp(asExpr);
761 }
762 }
763 }
764 rewriter.eraseOp(yield);
765 // Assign the element value to the temp element for this iteration.
766 auto tempElement =
767 hlfir::getElementAt(loc, builder, temp, loopNest.oneBasedIndices);
768 // If the elemental result is a temporary of a derived type,
769 // we can avoid the deep copy implied by the AssignOp and just
770 // do the shallow copy with load/store. This helps avoiding the overhead
771 // of deallocating allocatable components of the temporary (if any)
772 // on each iteration of the elemental operation.
773 auto asExpr = elementValue.getDefiningOp<hlfir::AsExprOp>();
774 auto elemType = hlfir::getFortranElementType(elementValue.getType());
775 if (asExpr && asExpr.isMove() && mlir::isa<fir::RecordType>(elemType) &&
776 hlfir::mayHaveAllocatableComponent(elemType) &&
777 wasCreatedInCurrentBlock(elementValue, builder)) {
778 auto load = builder.create<fir::LoadOp>(loc, asExpr.getVar());
779 builder.create<fir::StoreOp>(loc, load, tempElement);
780 } else {
781 builder.create<hlfir::AssignOp>(loc, elementValue, tempElement,
782 /*realloc=*/false,
783 /*keep_lhs_length_if_realloc=*/false,
784 /*temporary_lhs=*/true);
785
786 // hlfir.yield_element implicitly marks the end-of-life its operand if
787 // it is an expression created in the hlfir.elemental (since it is its
788 // last use and an hlfir.destroy could not be created afterwards)
789 // Now that this node has been removed and the expression has been used in
790 // the assign, insert an hlfir.destroy to mark the expression end-of-life.
791 // If the expression creation allocated a buffer on the heap inside the
792 // loop, this will ensure the buffer properly deallocated.
793 if (mlir::isa<hlfir::ExprType>(elementValue.getType()) &&
794 wasCreatedInCurrentBlock(elementValue, builder))
795 builder.create<hlfir::DestroyOp>(loc, elementValue);
796 }
797 builder.restoreInsertionPoint(insPt);
798
799 mlir::Value bufferizedExpr =
800 packageBufferizedExpr(loc, builder, temp, cleanup);
801 // Explicitly delete the body of the elemental to get rid
802 // of any users of hlfir.expr values inside the body as early
803 // as possible.
804 rewriter.startOpModification(elemental);
805 rewriter.eraseBlock(elemental.getBody());
806 rewriter.finalizeOpModification(elemental);
807 rewriter.replaceOp(elemental, bufferizedExpr);
808 return mlir::success();
809 }
810
811private:
812 bool optimizeEmptyElementals = false;
813};
814struct CharExtremumOpConversion
815 : public mlir::OpConversionPattern<hlfir::CharExtremumOp> {
816 using mlir::OpConversionPattern<hlfir::CharExtremumOp>::OpConversionPattern;
817 explicit CharExtremumOpConversion(mlir::MLIRContext *ctx)
818 : mlir::OpConversionPattern<hlfir::CharExtremumOp>{ctx} {}
819 llvm::LogicalResult
820 matchAndRewrite(hlfir::CharExtremumOp char_extremum, OpAdaptor adaptor,
821 mlir::ConversionPatternRewriter &rewriter) const override {
822 mlir::Location loc = char_extremum->getLoc();
823 auto predicate = char_extremum.getPredicate();
824 bool predIsMin =
825 predicate == hlfir::CharExtremumPredicate::min ? true : false;
826 fir::FirOpBuilder builder(rewriter, char_extremum.getOperation());
827 assert(adaptor.getStrings().size() >= 2 &&
828 "must have at least two strings operands");
829 auto numOperands = adaptor.getStrings().size();
830
831 std::vector<hlfir::Entity> chars;
832 std::vector<
833 std::pair<fir::ExtendedValue, std::optional<hlfir::CleanupFunction>>>
834 pairs;
835 llvm::SmallVector<fir::CharBoxValue> opCBVs;
836 for (size_t i = 0; i < numOperands; ++i) {
837 chars.emplace_back(getBufferizedExprStorage(adaptor.getStrings()[i]));
838 pairs.emplace_back(
839 hlfir::translateToExtendedValue(loc, builder, chars[i]));
840 assert(!pairs[i].second && "expected variables");
841 opCBVs.emplace_back(*pairs[i].first.getCharBox());
842 }
843
844 fir::ExtendedValue res =
845 fir::factory::CharacterExprHelper{builder, loc}.createCharExtremum(
846 predIsMin, opCBVs);
847 mlir::Type addrType = fir::ReferenceType::get(
848 hlfir::getFortranElementType(char_extremum.getResult().getType()));
849 mlir::Value cast = builder.createConvert(loc, addrType, fir::getBase(res));
850 res = fir::substBase(res, cast);
851 hlfir::Entity hlfirTempRes =
852 hlfir::Entity{hlfir::genDeclare(loc, builder, res, ".tmp.char_extremum",
853 fir::FortranVariableFlagsAttr{})
854 .getBase()};
855 mlir::Value bufferizedExpr =
856 packageBufferizedExpr(loc, builder, hlfirTempRes, false);
857 rewriter.replaceOp(char_extremum, bufferizedExpr);
858 return mlir::success();
859 }
860};
861
862struct EvaluateInMemoryOpConversion
863 : public mlir::OpConversionPattern<hlfir::EvaluateInMemoryOp> {
864 using mlir::OpConversionPattern<
865 hlfir::EvaluateInMemoryOp>::OpConversionPattern;
866 explicit EvaluateInMemoryOpConversion(mlir::MLIRContext *ctx)
867 : mlir::OpConversionPattern<hlfir::EvaluateInMemoryOp>{ctx} {}
868 llvm::LogicalResult
869 matchAndRewrite(hlfir::EvaluateInMemoryOp evalInMemOp, OpAdaptor adaptor,
870 mlir::ConversionPatternRewriter &rewriter) const override {
871 mlir::Location loc = evalInMemOp->getLoc();
872 fir::FirOpBuilder builder(rewriter, evalInMemOp.getOperation());
873 auto [temp, isHeapAlloc] = hlfir::computeEvaluateOpInNewTemp(
874 loc, builder, evalInMemOp, adaptor.getShape(), adaptor.getTypeparams());
875 mlir::Value bufferizedExpr =
876 packageBufferizedExpr(loc, builder, temp, isHeapAlloc);
877 rewriter.replaceOp(evalInMemOp, bufferizedExpr);
878 return mlir::success();
879 }
880};
881
882class BufferizeHLFIR : public hlfir::impl::BufferizeHLFIRBase<BufferizeHLFIR> {
883public:
884 using BufferizeHLFIRBase<BufferizeHLFIR>::BufferizeHLFIRBase;
885
886 void runOnOperation() override {
887 // TODO: make this a pass operating on FuncOp. The issue is that
888 // FirOpBuilder helpers may generate new FuncOp because of runtime/llvm
889 // intrinsics calls creation. This may create race conflict if the pass is
890 // scheduled on FuncOp. A solution could be to provide an optional mutex
891 // when building a FirOpBuilder and locking around FuncOp and GlobalOp
892 // creation, but this needs a bit more thinking, so at this point the pass
893 // is scheduled on the moduleOp.
894 auto module = this->getOperation();
895 auto *context = &getContext();
896 mlir::RewritePatternSet patterns(context);
897 patterns.insert<ApplyOpConversion, AsExprOpConversion, AssignOpConversion,
898 AssociateOpConversion, CharExtremumOpConversion,
899 ConcatOpConversion, DestroyOpConversion,
900 EndAssociateOpConversion, EvaluateInMemoryOpConversion,
901 NoReassocOpConversion, SetLengthOpConversion,
902 ShapeOfOpConversion, GetLengthOpConversion>(context);
903 patterns.insert<ElementalOpConversion>(context, optimizeEmptyElementals);
904 mlir::ConversionTarget target(*context);
905 // Note that YieldElementOp is not marked as an illegal operation.
906 // It must be erased by its parent converter and there is no explicit
907 // conversion pattern to YieldElementOp itself. If any YieldElementOp
908 // survives this pass, the verifier will detect it because it has to be
909 // a child of ElementalOp and ElementalOp's are explicitly illegal.
910 target.addIllegalOp<hlfir::ApplyOp, hlfir::AssociateOp, hlfir::ElementalOp,
911 hlfir::EndAssociateOp, hlfir::SetLengthOp>();
912
913 target.markUnknownOpDynamicallyLegal([](mlir::Operation *op) {
914 return llvm::all_of(op->getResultTypes(),
915 [](mlir::Type ty) {
916 return !mlir::isa<hlfir::ExprType>(ty);
917 }) &&
918 llvm::all_of(op->getOperandTypes(), [](mlir::Type ty) {
919 return !mlir::isa<hlfir::ExprType>(ty);
920 });
921 });
922 if (mlir::failed(
923 mlir::applyFullConversion(module, target, std::move(patterns)))) {
924 mlir::emitError(mlir::UnknownLoc::get(context),
925 "failure in HLFIR bufferization pass");
926 signalPassFailure();
927 }
928 }
929};
930} // namespace
931

source code of flang/lib/Optimizer/HLFIR/Transforms/BufferizeHLFIR.cpp