1//===- BufferizeHLFIR.cpp - Bufferize HLFIR ------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8// This file defines a pass that bufferize hlfir.expr. It translates operations
9// producing or consuming hlfir.expr into operations operating on memory.
10// An hlfir.expr is translated to a tuple<variable address, cleanupflag>
11// where cleanupflag is set to true if storage for the expression was allocated
12// on the heap.
13//===----------------------------------------------------------------------===//
14
15#include "flang/Optimizer/Builder/Character.h"
16#include "flang/Optimizer/Builder/FIRBuilder.h"
17#include "flang/Optimizer/Builder/HLFIRTools.h"
18#include "flang/Optimizer/Builder/MutableBox.h"
19#include "flang/Optimizer/Builder/Runtime/Allocatable.h"
20#include "flang/Optimizer/Builder/Runtime/Derived.h"
21#include "flang/Optimizer/Builder/Todo.h"
22#include "flang/Optimizer/Dialect/FIRDialect.h"
23#include "flang/Optimizer/Dialect/FIROps.h"
24#include "flang/Optimizer/Dialect/FIRType.h"
25#include "flang/Optimizer/Dialect/Support/FIRContext.h"
26#include "flang/Optimizer/HLFIR/HLFIRDialect.h"
27#include "flang/Optimizer/HLFIR/HLFIROps.h"
28#include "flang/Optimizer/HLFIR/Passes.h"
29#include "mlir/IR/Dominance.h"
30#include "mlir/IR/PatternMatch.h"
31#include "mlir/Pass/Pass.h"
32#include "mlir/Pass/PassManager.h"
33#include "mlir/Support/LogicalResult.h"
34#include "mlir/Transforms/DialectConversion.h"
35#include "llvm/ADT/TypeSwitch.h"
36
37namespace hlfir {
38#define GEN_PASS_DEF_BUFFERIZEHLFIR
39#include "flang/Optimizer/HLFIR/Passes.h.inc"
40} // namespace hlfir
41
42namespace {
43
44/// Helper to create tuple from a bufferized expr storage and clean up
45/// instruction flag. The storage is an HLFIR variable so that it can
46/// be manipulated as a variable later (all shape and length information
47/// cam be retrieved from it).
48static mlir::Value packageBufferizedExpr(mlir::Location loc,
49 fir::FirOpBuilder &builder,
50 hlfir::Entity storage,
51 mlir::Value mustFree) {
52 auto tupleType = mlir::TupleType::get(
53 builder.getContext(),
54 mlir::TypeRange{storage.getType(), mustFree.getType()});
55 auto undef = builder.create<fir::UndefOp>(loc, tupleType);
56 auto insert = builder.create<fir::InsertValueOp>(
57 loc, tupleType, undef, mustFree,
58 builder.getArrayAttr(
59 {builder.getIntegerAttr(builder.getIndexType(), 1)}));
60 return builder.create<fir::InsertValueOp>(
61 loc, tupleType, insert, storage,
62 builder.getArrayAttr(
63 {builder.getIntegerAttr(builder.getIndexType(), 0)}));
64}
65
66/// Helper to create tuple from a bufferized expr storage and constant
67/// boolean clean-up flag.
68static mlir::Value packageBufferizedExpr(mlir::Location loc,
69 fir::FirOpBuilder &builder,
70 hlfir::Entity storage, bool mustFree) {
71 mlir::Value mustFreeValue = builder.createBool(loc, mustFree);
72 return packageBufferizedExpr(loc, builder, storage, mustFreeValue);
73}
74
75/// Helper to extract the storage from a tuple created by packageBufferizedExpr.
76/// It assumes no tuples are used as HLFIR operation operands, which is
77/// currently enforced by the verifiers that only accept HLFIR value or
78/// variable types which do not include tuples.
79static hlfir::Entity getBufferizedExprStorage(mlir::Value bufferizedExpr) {
80 auto tupleType = bufferizedExpr.getType().dyn_cast<mlir::TupleType>();
81 if (!tupleType)
82 return hlfir::Entity{bufferizedExpr};
83 assert(tupleType.size() == 2 && "unexpected tuple type");
84 if (auto insert = bufferizedExpr.getDefiningOp<fir::InsertValueOp>())
85 if (insert.getVal().getType() == tupleType.getType(0))
86 return hlfir::Entity{insert.getVal()};
87 TODO(bufferizedExpr.getLoc(), "general extract storage case");
88}
89
90/// Helper to extract the clean-up flag from a tuple created by
91/// packageBufferizedExpr.
92static mlir::Value getBufferizedExprMustFreeFlag(mlir::Value bufferizedExpr) {
93 auto tupleType = bufferizedExpr.getType().dyn_cast<mlir::TupleType>();
94 if (!tupleType)
95 return bufferizedExpr;
96 assert(tupleType.size() == 2 && "unexpected tuple type");
97 if (auto insert = bufferizedExpr.getDefiningOp<fir::InsertValueOp>())
98 if (auto insert0 = insert.getAdt().getDefiningOp<fir::InsertValueOp>())
99 if (insert0.getVal().getType() == tupleType.getType(1))
100 return insert0.getVal();
101 TODO(bufferizedExpr.getLoc(), "general extract storage case");
102}
103
104static std::pair<hlfir::Entity, mlir::Value>
105createArrayTemp(mlir::Location loc, fir::FirOpBuilder &builder,
106 mlir::Type exprType, mlir::Value shape,
107 mlir::ValueRange extents, mlir::ValueRange lenParams,
108 std::optional<hlfir::Entity> polymorphicMold) {
109 mlir::Type sequenceType = hlfir::getFortranElementOrSequenceType(exprType);
110 llvm::StringRef tmpName{".tmp.array"};
111
112 if (polymorphicMold) {
113 // Create *allocated* polymorphic temporary using the dynamic type
114 // of the mold and the provided shape/extents. The created temporary
115 // array will be written element per element, that is why it has to be
116 // allocated.
117 mlir::Type boxHeapType = fir::HeapType::get(sequenceType);
118 mlir::Value alloc = fir::factory::genNullBoxStorage(
119 builder, loc, fir::ClassType::get(boxHeapType));
120 mlir::Value isHeapAlloc = builder.createBool(loc, true);
121 fir::FortranVariableFlagsAttr declAttrs =
122 fir::FortranVariableFlagsAttr::get(
123 builder.getContext(), fir::FortranVariableFlagsEnum::allocatable);
124
125 auto declareOp = builder.create<hlfir::DeclareOp>(loc, alloc, tmpName,
126 /*shape=*/nullptr,
127 lenParams, declAttrs);
128
129 int rank = extents.size();
130 fir::runtime::genAllocatableApplyMold(builder, loc, alloc,
131 polymorphicMold->getFirBase(), rank);
132 if (!extents.empty()) {
133 mlir::Type idxTy = builder.getIndexType();
134 mlir::Value one = builder.createIntegerConstant(loc, idxTy, 1);
135 unsigned dim = 0;
136 for (mlir::Value extent : extents) {
137 mlir::Value dimIndex = builder.createIntegerConstant(loc, idxTy, dim++);
138 fir::runtime::genAllocatableSetBounds(builder, loc, alloc, dimIndex,
139 one, extent);
140 }
141 }
142 if (!lenParams.empty()) {
143 // We should call AllocatableSetDerivedLength() here.
144 // TODO: does the mold provide the length parameters or
145 // the operation itself or should they be in sync?
146 TODO(loc, "polymorphic type with length parameters in HLFIR");
147 }
148 fir::runtime::genAllocatableAllocate(builder, loc, alloc);
149
150 return {hlfir::Entity{declareOp.getBase()}, isHeapAlloc};
151 }
152
153 mlir::Value allocmem = builder.createHeapTemporary(loc, sequenceType, tmpName,
154 extents, lenParams);
155 auto declareOp =
156 builder.create<hlfir::DeclareOp>(loc, allocmem, tmpName, shape, lenParams,
157 fir::FortranVariableFlagsAttr{});
158 mlir::Value trueVal = builder.createBool(loc, true);
159 return {hlfir::Entity{declareOp.getBase()}, trueVal};
160}
161
162/// Copy \p source into a new temporary and package the temporary into a
163/// <temp,cleanup> tuple. The temporary may be heap or stack allocated.
164static mlir::Value copyInTempAndPackage(mlir::Location loc,
165 fir::FirOpBuilder &builder,
166 hlfir::Entity source) {
167 auto [temp, cleanup] = hlfir::createTempFromMold(loc, builder, source);
168 builder.create<hlfir::AssignOp>(loc, source, temp, temp.isAllocatable(),
169 /*keep_lhs_length_if_realloc=*/false,
170 /*temporary_lhs=*/true);
171 // Dereference allocatable temporary directly to simplify processing
172 // of its uses.
173 if (temp.isAllocatable())
174 temp = hlfir::derefPointersAndAllocatables(loc, builder, temp);
175 return packageBufferizedExpr(loc, builder, temp, cleanup);
176}
177
178struct AsExprOpConversion : public mlir::OpConversionPattern<hlfir::AsExprOp> {
179 using mlir::OpConversionPattern<hlfir::AsExprOp>::OpConversionPattern;
180 explicit AsExprOpConversion(mlir::MLIRContext *ctx)
181 : mlir::OpConversionPattern<hlfir::AsExprOp>{ctx} {}
182 mlir::LogicalResult
183 matchAndRewrite(hlfir::AsExprOp asExpr, OpAdaptor adaptor,
184 mlir::ConversionPatternRewriter &rewriter) const override {
185 mlir::Location loc = asExpr->getLoc();
186 auto module = asExpr->getParentOfType<mlir::ModuleOp>();
187 fir::FirOpBuilder builder(rewriter, module);
188 if (asExpr.isMove()) {
189 // Move variable storage for the hlfir.expr buffer.
190 mlir::Value bufferizedExpr = packageBufferizedExpr(
191 loc, builder, hlfir::Entity{adaptor.getVar()}, adaptor.getMustFree());
192 rewriter.replaceOp(asExpr, bufferizedExpr);
193 return mlir::success();
194 }
195 // Otherwise, create a copy in a new buffer.
196 hlfir::Entity source = hlfir::Entity{adaptor.getVar()};
197 mlir::Value bufferizedExpr = copyInTempAndPackage(loc, builder, source);
198 rewriter.replaceOp(asExpr, bufferizedExpr);
199 return mlir::success();
200 }
201};
202
203struct ShapeOfOpConversion
204 : public mlir::OpConversionPattern<hlfir::ShapeOfOp> {
205 using mlir::OpConversionPattern<hlfir::ShapeOfOp>::OpConversionPattern;
206
207 mlir::LogicalResult
208 matchAndRewrite(hlfir::ShapeOfOp shapeOf, OpAdaptor adaptor,
209 mlir::ConversionPatternRewriter &rewriter) const override {
210 mlir::Location loc = shapeOf.getLoc();
211 mlir::ModuleOp mod = shapeOf->getParentOfType<mlir::ModuleOp>();
212 fir::FirOpBuilder builder(rewriter, mod);
213
214 mlir::Value shape;
215 hlfir::Entity bufferizedExpr{getBufferizedExprStorage(adaptor.getExpr())};
216 if (bufferizedExpr.isVariable()) {
217 shape = hlfir::genShape(loc, builder, bufferizedExpr);
218 } else {
219 // everything else failed so try to create a shape from static type info
220 hlfir::ExprType exprTy =
221 adaptor.getExpr().getType().dyn_cast_or_null<hlfir::ExprType>();
222 if (exprTy)
223 shape = hlfir::genExprShape(builder, loc, exprTy);
224 }
225 // expected to never happen
226 if (!shape)
227 return emitError(loc,
228 "Unresolvable hlfir.shape_of where extents are unknown");
229
230 rewriter.replaceOp(shapeOf, shape);
231 return mlir::success();
232 }
233};
234
235struct ApplyOpConversion : public mlir::OpConversionPattern<hlfir::ApplyOp> {
236 using mlir::OpConversionPattern<hlfir::ApplyOp>::OpConversionPattern;
237 explicit ApplyOpConversion(mlir::MLIRContext *ctx)
238 : mlir::OpConversionPattern<hlfir::ApplyOp>{ctx} {}
239 mlir::LogicalResult
240 matchAndRewrite(hlfir::ApplyOp apply, OpAdaptor adaptor,
241 mlir::ConversionPatternRewriter &rewriter) const override {
242 mlir::Location loc = apply->getLoc();
243 hlfir::Entity bufferizedExpr = getBufferizedExprStorage(adaptor.getExpr());
244 mlir::Type resultType = hlfir::getVariableElementType(bufferizedExpr);
245 mlir::Value result = rewriter.create<hlfir::DesignateOp>(
246 loc, resultType, bufferizedExpr, adaptor.getIndices(),
247 adaptor.getTypeparams());
248 if (fir::isa_trivial(apply.getType())) {
249 result = rewriter.create<fir::LoadOp>(loc, result);
250 } else {
251 fir::FirOpBuilder builder(rewriter, apply.getOperation());
252 result =
253 packageBufferizedExpr(loc, builder, hlfir::Entity{result}, false);
254 }
255 rewriter.replaceOp(apply, result);
256 return mlir::success();
257 }
258};
259
260struct AssignOpConversion : public mlir::OpConversionPattern<hlfir::AssignOp> {
261 using mlir::OpConversionPattern<hlfir::AssignOp>::OpConversionPattern;
262 explicit AssignOpConversion(mlir::MLIRContext *ctx)
263 : mlir::OpConversionPattern<hlfir::AssignOp>{ctx} {}
264 mlir::LogicalResult
265 matchAndRewrite(hlfir::AssignOp assign, OpAdaptor adaptor,
266 mlir::ConversionPatternRewriter &rewriter) const override {
267 llvm::SmallVector<mlir::Value> newOperands;
268 for (mlir::Value operand : adaptor.getOperands())
269 newOperands.push_back(getBufferizedExprStorage(operand));
270 rewriter.startOpModification(assign);
271 assign->setOperands(newOperands);
272 rewriter.finalizeOpModification(assign);
273 return mlir::success();
274 }
275};
276
277struct ConcatOpConversion : public mlir::OpConversionPattern<hlfir::ConcatOp> {
278 using mlir::OpConversionPattern<hlfir::ConcatOp>::OpConversionPattern;
279 explicit ConcatOpConversion(mlir::MLIRContext *ctx)
280 : mlir::OpConversionPattern<hlfir::ConcatOp>{ctx} {}
281 mlir::LogicalResult
282 matchAndRewrite(hlfir::ConcatOp concat, OpAdaptor adaptor,
283 mlir::ConversionPatternRewriter &rewriter) const override {
284 mlir::Location loc = concat->getLoc();
285 fir::FirOpBuilder builder(rewriter, concat.getOperation());
286 assert(adaptor.getStrings().size() >= 2 &&
287 "must have at least two strings operands");
288 if (adaptor.getStrings().size() > 2)
289 TODO(loc, "codegen of optimized chained concatenation of more than two "
290 "strings");
291 hlfir::Entity lhs = getBufferizedExprStorage(adaptor.getStrings()[0]);
292 hlfir::Entity rhs = getBufferizedExprStorage(adaptor.getStrings()[1]);
293 auto [lhsExv, c1] = hlfir::translateToExtendedValue(loc, builder, lhs);
294 auto [rhsExv, c2] = hlfir::translateToExtendedValue(loc, builder, rhs);
295 assert(!c1 && !c2 && "expected variables");
296 fir::ExtendedValue res =
297 fir::factory::CharacterExprHelper{builder, loc}.createConcatenate(
298 *lhsExv.getCharBox(), *rhsExv.getCharBox());
299 // Ensure the memory type is the same as the result type.
300 mlir::Type addrType = fir::ReferenceType::get(
301 hlfir::getFortranElementType(concat.getResult().getType()));
302 mlir::Value cast = builder.createConvert(loc, addrType, fir::getBase(res));
303 res = fir::substBase(res, cast);
304 hlfir::Entity hlfirTempRes =
305 hlfir::Entity{hlfir::genDeclare(loc, builder, res, "tmp",
306 fir::FortranVariableFlagsAttr{})
307 .getBase()};
308 mlir::Value bufferizedExpr =
309 packageBufferizedExpr(loc, builder, hlfirTempRes, false);
310 rewriter.replaceOp(concat, bufferizedExpr);
311 return mlir::success();
312 }
313};
314
315struct SetLengthOpConversion
316 : public mlir::OpConversionPattern<hlfir::SetLengthOp> {
317 using mlir::OpConversionPattern<hlfir::SetLengthOp>::OpConversionPattern;
318 explicit SetLengthOpConversion(mlir::MLIRContext *ctx)
319 : mlir::OpConversionPattern<hlfir::SetLengthOp>{ctx} {}
320 mlir::LogicalResult
321 matchAndRewrite(hlfir::SetLengthOp setLength, OpAdaptor adaptor,
322 mlir::ConversionPatternRewriter &rewriter) const override {
323 mlir::Location loc = setLength->getLoc();
324 fir::FirOpBuilder builder(rewriter, setLength.getOperation());
325 // Create a temp with the new length.
326 hlfir::Entity string = getBufferizedExprStorage(adaptor.getString());
327 auto charType = hlfir::getFortranElementType(setLength.getType());
328 llvm::StringRef tmpName{".tmp"};
329 llvm::SmallVector<mlir::Value, 1> lenParams{adaptor.getLength()};
330 auto alloca = builder.createTemporary(loc, charType, tmpName,
331 /*shape=*/std::nullopt, lenParams);
332 auto declareOp = builder.create<hlfir::DeclareOp>(
333 loc, alloca, tmpName, /*shape=*/mlir::Value{}, lenParams,
334 fir::FortranVariableFlagsAttr{});
335 hlfir::Entity temp{declareOp.getBase()};
336 // Assign string value to the created temp.
337 builder.create<hlfir::AssignOp>(loc, string, temp,
338 /*realloc=*/false,
339 /*keep_lhs_length_if_realloc=*/false,
340 /*temporary_lhs=*/true);
341 mlir::Value bufferizedExpr =
342 packageBufferizedExpr(loc, builder, temp, false);
343 rewriter.replaceOp(setLength, bufferizedExpr);
344 return mlir::success();
345 }
346};
347
348struct GetLengthOpConversion
349 : public mlir::OpConversionPattern<hlfir::GetLengthOp> {
350 using mlir::OpConversionPattern<hlfir::GetLengthOp>::OpConversionPattern;
351 explicit GetLengthOpConversion(mlir::MLIRContext *ctx)
352 : mlir::OpConversionPattern<hlfir::GetLengthOp>{ctx} {}
353 mlir::LogicalResult
354 matchAndRewrite(hlfir::GetLengthOp getLength, OpAdaptor adaptor,
355 mlir::ConversionPatternRewriter &rewriter) const override {
356 mlir::Location loc = getLength->getLoc();
357 fir::FirOpBuilder builder(rewriter, getLength.getOperation());
358 hlfir::Entity bufferizedExpr = getBufferizedExprStorage(adaptor.getExpr());
359 mlir::Value length = hlfir::genCharLength(loc, builder, bufferizedExpr);
360 if (!length)
361 return rewriter.notifyMatchFailure(
362 getLength, "could not deduce length from GetLengthOp operand");
363 rewriter.replaceOp(getLength, length);
364 return mlir::success();
365 }
366};
367
368/// The current hlfir.associate lowering does not handle multiple uses of a
369/// non-trivial expression value because it generates the cleanup for the
370/// expression bufferization at hlfir.end_associate. If there was more than one
371/// hlfir.end_associate, it would be cleaned up multiple times, perhaps before
372/// one of the other uses.
373/// Note that we have to be careful about expressions used by a single
374/// hlfir.end_associate that may be executed more times than the producer
375/// of the expression value. This may also cause multiple clean-ups
376/// for the same memory (e.g. cause double-free errors). For example,
377/// hlfir.end_associate inside hlfir.elemental may cause such issues
378/// for expressions produced outside of hlfir.elemental.
379static bool allOtherUsesAreSafeForAssociate(mlir::Value value,
380 mlir::Operation *currentUse,
381 mlir::Operation *endAssociate) {
382 // If value producer is from a different region than
383 // hlfir.associate/end_associate, then conservatively assume
384 // that the hlfir.end_associate may execute more times than
385 // the value producer.
386 // TODO: this may be improved for operations that cannot
387 // result in multiple executions (e.g. ifOp).
388 if (value.getParentRegion() != currentUse->getParentRegion() ||
389 (endAssociate &&
390 value.getParentRegion() != endAssociate->getParentRegion()))
391 return false;
392
393 for (mlir::Operation *useOp : value.getUsers()) {
394 // Ignore DestroyOp's that do not imply finalization.
395 // If finalization is implied, then we must delegate
396 // the finalization to the correspoding EndAssociateOp,
397 // but we currently do not; so we disable the buffer
398 // reuse in this case.
399 if (auto destroy = mlir::dyn_cast<hlfir::DestroyOp>(useOp)) {
400 if (destroy.mustFinalizeExpr())
401 return false;
402 else
403 continue;
404 }
405
406 if (useOp != currentUse) {
407 // hlfir.shape_of and hlfir.get_length will not disrupt cleanup so it is
408 // safe for hlfir.associate. These operations might read from the box and
409 // so they need to come before the hflir.end_associate (which may
410 // deallocate).
411 if (mlir::isa<hlfir::ShapeOfOp>(useOp) ||
412 mlir::isa<hlfir::GetLengthOp>(useOp)) {
413 if (!endAssociate)
414 continue;
415 // If useOp dominates the endAssociate, then it is definitely safe.
416 if (useOp->getBlock() != endAssociate->getBlock())
417 if (mlir::DominanceInfo{}.dominates(useOp, endAssociate))
418 continue;
419 if (useOp->isBeforeInBlock(endAssociate))
420 continue;
421 }
422 return false;
423 }
424 }
425 return true;
426}
427
428static void eraseAllUsesInDestroys(mlir::Value value,
429 mlir::ConversionPatternRewriter &rewriter) {
430 for (mlir::Operation *useOp : value.getUsers())
431 if (auto destroy = mlir::dyn_cast<hlfir::DestroyOp>(useOp)) {
432 assert(!destroy.mustFinalizeExpr() &&
433 "deleting DestroyOp with finalize attribute");
434 rewriter.eraseOp(destroy);
435 }
436}
437
438struct AssociateOpConversion
439 : public mlir::OpConversionPattern<hlfir::AssociateOp> {
440 using mlir::OpConversionPattern<hlfir::AssociateOp>::OpConversionPattern;
441 explicit AssociateOpConversion(mlir::MLIRContext *ctx)
442 : mlir::OpConversionPattern<hlfir::AssociateOp>{ctx} {}
443 mlir::LogicalResult
444 matchAndRewrite(hlfir::AssociateOp associate, OpAdaptor adaptor,
445 mlir::ConversionPatternRewriter &rewriter) const override {
446 mlir::Location loc = associate->getLoc();
447 fir::FirOpBuilder builder(rewriter, associate.getOperation());
448 mlir::Value bufferizedExpr = getBufferizedExprStorage(adaptor.getSource());
449 const bool isTrivialValue = fir::isa_trivial(bufferizedExpr.getType());
450
451 auto getEndAssociate =
452 [](hlfir::AssociateOp associate) -> mlir::Operation * {
453 for (mlir::Operation *useOp : associate->getUsers())
454 if (mlir::isa<hlfir::EndAssociateOp>(useOp))
455 return useOp;
456 // happens in some hand coded mlir in tests
457 return nullptr;
458 };
459
460 auto replaceWith = [&](mlir::Value hlfirVar, mlir::Value firVar,
461 mlir::Value flag) {
462 // 0-dim variables may need special handling:
463 // %0 = hlfir.as_expr %x move %true :
464 // (!fir.box<!fir.heap<!fir.type<_T{y:i32}>>>, i1) ->
465 // !hlfir.expr<!fir.type<_T{y:i32}>>
466 // %1:3 = hlfir.associate %0 {adapt.valuebyref} :
467 // (!hlfir.expr<!fir.type<_T{y:i32}>>) ->
468 // (!fir.ref<!fir.type<_T{y:i32}>>,
469 // !fir.ref<!fir.type<_T{y:i32}>>,
470 // i1)
471 //
472 // !fir.box<!fir.heap<!fir.type<_T{y:i32}>>> value must be
473 // propagated as the box address !fir.ref<!fir.type<_T{y:i32}>>.
474 auto adjustVar = [&](mlir::Value sourceVar, mlir::Type assocType) {
475 if (mlir::isa<fir::ReferenceType>(sourceVar.getType()) &&
476 mlir::isa<fir::ClassType>(
477 fir::unwrapRefType(sourceVar.getType()))) {
478 // Association of a polymorphic value.
479 sourceVar = builder.create<fir::LoadOp>(loc, sourceVar);
480 assert(mlir::isa<fir::ClassType>(sourceVar.getType()) &&
481 fir::isAllocatableType(sourceVar.getType()));
482 assert(sourceVar.getType() == assocType);
483 } else if ((sourceVar.getType().isa<fir::BaseBoxType>() &&
484 !assocType.isa<fir::BaseBoxType>()) ||
485 ((sourceVar.getType().isa<fir::BoxCharType>() &&
486 !assocType.isa<fir::BoxCharType>()))) {
487 sourceVar = builder.create<fir::BoxAddrOp>(loc, assocType, sourceVar);
488 } else {
489 sourceVar = builder.createConvert(loc, assocType, sourceVar);
490 }
491 return sourceVar;
492 };
493
494 mlir::Type associateHlfirVarType = associate.getResultTypes()[0];
495 hlfirVar = adjustVar(hlfirVar, associateHlfirVarType);
496 associate.getResult(0).replaceAllUsesWith(hlfirVar);
497
498 mlir::Type associateFirVarType = associate.getResultTypes()[1];
499 firVar = adjustVar(firVar, associateFirVarType);
500 associate.getResult(1).replaceAllUsesWith(firVar);
501 associate.getResult(2).replaceAllUsesWith(flag);
502 // FIXME: note that the AssociateOp that is being erased
503 // here will continue to be a user of the original Source
504 // operand (e.g. a result of hlfir.elemental), because
505 // the erasure is not immediate in the rewriter.
506 // In case there are multiple uses of the Source operand,
507 // the allOtherUsesAreSafeForAssociate() below will always
508 // see them, so there is no way to reuse the buffer.
509 // I think we have to run this analysis before doing
510 // the conversions, so that we can analyze HLFIR in its
511 // original form and decide which of the AssociateOp
512 // users of hlfir.expr can reuse the buffer (if it can).
513 rewriter.eraseOp(associate);
514 };
515
516 // If this is the last use of the expression value and this is an hlfir.expr
517 // that was bufferized, re-use the storage.
518 // Otherwise, create a temp and assign the storage to it.
519 //
520 // WARNING: it is important to use the original Source operand
521 // of the AssociateOp to look for the users, because its replacement
522 // has zero materialized users at this point.
523 // So allOtherUsesAreSafeForAssociate() may incorrectly return
524 // true here.
525 if (!isTrivialValue && allOtherUsesAreSafeForAssociate(
526 associate.getSource(), associate.getOperation(),
527 getEndAssociate(associate))) {
528 // Re-use hlfir.expr buffer if this is the only use of the hlfir.expr
529 // outside of the hlfir.destroy. Take on the cleaning-up responsibility
530 // for the related hlfir.end_associate, and erase the hlfir.destroy (if
531 // any).
532 mlir::Value mustFree = getBufferizedExprMustFreeFlag(adaptor.getSource());
533 mlir::Value firBase = hlfir::Entity{bufferizedExpr}.getFirBase();
534 replaceWith(bufferizedExpr, firBase, mustFree);
535 eraseAllUsesInDestroys(associate.getSource(), rewriter);
536 return mlir::success();
537 }
538 if (isTrivialValue) {
539 llvm::SmallVector<mlir::NamedAttribute, 1> attrs;
540 if (associate->hasAttr(fir::getAdaptToByRefAttrName())) {
541 attrs.push_back(fir::getAdaptToByRefAttr(builder));
542 }
543 llvm::StringRef name = "";
544 if (associate.getUniqName())
545 name = *associate.getUniqName();
546 auto temp =
547 builder.createTemporary(loc, bufferizedExpr.getType(), name, attrs);
548 builder.create<fir::StoreOp>(loc, bufferizedExpr, temp);
549 mlir::Value mustFree = builder.createBool(loc, false);
550 replaceWith(temp, temp, mustFree);
551 return mlir::success();
552 }
553 // non-trivial value with more than one use. We will have to make a copy and
554 // use that
555 hlfir::Entity source = hlfir::Entity{bufferizedExpr};
556 mlir::Value bufferTuple = copyInTempAndPackage(loc, builder, source);
557 bufferizedExpr = getBufferizedExprStorage(bufferTuple);
558 replaceWith(bufferizedExpr, hlfir::Entity{bufferizedExpr}.getFirBase(),
559 getBufferizedExprMustFreeFlag(bufferTuple));
560 return mlir::success();
561 }
562};
563
564static void genBufferDestruction(mlir::Location loc, fir::FirOpBuilder &builder,
565 mlir::Value var, mlir::Value mustFree,
566 bool mustFinalize) {
567 auto genFreeOrFinalize = [&](bool doFree, bool deallocComponents,
568 bool doFinalize) {
569 if (!doFree && !deallocComponents && !doFinalize)
570 return;
571
572 mlir::Value addr = var;
573
574 // fir::FreeMemOp operand type must be a fir::HeapType.
575 mlir::Type heapType = fir::HeapType::get(
576 hlfir::getFortranElementOrSequenceType(var.getType()));
577 if (mlir::isa<fir::ReferenceType>(var.getType()) &&
578 mlir::isa<fir::ClassType>(fir::unwrapRefType(var.getType()))) {
579 // A temporary for a polymorphic expression is represented
580 // via an allocatable. Variable type in this case
581 // is !fir.ref<!fir.class<!fir.heap<!fir.type<>>>>.
582 // We need to free the allocatable data, not the box
583 // that is allocated on the stack.
584 var = builder.create<fir::LoadOp>(loc, var);
585 assert(mlir::isa<fir::ClassType>(var.getType()) &&
586 fir::isAllocatableType(var.getType()));
587 addr = builder.create<fir::BoxAddrOp>(loc, heapType, var);
588 // Lowering currently does not produce DestroyOp with 'finalize'
589 // for polymorphic temporaries. It will have to do so, for example,
590 // for MERGE with polymorphic results.
591 if (mustFinalize)
592 TODO(loc, "finalizing polymorphic temporary in HLFIR");
593 } else if (var.getType().isa<fir::BaseBoxType, fir::BoxCharType>()) {
594 if (mustFinalize && !mlir::isa<fir::BaseBoxType>(var.getType()))
595 fir::emitFatalError(loc, "non-finalizable variable");
596
597 addr = builder.create<fir::BoxAddrOp>(loc, heapType, var);
598 } else {
599 if (!var.getType().isa<fir::HeapType>())
600 addr = builder.create<fir::ConvertOp>(loc, heapType, var);
601
602 if (mustFinalize || deallocComponents) {
603 // Embox the raw pointer using proper shape and type params
604 // (note that the shape might be visible via the array finalization
605 // routines).
606 if (!hlfir::isFortranEntity(var))
607 TODO(loc, "need a Fortran entity to create a box");
608
609 hlfir::Entity entity{var};
610 llvm::SmallVector<mlir::Value> lenParams;
611 hlfir::genLengthParameters(loc, builder, entity, lenParams);
612 mlir::Value shape;
613 if (entity.isArray())
614 shape = hlfir::genShape(loc, builder, entity);
615 mlir::Type boxType = fir::BoxType::get(heapType);
616 var = builder.createBox(loc, boxType, addr, shape, /*slice=*/nullptr,
617 lenParams, /*tdesc=*/nullptr);
618 }
619 }
620
621 if (mustFinalize)
622 fir::runtime::genDerivedTypeFinalize(builder, loc, var);
623
624 // If there are allocatable components, they need to be deallocated
625 // (regardless of the mustFree and mustFinalize settings).
626 if (deallocComponents)
627 fir::runtime::genDerivedTypeDestroyWithoutFinalization(builder, loc, var);
628
629 if (doFree)
630 builder.create<fir::FreeMemOp>(loc, addr);
631 };
632 bool deallocComponents = hlfir::mayHaveAllocatableComponent(var.getType());
633
634 auto genFree = [&]() {
635 genFreeOrFinalize(/*doFree=*/true, /*deallocComponents=*/false,
636 /*doFinalize=*/false);
637 };
638 if (auto cstMustFree = fir::getIntIfConstant(mustFree)) {
639 genFreeOrFinalize(*cstMustFree != 0 ? true : false, deallocComponents,
640 mustFinalize);
641 return;
642 }
643
644 // If mustFree is dynamic, first, deallocate any allocatable
645 // components and finalize.
646 genFreeOrFinalize(/*doFree=*/false, deallocComponents,
647 /*doFinalize=*/mustFinalize);
648 // Conditionally free the memory.
649 builder.genIfThen(loc, mustFree).genThen(genFree).end();
650}
651
652struct EndAssociateOpConversion
653 : public mlir::OpConversionPattern<hlfir::EndAssociateOp> {
654 using mlir::OpConversionPattern<hlfir::EndAssociateOp>::OpConversionPattern;
655 explicit EndAssociateOpConversion(mlir::MLIRContext *ctx)
656 : mlir::OpConversionPattern<hlfir::EndAssociateOp>{ctx} {}
657 mlir::LogicalResult
658 matchAndRewrite(hlfir::EndAssociateOp endAssociate, OpAdaptor adaptor,
659 mlir::ConversionPatternRewriter &rewriter) const override {
660 mlir::Location loc = endAssociate->getLoc();
661 fir::FirOpBuilder builder(rewriter, endAssociate.getOperation());
662 genBufferDestruction(loc, builder, adaptor.getVar(), adaptor.getMustFree(),
663 /*mustFinalize=*/false);
664 rewriter.eraseOp(endAssociate);
665 return mlir::success();
666 }
667};
668
669struct DestroyOpConversion
670 : public mlir::OpConversionPattern<hlfir::DestroyOp> {
671 using mlir::OpConversionPattern<hlfir::DestroyOp>::OpConversionPattern;
672 explicit DestroyOpConversion(mlir::MLIRContext *ctx)
673 : mlir::OpConversionPattern<hlfir::DestroyOp>{ctx} {}
674 mlir::LogicalResult
675 matchAndRewrite(hlfir::DestroyOp destroy, OpAdaptor adaptor,
676 mlir::ConversionPatternRewriter &rewriter) const override {
677 // If expr was bufferized on the heap, now is time to deallocate the buffer.
678 mlir::Location loc = destroy->getLoc();
679 hlfir::Entity bufferizedExpr = getBufferizedExprStorage(adaptor.getExpr());
680 if (!fir::isa_trivial(bufferizedExpr.getType())) {
681 fir::FirOpBuilder builder(rewriter, destroy.getOperation());
682 mlir::Value mustFree = getBufferizedExprMustFreeFlag(adaptor.getExpr());
683 // Passing FIR base might be enough for cases when
684 // component deallocation and finalization are not required.
685 // If extra BoxAddr operations become a performance problem,
686 // we may pass both bases and let genBufferDestruction decide
687 // which one to use.
688 mlir::Value base = bufferizedExpr.getBase();
689 genBufferDestruction(loc, builder, base, mustFree,
690 destroy.mustFinalizeExpr());
691 }
692
693 rewriter.eraseOp(destroy);
694 return mlir::success();
695 }
696};
697
698struct NoReassocOpConversion
699 : public mlir::OpConversionPattern<hlfir::NoReassocOp> {
700 using mlir::OpConversionPattern<hlfir::NoReassocOp>::OpConversionPattern;
701 explicit NoReassocOpConversion(mlir::MLIRContext *ctx)
702 : mlir::OpConversionPattern<hlfir::NoReassocOp>{ctx} {}
703 mlir::LogicalResult
704 matchAndRewrite(hlfir::NoReassocOp noreassoc, OpAdaptor adaptor,
705 mlir::ConversionPatternRewriter &rewriter) const override {
706 mlir::Location loc = noreassoc->getLoc();
707 fir::FirOpBuilder builder(rewriter, noreassoc.getOperation());
708 mlir::Value bufferizedExpr = getBufferizedExprStorage(adaptor.getVal());
709 mlir::Value result =
710 builder.create<hlfir::NoReassocOp>(loc, bufferizedExpr);
711
712 if (!fir::isa_trivial(bufferizedExpr.getType())) {
713 // NoReassocOp should not be needed on the mustFree path.
714 mlir::Value mustFree = getBufferizedExprMustFreeFlag(adaptor.getVal());
715 result =
716 packageBufferizedExpr(loc, builder, hlfir::Entity{result}, mustFree);
717 }
718 rewriter.replaceOp(noreassoc, result);
719 return mlir::success();
720 }
721};
722
723/// Was \p value created in the mlir block where \p builder is currently set ?
724static bool wasCreatedInCurrentBlock(mlir::Value value,
725 fir::FirOpBuilder &builder) {
726 if (mlir::Operation *op = value.getDefiningOp())
727 return op->getBlock() == builder.getBlock();
728 return false;
729}
730
731/// This Listener allows setting both the builder and the rewriter as
732/// listeners. This is required when a pattern uses a firBuilder helper that
733/// may create illegal operations that will need to be translated and requires
734/// notifying the rewriter.
735struct HLFIRListener : public mlir::OpBuilder::Listener {
736 HLFIRListener(fir::FirOpBuilder &builder,
737 mlir::ConversionPatternRewriter &rewriter)
738 : builder{builder}, rewriter{rewriter} {}
739 void notifyOperationInserted(mlir::Operation *op,
740 mlir::OpBuilder::InsertPoint previous) override {
741 builder.notifyOperationInserted(op, previous);
742 rewriter.getListener()->notifyOperationInserted(op, previous);
743 }
744 virtual void notifyBlockInserted(mlir::Block *block, mlir::Region *previous,
745 mlir::Region::iterator previousIt) override {
746 builder.notifyBlockInserted(block, previous, previousIt);
747 rewriter.getListener()->notifyBlockInserted(block, previous, previousIt);
748 }
749 fir::FirOpBuilder &builder;
750 mlir::ConversionPatternRewriter &rewriter;
751};
752
753struct ElementalOpConversion
754 : public mlir::OpConversionPattern<hlfir::ElementalOp> {
755 using mlir::OpConversionPattern<hlfir::ElementalOp>::OpConversionPattern;
756 explicit ElementalOpConversion(mlir::MLIRContext *ctx)
757 : mlir::OpConversionPattern<hlfir::ElementalOp>{ctx} {
758 // This pattern recursively converts nested ElementalOp's
759 // by cloning and then converting them, so we have to allow
760 // for recursive pattern application. The recursion is bounded
761 // by the nesting level of ElementalOp's.
762 setHasBoundedRewriteRecursion();
763 }
764 mlir::LogicalResult
765 matchAndRewrite(hlfir::ElementalOp elemental, OpAdaptor adaptor,
766 mlir::ConversionPatternRewriter &rewriter) const override {
767 mlir::Location loc = elemental->getLoc();
768 fir::FirOpBuilder builder(rewriter, elemental.getOperation());
769 // The body of the elemental op may contain operation that will require
770 // to be translated. Notify the rewriter about the cloned operations.
771 HLFIRListener listener{builder, rewriter};
772 builder.setListener(&listener);
773
774 mlir::Value shape = adaptor.getShape();
775 std::optional<hlfir::Entity> mold;
776 if (adaptor.getMold())
777 mold = getBufferizedExprStorage(adaptor.getMold());
778 auto extents = hlfir::getIndexExtents(loc, builder, shape);
779 auto [temp, cleanup] =
780 createArrayTemp(loc, builder, elemental.getType(), shape, extents,
781 adaptor.getTypeparams(), mold);
782 // If the box load is needed, we'd better place it outside
783 // of the loop nest.
784 temp = derefPointersAndAllocatables(loc, builder, temp);
785
786 // Generate a loop nest looping around the fir.elemental shape and clone
787 // fir.elemental region inside the inner loop.
788 hlfir::LoopNest loopNest =
789 hlfir::genLoopNest(loc, builder, extents, !elemental.isOrdered());
790 auto insPt = builder.saveInsertionPoint();
791 builder.setInsertionPointToStart(loopNest.innerLoop.getBody());
792 auto yield = hlfir::inlineElementalOp(loc, builder, elemental,
793 loopNest.oneBasedIndices);
794 hlfir::Entity elementValue(yield.getElementValue());
795 // Skip final AsExpr if any. It would create an element temporary,
796 // which is no needed since the element will be assigned right away in
797 // the array temporary. An hlfir.as_expr may have been added if the
798 // elemental is a "view" over a variable (e.g parentheses or transpose).
799 if (auto asExpr = elementValue.getDefiningOp<hlfir::AsExprOp>()) {
800 if (asExpr->hasOneUse() && !asExpr.isMove()) {
801 elementValue = hlfir::Entity{asExpr.getVar()};
802 rewriter.eraseOp(asExpr);
803 }
804 }
805 rewriter.eraseOp(yield);
806 // Assign the element value to the temp element for this iteration.
807 auto tempElement =
808 hlfir::getElementAt(loc, builder, temp, loopNest.oneBasedIndices);
809 // If the elemental result is a temporary of a derived type,
810 // we can avoid the deep copy implied by the AssignOp and just
811 // do the shallow copy with load/store. This helps avoiding the overhead
812 // of deallocating allocatable components of the temporary (if any)
813 // on each iteration of the elemental operation.
814 auto asExpr = elementValue.getDefiningOp<hlfir::AsExprOp>();
815 auto elemType = hlfir::getFortranElementType(elementValue.getType());
816 if (asExpr && asExpr.isMove() && mlir::isa<fir::RecordType>(elemType) &&
817 hlfir::mayHaveAllocatableComponent(elemType) &&
818 wasCreatedInCurrentBlock(elementValue, builder)) {
819 auto load = builder.create<fir::LoadOp>(loc, asExpr.getVar());
820 builder.create<fir::StoreOp>(loc, load, tempElement);
821 } else {
822 builder.create<hlfir::AssignOp>(loc, elementValue, tempElement,
823 /*realloc=*/false,
824 /*keep_lhs_length_if_realloc=*/false,
825 /*temporary_lhs=*/true);
826
827 // hlfir.yield_element implicitly marks the end-of-life its operand if
828 // it is an expression created in the hlfir.elemental (since it is its
829 // last use and an hlfir.destroy could not be created afterwards)
830 // Now that this node has been removed and the expression has been used in
831 // the assign, insert an hlfir.destroy to mark the expression end-of-life.
832 // If the expression creation allocated a buffer on the heap inside the
833 // loop, this will ensure the buffer properly deallocated.
834 if (elementValue.getType().isa<hlfir::ExprType>() &&
835 wasCreatedInCurrentBlock(elementValue, builder))
836 builder.create<hlfir::DestroyOp>(loc, elementValue);
837 }
838 builder.restoreInsertionPoint(insPt);
839
840 mlir::Value bufferizedExpr =
841 packageBufferizedExpr(loc, builder, temp, cleanup);
842 // Explicitly delete the body of the elemental to get rid
843 // of any users of hlfir.expr values inside the body as early
844 // as possible.
845 rewriter.startOpModification(elemental);
846 rewriter.eraseBlock(elemental.getBody());
847 rewriter.finalizeOpModification(elemental);
848 rewriter.replaceOp(elemental, bufferizedExpr);
849 return mlir::success();
850 }
851};
852struct CharExtremumOpConversion
853 : public mlir::OpConversionPattern<hlfir::CharExtremumOp> {
854 using mlir::OpConversionPattern<hlfir::CharExtremumOp>::OpConversionPattern;
855 explicit CharExtremumOpConversion(mlir::MLIRContext *ctx)
856 : mlir::OpConversionPattern<hlfir::CharExtremumOp>{ctx} {}
857 mlir::LogicalResult
858 matchAndRewrite(hlfir::CharExtremumOp char_extremum, OpAdaptor adaptor,
859 mlir::ConversionPatternRewriter &rewriter) const override {
860 mlir::Location loc = char_extremum->getLoc();
861 auto predicate = char_extremum.getPredicate();
862 bool predIsMin =
863 predicate == hlfir::CharExtremumPredicate::min ? true : false;
864 fir::FirOpBuilder builder(rewriter, char_extremum.getOperation());
865 assert(adaptor.getStrings().size() >= 2 &&
866 "must have at least two strings operands");
867 auto numOperands = adaptor.getStrings().size();
868
869 std::vector<hlfir::Entity> chars;
870 std::vector<
871 std::pair<fir::ExtendedValue, std::optional<hlfir::CleanupFunction>>>
872 pairs;
873 llvm::SmallVector<fir::CharBoxValue> opCBVs;
874 for (size_t i = 0; i < numOperands; ++i) {
875 chars.emplace_back(getBufferizedExprStorage(adaptor.getStrings()[i]));
876 pairs.emplace_back(
877 hlfir::translateToExtendedValue(loc, builder, chars[i]));
878 assert(!pairs[i].second && "expected variables");
879 opCBVs.emplace_back(*pairs[i].first.getCharBox());
880 }
881
882 fir::ExtendedValue res =
883 fir::factory::CharacterExprHelper{builder, loc}.createCharExtremum(
884 predIsMin, opCBVs);
885 mlir::Type addrType = fir::ReferenceType::get(
886 hlfir::getFortranElementType(char_extremum.getResult().getType()));
887 mlir::Value cast = builder.createConvert(loc, addrType, fir::getBase(res));
888 res = fir::substBase(res, cast);
889 hlfir::Entity hlfirTempRes =
890 hlfir::Entity{hlfir::genDeclare(loc, builder, res, ".tmp.char_extremum",
891 fir::FortranVariableFlagsAttr{})
892 .getBase()};
893 mlir::Value bufferizedExpr =
894 packageBufferizedExpr(loc, builder, hlfirTempRes, false);
895 rewriter.replaceOp(char_extremum, bufferizedExpr);
896 return mlir::success();
897 }
898};
899
900class BufferizeHLFIR : public hlfir::impl::BufferizeHLFIRBase<BufferizeHLFIR> {
901public:
902 void runOnOperation() override {
903 // TODO: make this a pass operating on FuncOp. The issue is that
904 // FirOpBuilder helpers may generate new FuncOp because of runtime/llvm
905 // intrinsics calls creation. This may create race conflict if the pass is
906 // scheduled on FuncOp. A solution could be to provide an optional mutex
907 // when building a FirOpBuilder and locking around FuncOp and GlobalOp
908 // creation, but this needs a bit more thinking, so at this point the pass
909 // is scheduled on the moduleOp.
910 auto module = this->getOperation();
911 auto *context = &getContext();
912 mlir::RewritePatternSet patterns(context);
913 patterns.insert<ApplyOpConversion, AsExprOpConversion, AssignOpConversion,
914 AssociateOpConversion, CharExtremumOpConversion,
915 ConcatOpConversion, DestroyOpConversion,
916 ElementalOpConversion, EndAssociateOpConversion,
917 NoReassocOpConversion, SetLengthOpConversion,
918 ShapeOfOpConversion, GetLengthOpConversion>(context);
919 mlir::ConversionTarget target(*context);
920 // Note that YieldElementOp is not marked as an illegal operation.
921 // It must be erased by its parent converter and there is no explicit
922 // conversion pattern to YieldElementOp itself. If any YieldElementOp
923 // survives this pass, the verifier will detect it because it has to be
924 // a child of ElementalOp and ElementalOp's are explicitly illegal.
925 target.addIllegalOp<hlfir::ApplyOp, hlfir::AssociateOp, hlfir::ElementalOp,
926 hlfir::EndAssociateOp, hlfir::SetLengthOp>();
927
928 target.markUnknownOpDynamicallyLegal([](mlir::Operation *op) {
929 return llvm::all_of(
930 op->getResultTypes(),
931 [](mlir::Type ty) { return !ty.isa<hlfir::ExprType>(); }) &&
932 llvm::all_of(op->getOperandTypes(), [](mlir::Type ty) {
933 return !ty.isa<hlfir::ExprType>();
934 });
935 });
936 if (mlir::failed(
937 mlir::applyFullConversion(module, target, std::move(patterns)))) {
938 mlir::emitError(mlir::UnknownLoc::get(context),
939 "failure in HLFIR bufferization pass");
940 signalPassFailure();
941 }
942 }
943};
944} // namespace
945
946std::unique_ptr<mlir::Pass> hlfir::createBufferizeHLFIRPass() {
947 return std::make_unique<BufferizeHLFIR>();
948}
949

source code of flang/lib/Optimizer/HLFIR/Transforms/BufferizeHLFIR.cpp