1	//===-- FIROps.cpp --------------------------------------------------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// Coding style: https://mlir.llvm.org/getting_started/DeveloperGuide/
10	//
11	//===----------------------------------------------------------------------===//
12
13	#include "flang/Optimizer/Dialect/FIROps.h"
14	#include "flang/Optimizer/Dialect/FIRAttr.h"
15	#include "flang/Optimizer/Dialect/FIRDialect.h"
16	#include "flang/Optimizer/Dialect/FIROpsSupport.h"
17	#include "flang/Optimizer/Dialect/FIRType.h"
18	#include "flang/Optimizer/Dialect/Support/FIRContext.h"
19	#include "flang/Optimizer/Dialect/Support/KindMapping.h"
20	#include "flang/Optimizer/Support/Utils.h"
21	#include "mlir/Dialect/CommonFolders.h"
22	#include "mlir/Dialect/Func/IR/FuncOps.h"
23	#include "mlir/Dialect/OpenACC/OpenACC.h"
24	#include "mlir/Dialect/OpenMP/OpenMPDialect.h"
25	#include "mlir/IR/Attributes.h"
26	#include "mlir/IR/BuiltinAttributes.h"
27	#include "mlir/IR/BuiltinOps.h"
28	#include "mlir/IR/Diagnostics.h"
29	#include "mlir/IR/Matchers.h"
30	#include "mlir/IR/OpDefinition.h"
31	#include "mlir/IR/PatternMatch.h"
32	#include "mlir/IR/TypeRange.h"
33	#include "llvm/ADT/STLExtras.h"
34	#include "llvm/ADT/SmallVector.h"
35	#include "llvm/ADT/TypeSwitch.h"
36	#include "llvm/Support/CommandLine.h"
37
38	namespace {
39	#include "flang/Optimizer/Dialect/CanonicalizationPatterns.inc"
40	} // namespace
41
42	static llvm::cl::opt<bool> clUseStrictVolatileVerification(
43	"strict-fir-volatile-verifier", llvm::cl::init(false),
44	llvm::cl::desc(
45	"use stricter verifier for FIR operations with volatile types"));
46
47	bool fir::useStrictVolatileVerification() {
48	return clUseStrictVolatileVerification;
49	}
50
51	static void propagateAttributes(mlir::Operation *fromOp,
52	mlir::Operation *toOp) {
53	if (!fromOp || !toOp)
54	return;
55
56	for (mlir::NamedAttribute attr : fromOp->getAttrs()) {
57	if (attr.getName().getValue().starts_with(
58	Prefix: mlir::acc::OpenACCDialect::getDialectNamespace()))
59	toOp->setAttr(name: attr.getName(), value: attr.getValue());
60	}
61	}
62
63	/// Return true if a sequence type is of some incomplete size or a record type
64	/// is malformed or contains an incomplete sequence type. An incomplete sequence
65	/// type is one with more unknown extents in the type than have been provided
66	/// via `dynamicExtents`. Sequence types with an unknown rank are incomplete by
67	/// definition.
68	static bool verifyInType(mlir::Type inType,
69	llvm::SmallVectorImpl<llvm::StringRef> &visited,
70	unsigned dynamicExtents = 0) {
71	if (auto st = mlir::dyn_cast<fir::SequenceType>(inType)) {
72	auto shape = st.getShape();
73	if (shape.size() == 0)
74	return true;
75	for (std::size_t i = 0, end = shape.size(); i < end; ++i) {
76	if (shape[i] != fir::SequenceType::getUnknownExtent())
77	continue;
78	if (dynamicExtents-- == 0)
79	return true;
80	}
81	} else if (auto rt = mlir::dyn_cast<fir::RecordType>(inType)) {
82	// don't recurse if we're already visiting this one
83	if (llvm::is_contained(visited, rt.getName()))
84	return false;
85	// keep track of record types currently being visited
86	visited.push_back(Elt: rt.getName());
87	for (auto &field : rt.getTypeList())
88	if (verifyInType(field.second, visited))
89	return true;
90	visited.pop_back();
91	}
92	return false;
93	}
94
95	static bool verifyTypeParamCount(mlir::Type inType, unsigned numParams) {
96	auto ty = fir::unwrapSequenceType(inType);
97	if (numParams > 0) {
98	if (auto recTy = mlir::dyn_cast<fir::RecordType>(ty))
99	return numParams != recTy.getNumLenParams();
100	if (auto chrTy = mlir::dyn_cast<fir::CharacterType>(ty))
101	return !(numParams == 1 && chrTy.hasDynamicLen());
102	return true;
103	}
104	if (auto chrTy = mlir::dyn_cast<fir::CharacterType>(ty))
105	return !chrTy.hasConstantLen();
106	return false;
107	}
108
109	/// Parser shared by Alloca and Allocmem
110	///
111	/// operation ::= %res = (`fir.alloca` | `fir.allocmem`) $in_type
112	/// ( `(` $typeparams `)` )? ( `,` $shape )?
113	/// attr-dict-without-keyword
114	template <typename FN>
115	static mlir::ParseResult parseAllocatableOp(FN wrapResultType,
116	mlir::OpAsmParser &parser,
117	mlir::OperationState &result) {
118	mlir::Type intype;
119	if (parser.parseType(result&: intype))
120	return mlir::failure();
121	auto &builder = parser.getBuilder();
122	result.addAttribute(name: "in_type", attr: mlir::TypeAttr::get(type: intype));
123	llvm::SmallVector<mlir::OpAsmParser::UnresolvedOperand> operands;
124	llvm::SmallVector<mlir::Type> typeVec;
125	bool hasOperands = false;
126	std::int32_t typeparamsSize = 0;
127	if (!parser.parseOptionalLParen()) {
128	// parse the LEN params of the derived type. (<params> : <types>)
129	if (parser.parseOperandList(result&: operands, delimiter: mlir::OpAsmParser::Delimiter::None) ||
130	parser.parseColonTypeList(result&: typeVec) || parser.parseRParen())
131	return mlir::failure();
132	typeparamsSize = operands.size();
133	hasOperands = true;
134	}
135	std::int32_t shapeSize = 0;
136	if (!parser.parseOptionalComma()) {
137	// parse size to scale by, vector of n dimensions of type index
138	if (parser.parseOperandList(result&: operands, delimiter: mlir::OpAsmParser::Delimiter::None))
139	return mlir::failure();
140	shapeSize = operands.size() - typeparamsSize;
141	auto idxTy = builder.getIndexType();
142	for (std::int32_t i = typeparamsSize, end = operands.size(); i != end; ++i)
143	typeVec.push_back(Elt: idxTy);
144	hasOperands = true;
145	}
146	if (hasOperands &&
147	parser.resolveOperands(operands, types&: typeVec, loc: parser.getNameLoc(),
148	result&: result.operands))
149	return mlir::failure();
150	mlir::Type restype = wrapResultType(intype);
151	if (!restype) {
152	parser.emitError(loc: parser.getNameLoc(), message: "invalid allocate type: ") << intype;
153	return mlir::failure();
154	}
155	result.addAttribute(name: "operandSegmentSizes", attr: builder.getDenseI32ArrayAttr(
156	values: {typeparamsSize, shapeSize}));
157	if (parser.parseOptionalAttrDict(result&: result.attributes) ||
158	parser.addTypeToList(type: restype, result&: result.types))
159	return mlir::failure();
160	return mlir::success();
161	}
162
163	template <typename OP>
164	static void printAllocatableOp(mlir::OpAsmPrinter &p, OP &op) {
165	p << ' ' << op.getInType();
166	if (!op.getTypeparams().empty()) {
167	p << '(' << op.getTypeparams() << " : " << op.getTypeparams().getTypes()
168	<< ')';
169	}
170	// print the shape of the allocation (if any); all must be index type
171	for (auto sh : op.getShape()) {
172	p << ", ";
173	p.printOperand(sh);
174	}
175	p.printOptionalAttrDict(attrs: op->getAttrs(), elidedAttrs: {"in_type", "operandSegmentSizes"});
176	}
177
178	//===----------------------------------------------------------------------===//
179	// AllocaOp
180	//===----------------------------------------------------------------------===//
181
182	/// Create a legal memory reference as return type
183	static mlir::Type wrapAllocaResultType(mlir::Type intype) {
184	// FIR semantics: memory references to memory references are disallowed
185	if (mlir::isa<fir::ReferenceType>(intype))
186	return {};
187	return fir::ReferenceType::get(intype);
188	}
189
190	mlir::Type fir::AllocaOp::getAllocatedType() {
191	return mlir::cast<fir::ReferenceType>(getType()).getEleTy();
192	}
193
194	mlir::Type fir::AllocaOp::getRefTy(mlir::Type ty) {
195	return fir::ReferenceType::get(ty);
196	}
197
198	void fir::AllocaOp::build(mlir::OpBuilder &builder,
199	mlir::OperationState &result, mlir::Type inType,
200	llvm::StringRef uniqName, mlir::ValueRange typeparams,
201	mlir::ValueRange shape,
202	llvm::ArrayRef<mlir::NamedAttribute> attributes) {
203	auto nameAttr = builder.getStringAttr(uniqName);
204	build(builder, result, wrapAllocaResultType(inType), inType, nameAttr, {},
205	/*pinned=*/false, typeparams, shape);
206	result.addAttributes(attributes);
207	}
208
209	void fir::AllocaOp::build(mlir::OpBuilder &builder,
210	mlir::OperationState &result, mlir::Type inType,
211	llvm::StringRef uniqName, bool pinned,
212	mlir::ValueRange typeparams, mlir::ValueRange shape,
213	llvm::ArrayRef<mlir::NamedAttribute> attributes) {
214	auto nameAttr = builder.getStringAttr(uniqName);
215	build(builder, result, wrapAllocaResultType(inType), inType, nameAttr, {},
216	pinned, typeparams, shape);
217	result.addAttributes(attributes);
218	}
219
220	void fir::AllocaOp::build(mlir::OpBuilder &builder,
221	mlir::OperationState &result, mlir::Type inType,
222	llvm::StringRef uniqName, llvm::StringRef bindcName,
223	mlir::ValueRange typeparams, mlir::ValueRange shape,
224	llvm::ArrayRef<mlir::NamedAttribute> attributes) {
225	auto nameAttr =
226	uniqName.empty() ? mlir::StringAttr{} : builder.getStringAttr(uniqName);
227	auto bindcAttr =
228	bindcName.empty() ? mlir::StringAttr{} : builder.getStringAttr(bindcName);
229	build(builder, result, wrapAllocaResultType(inType), inType, nameAttr,
230	bindcAttr, /*pinned=*/false, typeparams, shape);
231	result.addAttributes(attributes);
232	}
233
234	void fir::AllocaOp::build(mlir::OpBuilder &builder,
235	mlir::OperationState &result, mlir::Type inType,
236	llvm::StringRef uniqName, llvm::StringRef bindcName,
237	bool pinned, mlir::ValueRange typeparams,
238	mlir::ValueRange shape,
239	llvm::ArrayRef<mlir::NamedAttribute> attributes) {
240	auto nameAttr =
241	uniqName.empty() ? mlir::StringAttr{} : builder.getStringAttr(uniqName);
242	auto bindcAttr =
243	bindcName.empty() ? mlir::StringAttr{} : builder.getStringAttr(bindcName);
244	build(builder, result, wrapAllocaResultType(inType), inType, nameAttr,
245	bindcAttr, pinned, typeparams, shape);
246	result.addAttributes(attributes);
247	}
248
249	void fir::AllocaOp::build(mlir::OpBuilder &builder,
250	mlir::OperationState &result, mlir::Type inType,
251	mlir::ValueRange typeparams, mlir::ValueRange shape,
252	llvm::ArrayRef<mlir::NamedAttribute> attributes) {
253	build(builder, result, wrapAllocaResultType(inType), inType, {}, {},
254	/*pinned=*/false, typeparams, shape);
255	result.addAttributes(attributes);
256	}
257
258	void fir::AllocaOp::build(mlir::OpBuilder &builder,
259	mlir::OperationState &result, mlir::Type inType,
260	bool pinned, mlir::ValueRange typeparams,
261	mlir::ValueRange shape,
262	llvm::ArrayRef<mlir::NamedAttribute> attributes) {
263	build(builder, result, wrapAllocaResultType(inType), inType, {}, {}, pinned,
264	typeparams, shape);
265	result.addAttributes(attributes);
266	}
267
268	mlir::ParseResult fir::AllocaOp::parse(mlir::OpAsmParser &parser,
269	mlir::OperationState &result) {
270	return parseAllocatableOp(wrapAllocaResultType, parser, result);
271	}
272
273	void fir::AllocaOp::print(mlir::OpAsmPrinter &p) {
274	printAllocatableOp(p, *this);
275	}
276
277	llvm::LogicalResult fir::AllocaOp::verify() {
278	llvm::SmallVector<llvm::StringRef> visited;
279	if (verifyInType(getInType(), visited, numShapeOperands()))
280	return emitOpError("invalid type for allocation");
281	if (verifyTypeParamCount(getInType(), numLenParams()))
282	return emitOpError("LEN params do not correspond to type");
283	mlir::Type outType = getType();
284	if (!mlir::isa<fir::ReferenceType>(outType))
285	return emitOpError("must be a !fir.ref type");
286	return mlir::success();
287	}
288
289	bool fir::AllocaOp::ownsNestedAlloca(mlir::Operation *op) {
290	return op->hasTrait<mlir::OpTrait::IsIsolatedFromAbove>() ||
291	op->hasTrait<mlir::OpTrait::AutomaticAllocationScope>() ||
292	mlir::isa<mlir::LoopLikeOpInterface>(*op);
293	}
294
295	mlir::Region *fir::AllocaOp::getOwnerRegion() {
296	mlir::Operation *currentOp = getOperation();
297	while (mlir::Operation *parentOp = currentOp->getParentOp()) {
298	// If the operation was not registered, inquiries about its traits will be
299	// incorrect and it is not possible to reason about the operation. This
300	// should not happen in a normal Fortran compilation flow, but be foolproof.
301	if (!parentOp->isRegistered())
302	return nullptr;
303	if (fir::AllocaOp::ownsNestedAlloca(parentOp))
304	return currentOp->getParentRegion();
305	currentOp = parentOp;
306	}
307	return nullptr;
308	}
309
310	//===----------------------------------------------------------------------===//
311	// AllocMemOp
312	//===----------------------------------------------------------------------===//
313
314	/// Create a legal heap reference as return type
315	static mlir::Type wrapAllocMemResultType(mlir::Type intype) {
316	// Fortran semantics: C852 an entity cannot be both ALLOCATABLE and POINTER
317	// 8.5.3 note 1 prohibits ALLOCATABLE procedures as well
318	// FIR semantics: one may not allocate a memory reference value
319	if (mlir::isa<fir::ReferenceType, fir::HeapType, fir::PointerType,
320	mlir::FunctionType>(intype))
321	return {};
322	return fir::HeapType::get(intype);
323	}
324
325	mlir::Type fir::AllocMemOp::getAllocatedType() {
326	return mlir::cast<fir::HeapType>(getType()).getEleTy();
327	}
328
329	mlir::Type fir::AllocMemOp::getRefTy(mlir::Type ty) {
330	return fir::HeapType::get(ty);
331	}
332
333	void fir::AllocMemOp::build(mlir::OpBuilder &builder,
334	mlir::OperationState &result, mlir::Type inType,
335	llvm::StringRef uniqName,
336	mlir::ValueRange typeparams, mlir::ValueRange shape,
337	llvm::ArrayRef<mlir::NamedAttribute> attributes) {
338	auto nameAttr = builder.getStringAttr(uniqName);
339	build(builder, result, wrapAllocMemResultType(inType), inType, nameAttr, {},
340	typeparams, shape);
341	result.addAttributes(attributes);
342	}
343
344	void fir::AllocMemOp::build(mlir::OpBuilder &builder,
345	mlir::OperationState &result, mlir::Type inType,
346	llvm::StringRef uniqName, llvm::StringRef bindcName,
347	mlir::ValueRange typeparams, mlir::ValueRange shape,
348	llvm::ArrayRef<mlir::NamedAttribute> attributes) {
349	auto nameAttr = builder.getStringAttr(uniqName);
350	auto bindcAttr = builder.getStringAttr(bindcName);
351	build(builder, result, wrapAllocMemResultType(inType), inType, nameAttr,
352	bindcAttr, typeparams, shape);
353	result.addAttributes(attributes);
354	}
355
356	void fir::AllocMemOp::build(mlir::OpBuilder &builder,
357	mlir::OperationState &result, mlir::Type inType,
358	mlir::ValueRange typeparams, mlir::ValueRange shape,
359	llvm::ArrayRef<mlir::NamedAttribute> attributes) {
360	build(builder, result, wrapAllocMemResultType(inType), inType, {}, {},
361	typeparams, shape);
362	result.addAttributes(attributes);
363	}
364
365	mlir::ParseResult fir::AllocMemOp::parse(mlir::OpAsmParser &parser,
366	mlir::OperationState &result) {
367	return parseAllocatableOp(wrapAllocMemResultType, parser, result);
368	}
369
370	void fir::AllocMemOp::print(mlir::OpAsmPrinter &p) {
371	printAllocatableOp(p, *this);
372	}
373
374	llvm::LogicalResult fir::AllocMemOp::verify() {
375	llvm::SmallVector<llvm::StringRef> visited;
376	if (verifyInType(getInType(), visited, numShapeOperands()))
377	return emitOpError("invalid type for allocation");
378	if (verifyTypeParamCount(getInType(), numLenParams()))
379	return emitOpError("LEN params do not correspond to type");
380	mlir::Type outType = getType();
381	if (!mlir::dyn_cast<fir::HeapType>(outType))
382	return emitOpError("must be a !fir.heap type");
383	if (fir::isa_unknown_size_box(fir::dyn_cast_ptrEleTy(outType)))
384	return emitOpError("cannot allocate !fir.box of unknown rank or type");
385	return mlir::success();
386	}
387
388	//===----------------------------------------------------------------------===//
389	// ArrayCoorOp
390	//===----------------------------------------------------------------------===//
391
392	// CHARACTERs and derived types with LEN PARAMETERs are dependent types that
393	// require runtime values to fully define the type of an object.
394	static bool validTypeParams(mlir::Type dynTy, mlir::ValueRange typeParams,
395	bool allowParamsForBox = false) {
396	dynTy = fir::unwrapAllRefAndSeqType(dynTy);
397	if (mlir::isa<fir::BaseBoxType>(dynTy)) {
398	// A box value will contain type parameter values itself.
399	if (!allowParamsForBox)
400	return typeParams.size() == 0;
401
402	// A boxed value may have no length parameters, when the lengths
403	// are assumed. If dynamic lengths are used, then proceed
404	// to the verification below.
405	if (typeParams.size() == 0)
406	return true;
407
408	dynTy = fir::getFortranElementType(dynTy);
409	}
410	// Derived type must have all type parameters satisfied.
411	if (auto recTy = mlir::dyn_cast<fir::RecordType>(dynTy))
412	return typeParams.size() == recTy.getNumLenParams();
413	// Characters with non-constant LEN must have a type parameter value.
414	if (auto charTy = mlir::dyn_cast<fir::CharacterType>(dynTy))
415	if (charTy.hasDynamicLen())
416	return typeParams.size() == 1;
417	// Otherwise, any type parameters are invalid.
418	return typeParams.size() == 0;
419	}
420
421	llvm::LogicalResult fir::ArrayCoorOp::verify() {
422	auto eleTy = fir::dyn_cast_ptrOrBoxEleTy(getMemref().getType());
423	auto arrTy = mlir::dyn_cast<fir::SequenceType>(eleTy);
424	if (!arrTy)
425	return emitOpError("must be a reference to an array");
426	auto arrDim = arrTy.getDimension();
427
428	if (auto shapeOp = getShape()) {
429	auto shapeTy = shapeOp.getType();
430	unsigned shapeTyRank = 0;
431	if (auto s = mlir::dyn_cast<fir::ShapeType>(shapeTy)) {
432	shapeTyRank = s.getRank();
433	} else if (auto ss = mlir::dyn_cast<fir::ShapeShiftType>(shapeTy)) {
434	shapeTyRank = ss.getRank();
435	} else {
436	auto s = mlir::cast<fir::ShiftType>(shapeTy);
437	shapeTyRank = s.getRank();
438	// TODO: it looks like PreCGRewrite and CodeGen can support
439	// fir.shift with plain array reference, so we may consider
440	// removing this check.
441	if (!mlir::isa<fir::BaseBoxType>(getMemref().getType()))
442	return emitOpError("shift can only be provided with fir.box memref");
443	}
444	if (arrDim && arrDim != shapeTyRank)
445	return emitOpError("rank of dimension mismatched");
446	// TODO: support slicing with changing the number of dimensions,
447	// e.g. when array_coor represents an element access to array(:,1,:)
448	// slice: the shape is 3D and the number of indices is 2 in this case.
449	if (shapeTyRank != getIndices().size())
450	return emitOpError("number of indices do not match dim rank");
451	}
452
453	if (auto sliceOp = getSlice()) {
454	if (auto sl = mlir::dyn_cast_or_null<fir::SliceOp>(sliceOp.getDefiningOp()))
455	if (!sl.getSubstr().empty())
456	return emitOpError("array_coor cannot take a slice with substring");
457	if (auto sliceTy = mlir::dyn_cast<fir::SliceType>(sliceOp.getType()))
458	if (sliceTy.getRank() != arrDim)
459	return emitOpError("rank of dimension in slice mismatched");
460	}
461	if (!validTypeParams(getMemref().getType(), getTypeparams()))
462	return emitOpError("invalid type parameters");
463
464	return mlir::success();
465	}
466
467	// Pull in fir.embox and fir.rebox into fir.array_coor when possible.
468	struct SimplifyArrayCoorOp : public mlir::OpRewritePattern<fir::ArrayCoorOp> {
469	using mlir::OpRewritePattern<fir::ArrayCoorOp>::OpRewritePattern;
470	llvm::LogicalResult
471	matchAndRewrite(fir::ArrayCoorOp op,
472	mlir::PatternRewriter &rewriter) const override {
473	mlir::Value memref = op.getMemref();
474	if (!mlir::isa<fir::BaseBoxType>(memref.getType()))
475	return mlir::failure();
476
477	mlir::Value boxedMemref, boxedShape, boxedSlice;
478	if (auto emboxOp =
479	mlir::dyn_cast_or_null<fir::EmboxOp>(memref.getDefiningOp())) {
480	boxedMemref = emboxOp.getMemref();
481	boxedShape = emboxOp.getShape();
482	boxedSlice = emboxOp.getSlice();
483	// If any of operands, that are not currently supported for migration
484	// to ArrayCoorOp, is present, don't rewrite.
485	if (!emboxOp.getTypeparams().empty() || emboxOp.getSourceBox() ||
486	emboxOp.getAccessMap())
487	return mlir::failure();
488	} else if (auto reboxOp = mlir::dyn_cast_or_null<fir::ReboxOp>(
489	memref.getDefiningOp())) {
490	boxedMemref = reboxOp.getBox();
491	boxedShape = reboxOp.getShape();
492	// Avoid pulling in rebox that performs reshaping.
493	// There is no way to represent box reshaping with array_coor.
494	if (boxedShape && !mlir::isa<fir::ShiftType>(boxedShape.getType()))
495	return mlir::failure();
496	boxedSlice = reboxOp.getSlice();
497	} else {
498	return mlir::failure();
499	}
500
501	bool boxedShapeIsShift =
502	boxedShape && mlir::isa<fir::ShiftType>(boxedShape.getType());
503	bool boxedShapeIsShape =
504	boxedShape && mlir::isa<fir::ShapeType>(boxedShape.getType());
505	bool boxedShapeIsShapeShift =
506	boxedShape && mlir::isa<fir::ShapeShiftType>(boxedShape.getType());
507
508	// Slices changing the number of dimensions are not supported
509	// for array_coor yet.
510	unsigned origBoxRank;
511	if (mlir::isa<fir::BaseBoxType>(boxedMemref.getType()))
512	origBoxRank = fir::getBoxRank(boxedMemref.getType());
513	else if (auto arrTy = mlir::dyn_cast<fir::SequenceType>(
514	fir::unwrapRefType(boxedMemref.getType())))
515	origBoxRank = arrTy.getDimension();
516	else
517	return mlir::failure();
518
519	if (fir::getBoxRank(memref.getType()) != origBoxRank)
520	return mlir::failure();
521
522	// Slices with substring are not supported by array_coor.
523	if (boxedSlice)
524	if (auto sliceOp =
525	mlir::dyn_cast_or_null<fir::SliceOp>(boxedSlice.getDefiningOp()))
526	if (!sliceOp.getSubstr().empty())
527	return mlir::failure();
528
529	// If embox/rebox and array_coor have conflicting shapes or slices,
530	// do nothing.
531	if (op.getShape() && boxedShape && boxedShape != op.getShape())
532	return mlir::failure();
533	if (op.getSlice() && boxedSlice && boxedSlice != op.getSlice())
534	return mlir::failure();
535
536	std::optional<IndicesVectorTy> shiftedIndices;
537	// The embox/rebox and array_coor either have compatible
538	// shape/slice at this point or shape/slice is null
539	// in one of them but not in the other.
540	// The compatibility means they are equal or both null.
541	if (!op.getShape()) {
542	if (boxedShape) {
543	if (op.getSlice()) {
544	if (!boxedSlice) {
545	if (boxedShapeIsShift) {
546	// %0 = fir.rebox %arg(%shift)
547	// %1 = fir.array_coor %0 [%slice] %idx
548	// Both the slice indices and %idx are 1-based, so the rebox
549	// may be pulled in as:
550	// %1 = fir.array_coor %arg [%slice] %idx
551	boxedShape = nullptr;
552	} else if (boxedShapeIsShape) {
553	// %0 = fir.embox %arg(%shape)
554	// %1 = fir.array_coor %0 [%slice] %idx
555	// Pull in as:
556	// %1 = fir.array_coor %arg(%shape) [%slice] %idx
557	} else if (boxedShapeIsShapeShift) {
558	// %0 = fir.embox %arg(%shapeshift)
559	// %1 = fir.array_coor %0 [%slice] %idx
560	// Pull in as:
561	// %shape = fir.shape <extents from the %shapeshift>
562	// %1 = fir.array_coor %arg(%shape) [%slice] %idx
563	boxedShape = getShapeFromShapeShift(v: boxedShape, rewriter);
564	if (!boxedShape)
565	return mlir::failure();
566	} else {
567	return mlir::failure();
568	}
569	} else {
570	if (boxedShapeIsShift) {
571	// %0 = fir.rebox %arg(%shift) [%slice]
572	// %1 = fir.array_coor %0 [%slice] %idx
573	// This FIR may only be valid if the shape specifies
574	// that all lower bounds are 1s and the slice's start indices
575	// and strides are all 1s.
576	// We could pull in the rebox as:
577	// %1 = fir.array_coor %arg [%slice] %idx
578	// Do not do anything for the time being.
579	return mlir::failure();
580	} else if (boxedShapeIsShape) {
581	// %0 = fir.embox %arg(%shape) [%slice]
582	// %1 = fir.array_coor %0 [%slice] %idx
583	// This FIR may only be valid if the slice's start indices
584	// and strides are all 1s.
585	// We could pull in the embox as:
586	// %1 = fir.array_coor %arg(%shape) [%slice] %idx
587	return mlir::failure();
588	} else if (boxedShapeIsShapeShift) {
589	// %0 = fir.embox %arg(%shapeshift) [%slice]
590	// %1 = fir.array_coor %0 [%slice] %idx
591	// This FIR may only be valid if the shape specifies
592	// that all lower bounds are 1s and the slice's start indices
593	// and strides are all 1s.
594	// We could pull in the embox as:
595	// %shape = fir.shape <extents from the %shapeshift>
596	// %1 = fir.array_coor %arg(%shape) [%slice] %idx
597	return mlir::failure();
598	} else {
599	return mlir::failure();
600	}
601	}
602	} else { // !op.getSlice()
603	if (!boxedSlice) {
604	if (boxedShapeIsShift) {
605	// %0 = fir.rebox %arg(%shift)
606	// %1 = fir.array_coor %0 %idx
607	// Pull in as:
608	// %1 = fir.array_coor %arg %idx
609	boxedShape = nullptr;
610	} else if (boxedShapeIsShape) {
611	// %0 = fir.embox %arg(%shape)
612	// %1 = fir.array_coor %0 %idx
613	// Pull in as:
614	// %1 = fir.array_coor %arg(%shape) %idx
615	} else if (boxedShapeIsShapeShift) {
616	// %0 = fir.embox %arg(%shapeshift)
617	// %1 = fir.array_coor %0 %idx
618	// Pull in as:
619	// %shape = fir.shape <extents from the %shapeshift>
620	// %1 = fir.array_coor %arg(%shape) %idx
621	boxedShape = getShapeFromShapeShift(v: boxedShape, rewriter);
622	if (!boxedShape)
623	return mlir::failure();
624	} else {
625	return mlir::failure();
626	}
627	} else {
628	if (boxedShapeIsShift) {
629	// %0 = fir.embox %arg(%shift) [%slice]
630	// %1 = fir.array_coor %0 %idx
631	// Pull in as:
632	// %tmp = arith.addi %idx, %shift.origin
633	// %idx_shifted = arith.subi %tmp, 1
634	// %1 = fir.array_coor %arg(%shift) %[slice] %idx_shifted
635	shiftedIndices =
636	getShiftedIndices(v: boxedShape, indices: op.getIndices(), rewriter);
637	if (!shiftedIndices)
638	return mlir::failure();
639	} else if (boxedShapeIsShape) {
640	// %0 = fir.embox %arg(%shape) [%slice]
641	// %1 = fir.array_coor %0 %idx
642	// Pull in as:
643	// %1 = fir.array_coor %arg(%shape) %[slice] %idx
644	} else if (boxedShapeIsShapeShift) {
645	// %0 = fir.embox %arg(%shapeshift) [%slice]
646	// %1 = fir.array_coor %0 %idx
647	// Pull in as:
648	// %tmp = arith.addi %idx, %shapeshift.lb
649	// %idx_shifted = arith.subi %tmp, 1
650	// %1 = fir.array_coor %arg(%shapeshift) %[slice] %idx_shifted
651	shiftedIndices =
652	getShiftedIndices(v: boxedShape, indices: op.getIndices(), rewriter);
653	if (!shiftedIndices)
654	return mlir::failure();
655	} else {
656	return mlir::failure();
657	}
658	}
659	}
660	} else { // !boxedShape
661	if (op.getSlice()) {
662	if (!boxedSlice) {
663	// %0 = fir.rebox %arg
664	// %1 = fir.array_coor %0 [%slice] %idx
665	// Pull in as:
666	// %1 = fir.array_coor %arg [%slice] %idx
667	} else {
668	// %0 = fir.rebox %arg [%slice]
669	// %1 = fir.array_coor %0 [%slice] %idx
670	// This is a valid FIR iff the slice's lower bounds
671	// and strides are all 1s.
672	// Pull in as:
673	// %1 = fir.array_coor %arg [%slice] %idx
674	}
675	} else { // !op.getSlice()
676	if (!boxedSlice) {
677	// %0 = fir.rebox %arg
678	// %1 = fir.array_coor %0 %idx
679	// Pull in as:
680	// %1 = fir.array_coor %arg %idx
681	} else {
682	// %0 = fir.rebox %arg [%slice]
683	// %1 = fir.array_coor %0 %idx
684	// Pull in as:
685	// %1 = fir.array_coor %arg [%slice] %idx
686	}
687	}
688	}
689	} else { // op.getShape()
690	if (boxedShape) {
691	// Check if pulling in non-default shape is correct.
692	if (op.getSlice()) {
693	if (!boxedSlice) {
694	// %0 = fir.embox %arg(%shape)
695	// %1 = fir.array_coor %0(%shape) [%slice] %idx
696	// Pull in as:
697	// %1 = fir.array_coor %arg(%shape) [%slice] %idx
698	} else {
699	// %0 = fir.embox %arg(%shape) [%slice]
700	// %1 = fir.array_coor %0(%shape) [%slice] %idx
701	// Pull in as:
702	// %1 = fir.array_coor %arg(%shape) [%slice] %idx
703	}
704	} else { // !op.getSlice()
705	if (!boxedSlice) {
706	// %0 = fir.embox %arg(%shape)
707	// %1 = fir.array_coor %0(%shape) %idx
708	// Pull in as:
709	// %1 = fir.array_coor %arg(%shape) %idx
710	} else {
711	// %0 = fir.embox %arg(%shape) [%slice]
712	// %1 = fir.array_coor %0(%shape) %idx
713	// Pull in as:
714	// %1 = fir.array_coor %arg(%shape) [%slice] %idx
715	}
716	}
717	} else { // !boxedShape
718	if (op.getSlice()) {
719	if (!boxedSlice) {
720	// %0 = fir.rebox %arg
721	// %1 = fir.array_coor %0(%shape) [%slice] %idx
722	// Pull in as:
723	// %1 = fir.array_coor %arg(%shape) [%slice] %idx
724	} else {
725	// %0 = fir.rebox %arg [%slice]
726	// %1 = fir.array_coor %0(%shape) [%slice] %idx
727	return mlir::failure();
728	}
729	} else { // !op.getSlice()
730	if (!boxedSlice) {
731	// %0 = fir.rebox %arg
732	// %1 = fir.array_coor %0(%shape) %idx
733	// Pull in as:
734	// %1 = fir.array_coor %arg(%shape) %idx
735	} else {
736	// %0 = fir.rebox %arg [%slice]
737	// %1 = fir.array_coor %0(%shape) %idx
738	// Cannot pull in without adjusting the slice indices.
739	return mlir::failure();
740	}
741	}
742	}
743	}
744
745	// TODO: temporarily avoid producing array_coor with the shape shift
746	// and plain array reference (it seems to be a limitation of
747	// ArrayCoorOp verifier).
748	if (!mlir::isa<fir::BaseBoxType>(boxedMemref.getType())) {
749	if (boxedShape) {
750	if (mlir::isa<fir::ShiftType>(boxedShape.getType()))
751	return mlir::failure();
752	} else if (op.getShape() &&
753	mlir::isa<fir::ShiftType>(op.getShape().getType())) {
754	return mlir::failure();
755	}
756	}
757
758	rewriter.modifyOpInPlace(op, [&]() {
759	op.getMemrefMutable().assign(boxedMemref);
760	if (boxedShape)
761	op.getShapeMutable().assign(boxedShape);
762	if (boxedSlice)
763	op.getSliceMutable().assign(boxedSlice);
764	if (shiftedIndices)
765	op.getIndicesMutable().assign(*shiftedIndices);
766	});
767	return mlir::success();
768	}
769
770	private:
771	using IndicesVectorTy = std::vector<mlir::Value>;
772
773	// If v is a shape_shift operation:
774	// fir.shape_shift %l1, %e1, %l2, %e2, ...
775	// create:
776	// fir.shape %e1, %e2, ...
777	static mlir::Value getShapeFromShapeShift(mlir::Value v,
778	mlir::PatternRewriter &rewriter) {
779	auto shapeShiftOp =
780	mlir::dyn_cast_or_null<fir::ShapeShiftOp>(v.getDefiningOp());
781	if (!shapeShiftOp)
782	return nullptr;
783	mlir::OpBuilder::InsertionGuard guard(rewriter);
784	rewriter.setInsertionPoint(shapeShiftOp);
785	return rewriter.create<fir::ShapeOp>(shapeShiftOp.getLoc(),
786	shapeShiftOp.getExtents());
787	}
788
789	static std::optional<IndicesVectorTy>
790	getShiftedIndices(mlir::Value v, mlir::ValueRange indices,
791	mlir::PatternRewriter &rewriter) {
792	auto insertAdjustments = [&](mlir::Operation *op, mlir::ValueRange lbs) {
793	// Compute the shifted indices using the extended type.
794	// Note that this can probably result in less efficient
795	// MLIR and further LLVM IR due to the extra conversions.
796	mlir::OpBuilder::InsertPoint savedIP = rewriter.saveInsertionPoint();
797	rewriter.setInsertionPoint(op);
798	mlir::Location loc = op->getLoc();
799	mlir::Type idxTy = rewriter.getIndexType();
800	mlir::Value one = rewriter.create<mlir::arith::ConstantOp>(
801	loc, idxTy, rewriter.getIndexAttr(1));
802	rewriter.restoreInsertionPoint(ip: savedIP);
803	auto nsw = mlir::arith::IntegerOverflowFlags::nsw;
804
805	IndicesVectorTy shiftedIndices;
806	for (auto [lb, idx] : llvm::zip(t&: lbs, u&: indices)) {
807	mlir::Value extLb = rewriter.create<fir::ConvertOp>(loc, idxTy, lb);
808	mlir::Value extIdx = rewriter.create<fir::ConvertOp>(loc, idxTy, idx);
809	mlir::Value add =
810	rewriter.create<mlir::arith::AddIOp>(loc, extIdx, extLb, nsw);
811	mlir::Value sub =
812	rewriter.create<mlir::arith::SubIOp>(loc, add, one, nsw);
813	shiftedIndices.push_back(x: sub);
814	}
815
816	return shiftedIndices;
817	};
818
819	if (auto shiftOp =
820	mlir::dyn_cast_or_null<fir::ShiftOp>(v.getDefiningOp())) {
821	return insertAdjustments(shiftOp.getOperation(), shiftOp.getOrigins());
822	} else if (auto shapeShiftOp = mlir::dyn_cast_or_null<fir::ShapeShiftOp>(
823	v.getDefiningOp())) {
824	return insertAdjustments(shapeShiftOp.getOperation(),
825	shapeShiftOp.getOrigins());
826	}
827
828	return std::nullopt;
829	}
830	};
831
832	void fir::ArrayCoorOp::getCanonicalizationPatterns(
833	mlir::RewritePatternSet &patterns, mlir::MLIRContext *context) {
834	// TODO: !fir.shape<1> operand may be removed from array_coor always.
835	patterns.add<SimplifyArrayCoorOp>(context);
836	}
837
838	//===----------------------------------------------------------------------===//
839	// ArrayLoadOp
840	//===----------------------------------------------------------------------===//
841
842	static mlir::Type adjustedElementType(mlir::Type t) {
843	if (auto ty = mlir::dyn_cast<fir::ReferenceType>(t)) {
844	auto eleTy = ty.getEleTy();
845	if (fir::isa_char(eleTy))
846	return eleTy;
847	if (fir::isa_derived(eleTy))
848	return eleTy;
849	if (mlir::isa<fir::SequenceType>(eleTy))
850	return eleTy;
851	}
852	return t;
853	}
854
855	std::vector<mlir::Value> fir::ArrayLoadOp::getExtents() {
856	if (auto sh = getShape())
857	if (auto *op = sh.getDefiningOp()) {
858	if (auto shOp = mlir::dyn_cast<fir::ShapeOp>(op)) {
859	auto extents = shOp.getExtents();
860	return {extents.begin(), extents.end()};
861	}
862	return mlir::cast<fir::ShapeShiftOp>(op).getExtents();
863	}
864	return {};
865	}
866
867	void fir::ArrayLoadOp::getEffects(
868	llvm::SmallVectorImpl<
869	mlir::SideEffects::EffectInstance<mlir::MemoryEffects::Effect>>
870	&effects) {
871	effects.emplace_back(mlir::MemoryEffects::Read::get(), &getMemrefMutable(),
872	mlir::SideEffects::DefaultResource::get());
873	addVolatileMemoryEffects({getMemref().getType()}, effects);
874	}
875
876	llvm::LogicalResult fir::ArrayLoadOp::verify() {
877	auto eleTy = fir::dyn_cast_ptrOrBoxEleTy(getMemref().getType());
878	auto arrTy = mlir::dyn_cast<fir::SequenceType>(eleTy);
879	if (!arrTy)
880	return emitOpError("must be a reference to an array");
881	auto arrDim = arrTy.getDimension();
882
883	if (auto shapeOp = getShape()) {
884	auto shapeTy = shapeOp.getType();
885	unsigned shapeTyRank = 0u;
886	if (auto s = mlir::dyn_cast<fir::ShapeType>(shapeTy)) {
887	shapeTyRank = s.getRank();
888	} else if (auto ss = mlir::dyn_cast<fir::ShapeShiftType>(shapeTy)) {
889	shapeTyRank = ss.getRank();
890	} else {
891	auto s = mlir::cast<fir::ShiftType>(shapeTy);
892	shapeTyRank = s.getRank();
893	if (!mlir::isa<fir::BaseBoxType>(getMemref().getType()))
894	return emitOpError("shift can only be provided with fir.box memref");
895	}
896	if (arrDim && arrDim != shapeTyRank)
897	return emitOpError("rank of dimension mismatched");
898	}
899
900	if (auto sliceOp = getSlice()) {
901	if (auto sl = mlir::dyn_cast_or_null<fir::SliceOp>(sliceOp.getDefiningOp()))
902	if (!sl.getSubstr().empty())
903	return emitOpError("array_load cannot take a slice with substring");
904	if (auto sliceTy = mlir::dyn_cast<fir::SliceType>(sliceOp.getType()))
905	if (sliceTy.getRank() != arrDim)
906	return emitOpError("rank of dimension in slice mismatched");
907	}
908
909	if (!validTypeParams(getMemref().getType(), getTypeparams()))
910	return emitOpError("invalid type parameters");
911
912	return mlir::success();
913	}
914
915	//===----------------------------------------------------------------------===//
916	// ArrayMergeStoreOp
917	//===----------------------------------------------------------------------===//
918
919	llvm::LogicalResult fir::ArrayMergeStoreOp::verify() {
920	if (!mlir::isa<fir::ArrayLoadOp>(getOriginal().getDefiningOp()))
921	return emitOpError("operand #0 must be result of a fir.array_load op");
922	if (auto sl = getSlice()) {
923	if (auto sliceOp =
924	mlir::dyn_cast_or_null<fir::SliceOp>(sl.getDefiningOp())) {
925	if (!sliceOp.getSubstr().empty())
926	return emitOpError(
927	"array_merge_store cannot take a slice with substring");
928	if (!sliceOp.getFields().empty()) {
929	// This is an intra-object merge, where the slice is projecting the
930	// subfields that are to be overwritten by the merge operation.
931	auto eleTy = fir::dyn_cast_ptrOrBoxEleTy(getMemref().getType());
932	if (auto seqTy = mlir::dyn_cast<fir::SequenceType>(eleTy)) {
933	auto projTy =
934	fir::applyPathToType(seqTy.getEleTy(), sliceOp.getFields());
935	if (fir::unwrapSequenceType(getOriginal().getType()) != projTy)
936	return emitOpError(
937	"type of origin does not match sliced memref type");
938	if (fir::unwrapSequenceType(getSequence().getType()) != projTy)
939	return emitOpError(
940	"type of sequence does not match sliced memref type");
941	return mlir::success();
942	}
943	return emitOpError("referenced type is not an array");
944	}
945	}
946	return mlir::success();
947	}
948	auto eleTy = fir::dyn_cast_ptrOrBoxEleTy(getMemref().getType());
949	if (getOriginal().getType() != eleTy)
950	return emitOpError("type of origin does not match memref element type");
951	if (getSequence().getType() != eleTy)
952	return emitOpError("type of sequence does not match memref element type");
953	if (!validTypeParams(getMemref().getType(), getTypeparams()))
954	return emitOpError("invalid type parameters");
955	return mlir::success();
956	}
957
958	void fir::ArrayMergeStoreOp::getEffects(
959	llvm::SmallVectorImpl<
960	mlir::SideEffects::EffectInstance<mlir::MemoryEffects::Effect>>
961	&effects) {
962	effects.emplace_back(mlir::MemoryEffects::Write::get(), &getMemrefMutable(),
963	mlir::SideEffects::DefaultResource::get());
964	addVolatileMemoryEffects({getMemref().getType()}, effects);
965	}
966
967	//===----------------------------------------------------------------------===//
968	// ArrayFetchOp
969	//===----------------------------------------------------------------------===//
970
971	// Template function used for both array_fetch and array_update verification.
972	template <typename A>
973	mlir::Type validArraySubobject(A op) {
974	auto ty = op.getSequence().getType();
975	return fir::applyPathToType(ty, op.getIndices());
976	}
977
978	llvm::LogicalResult fir::ArrayFetchOp::verify() {
979	auto arrTy = mlir::cast<fir::SequenceType>(getSequence().getType());
980	auto indSize = getIndices().size();
981	if (indSize < arrTy.getDimension())
982	return emitOpError("number of indices != dimension of array");
983	if (indSize == arrTy.getDimension() &&
984	::adjustedElementType(getElement().getType()) != arrTy.getEleTy())
985	return emitOpError("return type does not match array");
986	auto ty = validArraySubobject(*this);
987	if (!ty || ty != ::adjustedElementType(getType()))
988	return emitOpError("return type and/or indices do not type check");
989	if (!mlir::isa<fir::ArrayLoadOp>(getSequence().getDefiningOp()))
990	return emitOpError("argument #0 must be result of fir.array_load");
991	if (!validTypeParams(arrTy, getTypeparams()))
992	return emitOpError("invalid type parameters");
993	return mlir::success();
994	}
995
996	//===----------------------------------------------------------------------===//
997	// ArrayAccessOp
998	//===----------------------------------------------------------------------===//
999
1000	llvm::LogicalResult fir::ArrayAccessOp::verify() {
1001	auto arrTy = mlir::cast<fir::SequenceType>(getSequence().getType());
1002	std::size_t indSize = getIndices().size();
1003	if (indSize < arrTy.getDimension())
1004	return emitOpError("number of indices != dimension of array");
1005	if (indSize == arrTy.getDimension() &&
1006	getElement().getType() != fir::ReferenceType::get(arrTy.getEleTy()))
1007	return emitOpError("return type does not match array");
1008	mlir::Type ty = validArraySubobject(*this);
1009	if (!ty || fir::ReferenceType::get(ty) != getType())
1010	return emitOpError("return type and/or indices do not type check");
1011	if (!validTypeParams(arrTy, getTypeparams()))
1012	return emitOpError("invalid type parameters");
1013	return mlir::success();
1014	}
1015
1016	//===----------------------------------------------------------------------===//
1017	// ArrayUpdateOp
1018	//===----------------------------------------------------------------------===//
1019
1020	llvm::LogicalResult fir::ArrayUpdateOp::verify() {
1021	if (fir::isa_ref_type(getMerge().getType()))
1022	return emitOpError("does not support reference type for merge");
1023	auto arrTy = mlir::cast<fir::SequenceType>(getSequence().getType());
1024	auto indSize = getIndices().size();
1025	if (indSize < arrTy.getDimension())
1026	return emitOpError("number of indices != dimension of array");
1027	if (indSize == arrTy.getDimension() &&
1028	::adjustedElementType(getMerge().getType()) != arrTy.getEleTy())
1029	return emitOpError("merged value does not have element type");
1030	auto ty = validArraySubobject(*this);
1031	if (!ty || ty != ::adjustedElementType(getMerge().getType()))
1032	return emitOpError("merged value and/or indices do not type check");
1033	if (!validTypeParams(arrTy, getTypeparams()))
1034	return emitOpError("invalid type parameters");
1035	return mlir::success();
1036	}
1037
1038	//===----------------------------------------------------------------------===//
1039	// ArrayModifyOp
1040	//===----------------------------------------------------------------------===//
1041
1042	llvm::LogicalResult fir::ArrayModifyOp::verify() {
1043	auto arrTy = mlir::cast<fir::SequenceType>(getSequence().getType());
1044	auto indSize = getIndices().size();
1045	if (indSize < arrTy.getDimension())
1046	return emitOpError("number of indices must match array dimension");
1047	return mlir::success();
1048	}
1049
1050	//===----------------------------------------------------------------------===//
1051	// BoxAddrOp
1052	//===----------------------------------------------------------------------===//
1053
1054	void fir::BoxAddrOp::build(mlir::OpBuilder &builder,
1055	mlir::OperationState &result, mlir::Value val) {
1056	mlir::Type type =
1057	llvm::TypeSwitch<mlir::Type, mlir::Type>(val.getType())
1058	.Case<fir::BaseBoxType>([&](fir::BaseBoxType ty) -> mlir::Type {
1059	mlir::Type eleTy = ty.getEleTy();
1060	if (fir::isa_ref_type(eleTy))
1061	return eleTy;
1062	return fir::ReferenceType::get(eleTy);
1063	})
1064	.Case<fir::BoxCharType>([&](fir::BoxCharType ty) -> mlir::Type {
1065	return fir::ReferenceType::get(ty.getEleTy());
1066	})
1067	.Case<fir::BoxProcType>(
1068	[&](fir::BoxProcType ty) { return ty.getEleTy(); })
1069	.Default([&](const auto &) { return mlir::Type{}; });
1070	assert(type && "bad val type");
1071	build(builder, result, type, val);
1072	}
1073
1074	mlir::OpFoldResult fir::BoxAddrOp::fold(FoldAdaptor adaptor) {
1075	if (auto *v = getVal().getDefiningOp()) {
1076	if (auto box = mlir::dyn_cast<fir::EmboxOp>(v)) {
1077	// Fold only if not sliced
1078	if (!box.getSlice() && box.getMemref().getType() == getType()) {
1079	propagateAttributes(getOperation(), box.getMemref().getDefiningOp());
1080	return box.getMemref();
1081	}
1082	}
1083	if (auto box = mlir::dyn_cast<fir::EmboxCharOp>(v))
1084	if (box.getMemref().getType() == getType())
1085	return box.getMemref();
1086	}
1087	return {};
1088	}
1089
1090	//===----------------------------------------------------------------------===//
1091	// BoxCharLenOp
1092	//===----------------------------------------------------------------------===//
1093
1094	mlir::OpFoldResult fir::BoxCharLenOp::fold(FoldAdaptor adaptor) {
1095	if (auto v = getVal().getDefiningOp()) {
1096	if (auto box = mlir::dyn_cast<fir::EmboxCharOp>(v))
1097	return box.getLen();
1098	}
1099	return {};
1100	}
1101
1102	//===----------------------------------------------------------------------===//
1103	// BoxDimsOp
1104	//===----------------------------------------------------------------------===//
1105
1106	/// Get the result types packed in a tuple tuple
1107	mlir::Type fir::BoxDimsOp::getTupleType() {
1108	// note: triple, but 4 is nearest power of 2
1109	llvm::SmallVector<mlir::Type> triple{
1110	getResult(0).getType(), getResult(1).getType(), getResult(2).getType()};
1111	return mlir::TupleType::get(getContext(), triple);
1112	}
1113
1114	//===----------------------------------------------------------------------===//
1115	// BoxRankOp
1116	//===----------------------------------------------------------------------===//
1117
1118	void fir::BoxRankOp::getEffects(
1119	llvm::SmallVectorImpl<
1120	mlir::SideEffects::EffectInstance<mlir::MemoryEffects::Effect>>
1121	&effects) {
1122	mlir::OpOperand &inputBox = getBoxMutable();
1123	if (fir::isBoxAddress(inputBox.get().getType()))
1124	effects.emplace_back(mlir::MemoryEffects::Read::get(), &inputBox,
1125	mlir::SideEffects::DefaultResource::get());
1126	}
1127
1128	//===----------------------------------------------------------------------===//
1129	// CallOp
1130	//===----------------------------------------------------------------------===//
1131
1132	mlir::FunctionType fir::CallOp::getFunctionType() {
1133	return mlir::FunctionType::get(getContext(), getOperandTypes(),
1134	getResultTypes());
1135	}
1136
1137	void fir::CallOp::print(mlir::OpAsmPrinter &p) {
1138	bool isDirect = getCallee().has_value();
1139	p << ' ';
1140	if (isDirect)
1141	p << *getCallee();
1142	else
1143	p << getOperand(0);
1144	p << '(' << (*this)->getOperands().drop_front(isDirect ? 0 : 1) << ')';
1145
1146	// Print `proc_attrs<...>`, if present.
1147	fir::FortranProcedureFlagsEnumAttr procAttrs = getProcedureAttrsAttr();
1148	if (procAttrs &&
1149	procAttrs.getValue() != fir::FortranProcedureFlagsEnum::none) {
1150	p << ' ' << fir::FortranProcedureFlagsEnumAttr::getMnemonic();
1151	p.printStrippedAttrOrType(procAttrs);
1152	}
1153
1154	// Print 'fastmath<...>' (if it has non-default value) before
1155	// any other attributes.
1156	mlir::arith::FastMathFlagsAttr fmfAttr = getFastmathAttr();
1157	if (fmfAttr.getValue() != mlir::arith::FastMathFlags::none) {
1158	p << ' ' << mlir::arith::FastMathFlagsAttr::getMnemonic();
1159	p.printStrippedAttrOrType(fmfAttr);
1160	}
1161
1162	p.printOptionalAttrDict((*this)->getAttrs(),
1163	{fir::CallOp::getCalleeAttrNameStr(),
1164	getFastmathAttrName(), getProcedureAttrsAttrName(),
1165	getArgAttrsAttrName(), getResAttrsAttrName()});
1166	p << " : ";
1167	mlir::call_interface_impl::printFunctionSignature(
1168	p, getArgs().drop_front(isDirect ? 0 : 1).getTypes(), getArgAttrsAttr(),
1169	/*isVariadic=*/false, getResultTypes(), getResAttrsAttr());
1170	}
1171
1172	mlir::ParseResult fir::CallOp::parse(mlir::OpAsmParser &parser,
1173	mlir::OperationState &result) {
1174	llvm::SmallVector<mlir::OpAsmParser::UnresolvedOperand> operands;
1175	if (parser.parseOperandList(operands))
1176	return mlir::failure();
1177
1178	mlir::NamedAttrList attrs;
1179	mlir::SymbolRefAttr funcAttr;
1180	bool isDirect = operands.empty();
1181	if (isDirect)
1182	if (parser.parseAttribute(funcAttr, fir::CallOp::getCalleeAttrNameStr(),
1183	attrs))
1184	return mlir::failure();
1185
1186	if (parser.parseOperandList(operands, mlir::OpAsmParser::Delimiter::Paren))
1187	return mlir::failure();
1188
1189	// Parse `proc_attrs<...>`, if present.
1190	fir::FortranProcedureFlagsEnumAttr procAttr;
1191	if (mlir::succeeded(parser.parseOptionalKeyword(
1192	fir::FortranProcedureFlagsEnumAttr::getMnemonic())))
1193	if (parser.parseCustomAttributeWithFallback(
1194	procAttr, mlir::Type{}, getProcedureAttrsAttrName(result.name),
1195	attrs))
1196	return mlir::failure();
1197
1198	// Parse 'fastmath<...>', if present.
1199	mlir::arith::FastMathFlagsAttr fmfAttr;
1200	llvm::StringRef fmfAttrName = getFastmathAttrName(result.name);
1201	if (mlir::succeeded(parser.parseOptionalKeyword(fmfAttrName)))
1202	if (parser.parseCustomAttributeWithFallback(fmfAttr, mlir::Type{},
1203	fmfAttrName, attrs))
1204	return mlir::failure();
1205
1206	if (parser.parseOptionalAttrDict(attrs) || parser.parseColon())
1207	return mlir::failure();
1208	llvm::SmallVector<mlir::Type> argTypes;
1209	llvm::SmallVector<mlir::Type> resTypes;
1210	llvm::SmallVector<mlir::DictionaryAttr> argAttrs;
1211	llvm::SmallVector<mlir::DictionaryAttr> resultAttrs;
1212	if (mlir::call_interface_impl::parseFunctionSignature(
1213	parser, argTypes, argAttrs, resTypes, resultAttrs))
1214	return parser.emitError(parser.getNameLoc(), "expected function type");
1215	mlir::FunctionType funcType =
1216	mlir::FunctionType::get(parser.getContext(), argTypes, resTypes);
1217	if (isDirect) {
1218	if (parser.resolveOperands(operands, funcType.getInputs(),
1219	parser.getNameLoc(), result.operands))
1220	return mlir::failure();
1221	} else {
1222	auto funcArgs =
1223	llvm::ArrayRef<mlir::OpAsmParser::UnresolvedOperand>(operands)
1224	.drop_front();
1225	if (parser.resolveOperand(operands[0], funcType, result.operands) ||
1226	parser.resolveOperands(funcArgs, funcType.getInputs(),
1227	parser.getNameLoc(), result.operands))
1228	return mlir::failure();
1229	}
1230	result.attributes = attrs;
1231	mlir::call_interface_impl::addArgAndResultAttrs(
1232	parser.getBuilder(), result, argAttrs, resultAttrs,
1233	getArgAttrsAttrName(result.name), getResAttrsAttrName(result.name));
1234	result.addTypes(funcType.getResults());
1235	return mlir::success();
1236	}
1237
1238	void fir::CallOp::build(mlir::OpBuilder &builder, mlir::OperationState &result,
1239	mlir::func::FuncOp callee, mlir::ValueRange operands) {
1240	result.addOperands(operands);
1241	result.addAttribute(getCalleeAttrNameStr(), mlir::SymbolRefAttr::get(callee));
1242	result.addTypes(callee.getFunctionType().getResults());
1243	}
1244
1245	void fir::CallOp::build(mlir::OpBuilder &builder, mlir::OperationState &result,
1246	mlir::SymbolRefAttr callee,
1247	llvm::ArrayRef<mlir::Type> results,
1248	mlir::ValueRange operands) {
1249	result.addOperands(operands);
1250	if (callee)
1251	result.addAttribute(getCalleeAttrNameStr(), callee);
1252	result.addTypes(results);
1253	}
1254
1255	//===----------------------------------------------------------------------===//
1256	// CharConvertOp
1257	//===----------------------------------------------------------------------===//
1258
1259	llvm::LogicalResult fir::CharConvertOp::verify() {
1260	auto unwrap = [&](mlir::Type t) {
1261	t = fir::unwrapSequenceType(fir::dyn_cast_ptrEleTy(t));
1262	return mlir::dyn_cast<fir::CharacterType>(t);
1263	};
1264	auto inTy = unwrap(getFrom().getType());
1265	auto outTy = unwrap(getTo().getType());
1266	if (!(inTy && outTy))
1267	return emitOpError("not a reference to a character");
1268	if (inTy.getFKind() == outTy.getFKind())
1269	return emitOpError("buffers must have different KIND values");
1270	return mlir::success();
1271	}
1272
1273	//===----------------------------------------------------------------------===//
1274	// CmpOp
1275	//===----------------------------------------------------------------------===//
1276
1277	template <typename OPTY>
1278	static void printCmpOp(mlir::OpAsmPrinter &p, OPTY op) {
1279	p << ' ';
1280	auto predSym = mlir::arith::symbolizeCmpFPredicate(
1281	op->template getAttrOfType<mlir::IntegerAttr>(
1282	OPTY::getPredicateAttrName())
1283	.getInt());
1284	assert(predSym.has_value() && "invalid symbol value for predicate");
1285	p << '"' << mlir::arith::stringifyCmpFPredicate(predSym.value()) << '"'
1286	<< ", ";
1287	p.printOperand(op.getLhs());
1288	p << ", ";
1289	p.printOperand(op.getRhs());
1290	p.printOptionalAttrDict(attrs: op->getAttrs(),
1291	/*elidedAttrs=*/{OPTY::getPredicateAttrName()});
1292	p << " : " << op.getLhs().getType();
1293	}
1294
1295	template <typename OPTY>
1296	static mlir::ParseResult parseCmpOp(mlir::OpAsmParser &parser,
1297	mlir::OperationState &result) {
1298	llvm::SmallVector<mlir::OpAsmParser::UnresolvedOperand> ops;
1299	mlir::NamedAttrList attrs;
1300	mlir::Attribute predicateNameAttr;
1301	mlir::Type type;
1302	if (parser.parseAttribute(predicateNameAttr, OPTY::getPredicateAttrName(),
1303	attrs) ||
1304	parser.parseComma() || parser.parseOperandList(result&: ops, requiredOperandCount: 2) ||
1305	parser.parseOptionalAttrDict(result&: attrs) || parser.parseColonType(result&: type) ||
1306	parser.resolveOperands(operands&: ops, type, result&: result.operands))
1307	return mlir::failure();
1308
1309	if (!mlir::isa<mlir::StringAttr>(Val: predicateNameAttr))
1310	return parser.emitError(loc: parser.getNameLoc(),
1311	message: "expected string comparison predicate attribute");
1312
1313	// Rewrite string attribute to an enum value.
1314	llvm::StringRef predicateName =
1315	mlir::cast<mlir::StringAttr>(Val&: predicateNameAttr).getValue();
1316	auto predicate = fir::CmpcOp::getPredicateByName(predicateName);
1317	auto builder = parser.getBuilder();
1318	mlir::Type i1Type = builder.getI1Type();
1319	attrs.set(OPTY::getPredicateAttrName(),
1320	builder.getI64IntegerAttr(value: static_cast<std::int64_t>(predicate)));
1321	result.attributes = attrs;
1322	result.addTypes(newTypes: {i1Type});
1323	return mlir::success();
1324	}
1325
1326	//===----------------------------------------------------------------------===//
1327	// CmpcOp
1328	//===----------------------------------------------------------------------===//
1329
1330	void fir::buildCmpCOp(mlir::OpBuilder &builder, mlir::OperationState &result,
1331	mlir::arith::CmpFPredicate predicate, mlir::Value lhs,
1332	mlir::Value rhs) {
1333	result.addOperands({lhs, rhs});
1334	result.types.push_back(builder.getI1Type());
1335	result.addAttribute(
1336	fir::CmpcOp::getPredicateAttrName(),
1337	builder.getI64IntegerAttr(static_cast<std::int64_t>(predicate)));
1338	}
1339
1340	mlir::arith::CmpFPredicate
1341	fir::CmpcOp::getPredicateByName(llvm::StringRef name) {
1342	auto pred = mlir::arith::symbolizeCmpFPredicate(name);
1343	assert(pred.has_value() && "invalid predicate name");
1344	return pred.value();
1345	}
1346
1347	void fir::CmpcOp::print(mlir::OpAsmPrinter &p) { printCmpOp(p, *this); }
1348
1349	mlir::ParseResult fir::CmpcOp::parse(mlir::OpAsmParser &parser,
1350	mlir::OperationState &result) {
1351	return parseCmpOp<fir::CmpcOp>(parser, result);
1352	}
1353
1354	//===----------------------------------------------------------------------===//
1355	// VolatileCastOp
1356	//===----------------------------------------------------------------------===//
1357
1358	static bool typesMatchExceptForVolatility(mlir::Type fromType,
1359	mlir::Type toType) {
1360	// If we can change only the volatility and get identical types, then we
1361	// match.
1362	if (fir::updateTypeWithVolatility(fromType, fir::isa_volatile_type(toType)) ==
1363	toType)
1364	return true;
1365
1366	// Otherwise, recurse on the element types if the base classes are the same.
1367	const bool match =
1368	llvm::TypeSwitch<mlir::Type, bool>(fromType)
1369	.Case<fir::BoxType, fir::ReferenceType, fir::ClassType>(
1370	[&](auto type) {
1371	using TYPE = decltype(type);
1372	// If we are not the same base class, then we don't match.
1373	auto castedToType = mlir::dyn_cast<TYPE>(toType);
1374	if (!castedToType)
1375	return false;
1376	// If we are the same base class, we match if the element types
1377	// match.
1378	return typesMatchExceptForVolatility(type.getEleTy(),
1379	castedToType.getEleTy());
1380	})
1381	.Default([](mlir::Type) { return false; });
1382
1383	return match;
1384	}
1385
1386	llvm::LogicalResult fir::VolatileCastOp::verify() {
1387	mlir::Type fromType = getValue().getType();
1388	mlir::Type toType = getType();
1389	if (!typesMatchExceptForVolatility(fromType, toType))
1390	return emitOpError("types must be identical except for volatility ")
1391	<< fromType << " / " << toType;
1392	return mlir::success();
1393	}
1394
1395	mlir::OpFoldResult fir::VolatileCastOp::fold(FoldAdaptor adaptor) {
1396	if (getValue().getType() == getType())
1397	return getValue();
1398	return {};
1399	}
1400
1401	//===----------------------------------------------------------------------===//
1402	// ConvertOp
1403	//===----------------------------------------------------------------------===//
1404
1405	void fir::ConvertOp::getCanonicalizationPatterns(
1406	mlir::RewritePatternSet &results, mlir::MLIRContext *context) {
1407	results.insert<ConvertConvertOptPattern, ConvertAscendingIndexOptPattern,
1408	ConvertDescendingIndexOptPattern, RedundantConvertOptPattern,
1409	CombineConvertOptPattern, CombineConvertTruncOptPattern,
1410	ForwardConstantConvertPattern, ChainedPointerConvertsPattern>(
1411	context);
1412	}
1413
1414	mlir::OpFoldResult fir::ConvertOp::fold(FoldAdaptor adaptor) {
1415	if (getValue().getType() == getType())
1416	return getValue();
1417	if (matchPattern(getValue(), mlir::m_Op<fir::ConvertOp>())) {
1418	auto inner = mlir::cast<fir::ConvertOp>(getValue().getDefiningOp());
1419	// (convert (convert 'a : logical -> i1) : i1 -> logical) ==> forward 'a
1420	if (auto toTy = mlir::dyn_cast<fir::LogicalType>(getType()))
1421	if (auto fromTy =
1422	mlir::dyn_cast<fir::LogicalType>(inner.getValue().getType()))
1423	if (mlir::isa<mlir::IntegerType>(inner.getType()) && (toTy == fromTy))
1424	return inner.getValue();
1425	// (convert (convert 'a : i1 -> logical) : logical -> i1) ==> forward 'a
1426	if (auto toTy = mlir::dyn_cast<mlir::IntegerType>(getType()))
1427	if (auto fromTy =
1428	mlir::dyn_cast<mlir::IntegerType>(inner.getValue().getType()))
1429	if (mlir::isa<fir::LogicalType>(inner.getType()) && (toTy == fromTy) &&
1430	(fromTy.getWidth() == 1))
1431	return inner.getValue();
1432	}
1433	return {};
1434	}
1435
1436	bool fir::ConvertOp::isInteger(mlir::Type ty) {
1437	return mlir::isa<mlir::IntegerType, mlir::IndexType, fir::IntegerType>(ty);
1438	}
1439
1440	bool fir::ConvertOp::isIntegerCompatible(mlir::Type ty) {
1441	return isInteger(ty) || mlir::isa<fir::LogicalType>(ty);
1442	}
1443
1444	bool fir::ConvertOp::isFloatCompatible(mlir::Type ty) {
1445	return mlir::isa<mlir::FloatType>(ty);
1446	}
1447
1448	bool fir::ConvertOp::isPointerCompatible(mlir::Type ty) {
1449	return mlir::isa<fir::ReferenceType, fir::PointerType, fir::HeapType,
1450	fir::LLVMPointerType, mlir::MemRefType, mlir::FunctionType,
1451	fir::TypeDescType, mlir::LLVM::LLVMPointerType>(ty);
1452	}
1453
1454	static std::optional<mlir::Type> getVectorElementType(mlir::Type ty) {
1455	mlir::Type elemTy;
1456	if (mlir::isa<fir::VectorType>(ty))
1457	elemTy = mlir::dyn_cast<fir::VectorType>(ty).getElementType();
1458	else if (mlir::isa<mlir::VectorType>(Val: ty))
1459	elemTy = mlir::dyn_cast<mlir::VectorType>(Val&: ty).getElementType();
1460	else
1461	return std::nullopt;
1462
1463	// e.g. fir.vector<4:ui32> => mlir.vector<4xi32>
1464	// e.g. mlir.vector<4xui32> => mlir.vector<4xi32>
1465	if (elemTy.isUnsignedInteger()) {
1466	elemTy = mlir::IntegerType::get(
1467	context: ty.getContext(), width: mlir::dyn_cast<mlir::IntegerType>(Val&: elemTy).getWidth());
1468	}
1469	return elemTy;
1470	}
1471
1472	static std::optional<uint64_t> getVectorLen(mlir::Type ty) {
1473	if (mlir::isa<fir::VectorType>(ty))
1474	return mlir::dyn_cast<fir::VectorType>(ty).getLen();
1475	else if (mlir::isa<mlir::VectorType>(Val: ty)) {
1476	// fir.vector only supports 1-D vector
1477	if (!(mlir::dyn_cast<mlir::VectorType>(Val&: ty).isScalable()))
1478	return mlir::dyn_cast<mlir::VectorType>(Val&: ty).getShape()[0];
1479	}
1480
1481	return std::nullopt;
1482	}
1483
1484	bool fir::ConvertOp::areVectorsCompatible(mlir::Type inTy, mlir::Type outTy) {
1485	if (!(mlir::isa<fir::VectorType>(inTy) &&
1486	mlir::isa<mlir::VectorType>(outTy)) &&
1487	!(mlir::isa<mlir::VectorType>(inTy) && mlir::isa<fir::VectorType>(outTy)))
1488	return false;
1489
1490	// Only support integer, unsigned and real vector
1491	// Both vectors must have the same element type
1492	std::optional<mlir::Type> inElemTy = getVectorElementType(inTy);
1493	std::optional<mlir::Type> outElemTy = getVectorElementType(outTy);
1494	if (!inElemTy.has_value() || !outElemTy.has_value() ||
1495	inElemTy.value() != outElemTy.value())
1496	return false;
1497
1498	// Both vectors must have the same number of elements
1499	std::optional<uint64_t> inLen = getVectorLen(inTy);
1500	std::optional<uint64_t> outLen = getVectorLen(outTy);
1501	if (!inLen.has_value() || !outLen.has_value() ||
1502	inLen.value() != outLen.value())
1503	return false;
1504
1505	return true;
1506	}
1507
1508	static bool areRecordsCompatible(mlir::Type inTy, mlir::Type outTy) {
1509	// Both records must have the same field types.
1510	// Trust frontend semantics for in-depth checks, such as if both records
1511	// have the BIND(C) attribute.
1512	auto inRecTy = mlir::dyn_cast<fir::RecordType>(inTy);
1513	auto outRecTy = mlir::dyn_cast<fir::RecordType>(outTy);
1514	return inRecTy && outRecTy && inRecTy.getTypeList() == outRecTy.getTypeList();
1515	}
1516
1517	bool fir::ConvertOp::canBeConverted(mlir::Type inType, mlir::Type outType) {
1518	if (inType == outType)
1519	return true;
1520	return (isPointerCompatible(inType) && isPointerCompatible(outType)) ||
1521	(isIntegerCompatible(inType) && isIntegerCompatible(outType)) ||
1522	(isInteger(inType) && isFloatCompatible(outType)) ||
1523	(isFloatCompatible(inType) && isInteger(outType)) ||
1524	(isFloatCompatible(inType) && isFloatCompatible(outType)) ||
1525	(isIntegerCompatible(inType) && isPointerCompatible(outType)) ||
1526	(isPointerCompatible(inType) && isIntegerCompatible(outType)) ||
1527	(mlir::isa<fir::BoxType>(inType) &&
1528	mlir::isa<fir::BoxType>(outType)) ||
1529	(mlir::isa<fir::BoxProcType>(inType) &&
1530	mlir::isa<fir::BoxProcType>(outType)) ||
1531	(fir::isa_complex(inType) && fir::isa_complex(outType)) ||
1532	(fir::isBoxedRecordType(inType) && fir::isPolymorphicType(outType)) ||
1533	(fir::isPolymorphicType(inType) && fir::isPolymorphicType(outType)) ||
1534	(fir::isPolymorphicType(inType) && mlir::isa<BoxType>(outType)) ||
1535	areVectorsCompatible(inType, outType) ||
1536	areRecordsCompatible(inType, outType);
1537	}
1538
1539	// In general, ptrtoint-like conversions are allowed to lose volatility
1540	// information because they are either:
1541	//
1542	// 1. passing an entity to an external function and there's nothing we can do
1543	// about volatility after that happens, or
1544	// 2. for code generation, at which point we represent volatility with
1545	// attributes on the LLVM instructions and intrinsics.
1546	//
1547	// For all other cases, volatility ought to match exactly.
1548	static mlir::LogicalResult verifyVolatility(mlir::Type inType,
1549	mlir::Type outType) {
1550	const bool toLLVMPointer = mlir::isa<mlir::LLVM::LLVMPointerType>(Val: outType);
1551	const bool toInteger = fir::isa_integer(outType);
1552
1553	// When converting references to classes or allocatables into boxes for
1554	// runtime arguments, we cast away all the volatility information and pass a
1555	// box<none>. This is allowed.
1556	const bool isBoxNoneLike = [&]() {
1557	if (fir::isBoxNone(outType))
1558	return true;
1559	if (auto referenceType = mlir::dyn_cast<fir::ReferenceType>(outType)) {
1560	if (fir::isBoxNone(referenceType.getElementType())) {
1561	return true;
1562	}
1563	}
1564	return false;
1565	}();
1566
1567	const bool isPtrToIntLike = toLLVMPointer || toInteger || isBoxNoneLike;
1568	if (isPtrToIntLike) {
1569	return mlir::success();
1570	}
1571
1572	// In all other cases, we need to check for an exact volatility match.
1573	return mlir::success(fir::isa_volatile_type(inType) ==
1574	fir::isa_volatile_type(outType));
1575	}
1576
1577	llvm::LogicalResult fir::ConvertOp::verify() {
1578	mlir::Type inType = getValue().getType();
1579	mlir::Type outType = getType();
1580	if (fir::useStrictVolatileVerification()) {
1581	if (failed(verifyVolatility(inType, outType))) {
1582	return emitOpError("this conversion does not preserve volatility: ")
1583	<< inType << " / " << outType;
1584	}
1585	}
1586	if (canBeConverted(inType, outType))
1587	return mlir::success();
1588	return emitOpError("invalid type conversion")
1589	<< getValue().getType() << " / " << getType();
1590	}
1591
1592	//===----------------------------------------------------------------------===//
1593	// CoordinateOp
1594	//===----------------------------------------------------------------------===//
1595
1596	void fir::CoordinateOp::build(mlir::OpBuilder &builder,
1597	mlir::OperationState &result,
1598	mlir::Type resultType, mlir::Value ref,
1599	mlir::ValueRange coor) {
1600	llvm::SmallVector<int32_t> fieldIndices;
1601	llvm::SmallVector<mlir::Value> dynamicIndices;
1602	bool anyField = false;
1603	for (mlir::Value index : coor) {
1604	if (auto field = index.getDefiningOp<fir::FieldIndexOp>()) {
1605	auto recTy = mlir::cast<fir::RecordType>(field.getOnType());
1606	fieldIndices.push_back(recTy.getFieldIndex(field.getFieldId()));
1607	anyField = true;
1608	} else {
1609	fieldIndices.push_back(fir::CoordinateOp::kDynamicIndex);
1610	dynamicIndices.push_back(index);
1611	}
1612	}
1613	auto typeAttr = mlir::TypeAttr::get(ref.getType());
1614	if (anyField) {
1615	build(builder, result, resultType, ref, dynamicIndices, typeAttr,
1616	builder.getDenseI32ArrayAttr(fieldIndices));
1617	} else {
1618	build(builder, result, resultType, ref, dynamicIndices, typeAttr, nullptr);
1619	}
1620	}
1621
1622	void fir::CoordinateOp::build(mlir::OpBuilder &builder,
1623	mlir::OperationState &result,
1624	mlir::Type resultType, mlir::Value ref,
1625	llvm::ArrayRef<fir::IntOrValue> coor) {
1626	llvm::SmallVector<int32_t> fieldIndices;
1627	llvm::SmallVector<mlir::Value> dynamicIndices;
1628	bool anyField = false;
1629	for (fir::IntOrValue index : coor) {
1630	llvm::TypeSwitch<fir::IntOrValue>(index)
1631	.Case<mlir::IntegerAttr>([&](mlir::IntegerAttr intAttr) {
1632	fieldIndices.push_back(intAttr.getInt());
1633	anyField = true;
1634	})
1635	.Case<mlir::Value>([&](mlir::Value value) {
1636	dynamicIndices.push_back(value);
1637	fieldIndices.push_back(fir::CoordinateOp::kDynamicIndex);
1638	});
1639	}
1640	auto typeAttr = mlir::TypeAttr::get(ref.getType());
1641	if (anyField) {
1642	build(builder, result, resultType, ref, dynamicIndices, typeAttr,
1643	builder.getDenseI32ArrayAttr(fieldIndices));
1644	} else {
1645	build(builder, result, resultType, ref, dynamicIndices, typeAttr, nullptr);
1646	}
1647	}
1648
1649	void fir::CoordinateOp::print(mlir::OpAsmPrinter &p) {
1650	p << ' ' << getRef();
1651	if (!getFieldIndicesAttr()) {
1652	p << ", " << getCoor();
1653	} else {
1654	mlir::Type eleTy = fir::getFortranElementType(getRef().getType());
1655	for (auto index : getIndices()) {
1656	p << ", ";
1657	llvm::TypeSwitch<fir::IntOrValue>(index)
1658	.Case<mlir::IntegerAttr>([&](mlir::IntegerAttr intAttr) {
1659	if (auto recordType = llvm::dyn_cast<fir::RecordType>(eleTy)) {
1660	int fieldId = intAttr.getInt();
1661	if (fieldId < static_cast<int>(recordType.getNumFields())) {
1662	auto nameAndType = recordType.getTypeList()[fieldId];
1663	p << std::get<std::string>(nameAndType);
1664	eleTy = fir::getFortranElementType(
1665	std::get<mlir::Type>(nameAndType));
1666	return;
1667	}
1668	}
1669	// Invalid index, still print it so that invalid IR can be
1670	// investigated.
1671	p << intAttr;
1672	})
1673	.Case<mlir::Value>([&](mlir::Value value) { p << value; });
1674	}
1675	}
1676	p.printOptionalAttrDict(
1677	(*this)->getAttrs(),
1678	/*elideAttrs=*/{getBaseTypeAttrName(), getFieldIndicesAttrName()});
1679	p << " : ";
1680	p.printFunctionalType(getOperandTypes(), (*this)->getResultTypes());
1681	}
1682
1683	mlir::ParseResult fir::CoordinateOp::parse(mlir::OpAsmParser &parser,
1684	mlir::OperationState &result) {
1685	mlir::OpAsmParser::UnresolvedOperand memref;
1686	if (parser.parseOperand(memref) || parser.parseComma())
1687	return mlir::failure();
1688	llvm::SmallVector<mlir::OpAsmParser::UnresolvedOperand> coorOperands;
1689	llvm::SmallVector<std::pair<llvm::StringRef, int>> fieldNames;
1690	llvm::SmallVector<int32_t> fieldIndices;
1691	while (true) {
1692	llvm::StringRef fieldName;
1693	if (mlir::succeeded(parser.parseOptionalKeyword(&fieldName))) {
1694	fieldNames.push_back({fieldName, static_cast<int>(fieldIndices.size())});
1695	// Actual value will be computed later when base type has been parsed.
1696	fieldIndices.push_back(0);
1697	} else {
1698	mlir::OpAsmParser::UnresolvedOperand index;
1699	if (parser.parseOperand(index))
1700	return mlir::failure();
1701	fieldIndices.push_back(fir::CoordinateOp::kDynamicIndex);
1702	coorOperands.push_back(index);
1703	}
1704	if (mlir::failed(parser.parseOptionalComma()))
1705	break;
1706	}
1707	llvm::SmallVector<mlir::OpAsmParser::UnresolvedOperand> allOperands;
1708	allOperands.push_back(memref);
1709	allOperands.append(coorOperands.begin(), coorOperands.end());
1710	mlir::FunctionType funcTy;
1711	auto loc = parser.getCurrentLocation();
1712	if (parser.parseOptionalAttrDict(result.attributes) ||
1713	parser.parseColonType(funcTy) ||
1714	parser.resolveOperands(allOperands, funcTy.getInputs(), loc,
1715	result.operands) ||
1716	parser.addTypesToList(funcTy.getResults(), result.types))
1717	return mlir::failure();
1718	result.addAttribute(getBaseTypeAttrName(result.name),
1719	mlir::TypeAttr::get(funcTy.getInput(0)));
1720	if (!fieldNames.empty()) {
1721	mlir::Type eleTy = fir::getFortranElementType(funcTy.getInput(0));
1722	for (auto [fieldName, operandPosition] : fieldNames) {
1723	auto recTy = llvm::dyn_cast<fir::RecordType>(eleTy);
1724	if (!recTy)
1725	return parser.emitError(
1726	loc, "base must be a derived type when field name appears");
1727	unsigned fieldNum = recTy.getFieldIndex(fieldName);
1728	if (fieldNum > recTy.getNumFields())
1729	return parser.emitError(loc)
1730	<< "field '" << fieldName
1731	<< "' is not a component or subcomponent of the base type";
1732	fieldIndices[operandPosition] = fieldNum;
1733	eleTy = fir::getFortranElementType(
1734	std::get<mlir::Type>(recTy.getTypeList()[fieldNum]));
1735	}
1736	result.addAttribute(getFieldIndicesAttrName(result.name),
1737	parser.getBuilder().getDenseI32ArrayAttr(fieldIndices));
1738	}
1739	return mlir::success();
1740	}
1741
1742	llvm::LogicalResult fir::CoordinateOp::verify() {
1743	const mlir::Type refTy = getRef().getType();
1744	if (fir::isa_ref_type(refTy)) {
1745	auto eleTy = fir::dyn_cast_ptrEleTy(refTy);
1746	if (auto arrTy = mlir::dyn_cast<fir::SequenceType>(eleTy)) {
1747	if (arrTy.hasUnknownShape())
1748	return emitOpError("cannot find coordinate in unknown shape");
1749	if (arrTy.getConstantRows() < arrTy.getDimension() - 1)
1750	return emitOpError("cannot find coordinate with unknown extents");
1751	}
1752	if (!(fir::isa_aggregate(eleTy) || fir::isa_complex(eleTy) ||
1753	fir::isa_char_string(eleTy)))
1754	return emitOpError("cannot apply to this element type");
1755	}
1756	auto eleTy = fir::dyn_cast_ptrOrBoxEleTy(refTy);
1757	unsigned dimension = 0;
1758	const unsigned numCoors = getCoor().size();
1759	for (auto coorOperand : llvm::enumerate(getCoor())) {
1760	auto co = coorOperand.value();
1761	if (dimension == 0 && mlir::isa<fir::SequenceType>(eleTy)) {
1762	dimension = mlir::cast<fir::SequenceType>(eleTy).getDimension();
1763	if (dimension == 0)
1764	return emitOpError("cannot apply to array of unknown rank");
1765	}
1766	if (auto *defOp = co.getDefiningOp()) {
1767	if (auto index = mlir::dyn_cast<fir::LenParamIndexOp>(defOp)) {
1768	// Recovering a LEN type parameter only makes sense from a boxed
1769	// value. For a bare reference, the LEN type parameters must be
1770	// passed as additional arguments to `index`.
1771	if (mlir::isa<fir::BoxType>(refTy)) {
1772	if (coorOperand.index() != numCoors - 1)
1773	return emitOpError("len_param_index must be last argument");
1774	if (getNumOperands() != 2)
1775	return emitOpError("too many operands for len_param_index case");
1776	}
1777	if (eleTy != index.getOnType())
1778	emitOpError(
1779	"len_param_index type not compatible with reference type");
1780	return mlir::success();
1781	} else if (auto index = mlir::dyn_cast<fir::FieldIndexOp>(defOp)) {
1782	if (eleTy != index.getOnType())
1783	emitOpError("field_index type not compatible with reference type");
1784	if (auto recTy = mlir::dyn_cast<fir::RecordType>(eleTy)) {
1785	eleTy = recTy.getType(index.getFieldName());
1786	continue;
1787	}
1788	return emitOpError("field_index not applied to !fir.type");
1789	}
1790	}
1791	if (dimension) {
1792	if (--dimension == 0)
1793	eleTy = mlir::cast<fir::SequenceType>(eleTy).getElementType();
1794	} else {
1795	if (auto t = mlir::dyn_cast<mlir::TupleType>(eleTy)) {
1796	// FIXME: Generally, we don't know which field of the tuple is being
1797	// referred to unless the operand is a constant. Just assume everything
1798	// is good in the tuple case for now.
1799	return mlir::success();
1800	} else if (auto t = mlir::dyn_cast<fir::RecordType>(eleTy)) {
1801	// FIXME: This is the same as the tuple case.
1802	return mlir::success();
1803	} else if (auto t = mlir::dyn_cast<mlir::ComplexType>(eleTy)) {
1804	eleTy = t.getElementType();
1805	} else if (auto t = mlir::dyn_cast<fir::CharacterType>(eleTy)) {
1806	if (t.getLen() == fir::CharacterType::singleton())
1807	return emitOpError("cannot apply to character singleton");
1808	eleTy = fir::CharacterType::getSingleton(t.getContext(), t.getFKind());
1809	if (fir::unwrapRefType(getType()) != eleTy)
1810	return emitOpError("character type mismatch");
1811	} else {
1812	return emitOpError("invalid parameters (too many)");
1813	}
1814	}
1815	}
1816	return mlir::success();
1817	}
1818
1819	fir::CoordinateIndicesAdaptor fir::CoordinateOp::getIndices() {
1820	return CoordinateIndicesAdaptor(getFieldIndicesAttr(), getCoor());
1821	}
1822
1823	//===----------------------------------------------------------------------===//
1824	// DispatchOp
1825	//===----------------------------------------------------------------------===//
1826
1827	llvm::LogicalResult fir::DispatchOp::verify() {
1828	// Check that pass_arg_pos is in range of actual operands. pass_arg_pos is
1829	// unsigned so check for less than zero is not needed.
1830	if (getPassArgPos() && *getPassArgPos() > (getArgOperands().size() - 1))
1831	return emitOpError(
1832	"pass_arg_pos must be smaller than the number of operands");
1833
1834	// Operand pointed by pass_arg_pos must have polymorphic type.
1835	if (getPassArgPos() &&
1836	!fir::isPolymorphicType(getArgOperands()[*getPassArgPos()].getType()))
1837	return emitOpError("pass_arg_pos must be a polymorphic operand");
1838	return mlir::success();
1839	}
1840
1841	mlir::FunctionType fir::DispatchOp::getFunctionType() {
1842	return mlir::FunctionType::get(getContext(), getOperandTypes(),
1843	getResultTypes());
1844	}
1845
1846	//===----------------------------------------------------------------------===//
1847	// TypeInfoOp
1848	//===----------------------------------------------------------------------===//
1849
1850	void fir::TypeInfoOp::build(mlir::OpBuilder &builder,
1851	mlir::OperationState &result, fir::RecordType type,
1852	fir::RecordType parentType,
1853	llvm::ArrayRef<mlir::NamedAttribute> attrs) {
1854	result.addRegion();
1855	result.addRegion();
1856	result.addAttribute(mlir::SymbolTable::getSymbolAttrName(),
1857	builder.getStringAttr(type.getName()));
1858	result.addAttribute(getTypeAttrName(result.name), mlir::TypeAttr::get(type));
1859	if (parentType)
1860	result.addAttribute(getParentTypeAttrName(result.name),
1861	mlir::TypeAttr::get(parentType));
1862	result.addAttributes(attrs);
1863	}
1864
1865	llvm::LogicalResult fir::TypeInfoOp::verify() {
1866	if (!getDispatchTable().empty())
1867	for (auto &op : getDispatchTable().front().without_terminator())
1868	if (!mlir::isa<fir::DTEntryOp>(op))
1869	return op.emitOpError("dispatch table must contain dt_entry");
1870
1871	if (!mlir::isa<fir::RecordType>(getType()))
1872	return emitOpError("type must be a fir.type");
1873
1874	if (getParentType() && !mlir::isa<fir::RecordType>(*getParentType()))
1875	return emitOpError("parent_type must be a fir.type");
1876	return mlir::success();
1877	}
1878
1879	//===----------------------------------------------------------------------===//
1880	// EmboxOp
1881	//===----------------------------------------------------------------------===//
1882
1883	// Conversions from reference types to box types must preserve volatility.
1884	static llvm::LogicalResult
1885	verifyEmboxOpVolatilityInvariants(mlir::Type memrefType,
1886	mlir::Type resultType) {
1887
1888	if (!fir::useStrictVolatileVerification())
1889	return mlir::success();
1890
1891	mlir::Type boxElementType =
1892	llvm::TypeSwitch<mlir::Type, mlir::Type>(resultType)
1893	.Case<fir::BoxType, fir::ClassType>(
1894	[&](auto type) { return type.getEleTy(); })
1895	.Default([&](mlir::Type type) { return type; });
1896
1897	// If the embox is simply wrapping a non-volatile type into a volatile box,
1898	// we're not losing any volatility information.
1899	if (boxElementType == memrefType) {
1900	return mlir::success();
1901	}
1902
1903	// Otherwise, the volatility of the input and result must match.
1904	const bool volatilityMatches =
1905	fir::isa_volatile_type(memrefType) == fir::isa_volatile_type(resultType);
1906
1907	return mlir::success(IsSuccess: volatilityMatches);
1908	}
1909
1910	llvm::LogicalResult fir::EmboxOp::verify() {
1911	auto eleTy = fir::dyn_cast_ptrEleTy(getMemref().getType());
1912	bool isArray = false;
1913	if (auto seqTy = mlir::dyn_cast<fir::SequenceType>(eleTy)) {
1914	eleTy = seqTy.getEleTy();
1915	isArray = true;
1916	}
1917	if (hasLenParams()) {
1918	auto lenPs = numLenParams();
1919	if (auto rt = mlir::dyn_cast<fir::RecordType>(eleTy)) {
1920	if (lenPs != rt.getNumLenParams())
1921	return emitOpError("number of LEN params does not correspond"
1922	" to the !fir.type type");
1923	} else if (auto strTy = mlir::dyn_cast<fir::CharacterType>(eleTy)) {
1924	if (strTy.getLen() != fir::CharacterType::unknownLen())
1925	return emitOpError("CHARACTER already has static LEN");
1926	} else {
1927	return emitOpError("LEN parameters require CHARACTER or derived type");
1928	}
1929	for (auto lp : getTypeparams())
1930	if (!fir::isa_integer(lp.getType()))
1931	return emitOpError("LEN parameters must be integral type");
1932	}
1933	if (getShape() && !isArray)
1934	return emitOpError("shape must not be provided for a scalar");
1935	if (getSlice() && !isArray)
1936	return emitOpError("slice must not be provided for a scalar");
1937	if (getSourceBox() && !mlir::isa<fir::ClassType>(getResult().getType()))
1938	return emitOpError("source_box must be used with fir.class result type");
1939	if (failed(verifyEmboxOpVolatilityInvariants(getMemref().getType(),
1940	getResult().getType())))
1941	return emitOpError(
1942	"cannot convert between volatile and non-volatile types:")
1943	<< " " << getMemref().getType() << " " << getResult().getType();
1944	return mlir::success();
1945	}
1946
1947	/// Returns true if \p extent matches the extent of the \p box's
1948	/// dimension \p dim.
1949	static bool isBoxExtent(mlir::Value box, std::int64_t dim, mlir::Value extent) {
1950	if (auto op = extent.getDefiningOp<fir::BoxDimsOp>())
1951	if (op.getVal() == box && op.getExtent() == extent)
1952	if (auto dimOperand = fir::getIntIfConstant(op.getDim()))
1953	return *dimOperand == dim;
1954	return false;
1955	}
1956
1957	/// Returns true if \p lb matches the lower bound of the \p box's
1958	/// dimension \p dim. If \p mayHaveNonDefaultLowerBounds is false,
1959	/// then \p lb may be an integer constant 1.
1960	static bool isBoxLb(mlir::Value box, std::int64_t dim, mlir::Value lb,
1961	bool mayHaveNonDefaultLowerBounds = true) {
1962	if (auto op = lb.getDefiningOp<fir::BoxDimsOp>()) {
1963	if (op.getVal() == box && op.getLowerBound() == lb)
1964	if (auto dimOperand = fir::getIntIfConstant(op.getDim()))
1965	return *dimOperand == dim;
1966	} else if (!mayHaveNonDefaultLowerBounds) {
1967	if (auto constantLb = fir::getIntIfConstant(lb))
1968	return *constantLb == 1;
1969	}
1970	return false;
1971	}
1972
1973	/// Returns true if \p ub matches the upper bound of the \p box's
1974	/// dimension \p dim. If \p mayHaveNonDefaultLowerBounds is false,
1975	/// then the dimension's lower bound may be an integer constant 1.
1976	/// Note that the upper bound is usually a result of computation
1977	/// involving the lower bound and the extent, and the function
1978	/// tries its best to recognize the computation pattern.
1979	/// The conservative result 'false' does not necessarily mean
1980	/// that \p ub is not an actual upper bound value.
1981	static bool isBoxUb(mlir::Value box, std::int64_t dim, mlir::Value ub,
1982	bool mayHaveNonDefaultLowerBounds = true) {
1983	if (auto sub1 = ub.getDefiningOp<mlir::arith::SubIOp>()) {
1984	auto one = fir::getIntIfConstant(sub1.getOperand(1));
1985	if (!one || *one != 1)
1986	return false;
1987	if (auto add = sub1.getOperand(0).getDefiningOp<mlir::arith::AddIOp>())
1988	if ((isBoxLb(box, dim, add.getOperand(0)) &&
1989	isBoxExtent(box, dim, add.getOperand(1))) ||
1990	(isBoxLb(box, dim, add.getOperand(1)) &&
1991	isBoxExtent(box, dim, add.getOperand(0))))
1992	return true;
1993	} else if (!mayHaveNonDefaultLowerBounds) {
1994	return isBoxExtent(box, dim, extent: ub);
1995	}
1996	return false;
1997	}
1998
1999	/// Checks if the given \p sliceOp specifies a contiguous
2000	/// array slice. If \p checkWhole is true, then the check
2001	/// is done for all dimensions, otherwise, only for the innermost
2002	/// dimension.
2003	/// The simplest way to prove that this is an contiguous slice
2004	/// is to check whether the slice stride(s) is 1.
2005	/// For more complex cases, extra information must be provided
2006	/// by the caller:
2007	/// * \p origBox - if not null, then the source array is represented
2008	/// with this !fir.box value. The box is used to recognize
2009	/// the full dimension slices, which are specified by the triplets
2010	/// computed from the dimensions' lower bounds and extents.
2011	/// * \p mayHaveNonDefaultLowerBounds may be set to false to indicate
2012	/// that the source entity has default lower bounds, so the full
2013	/// dimension slices computations may use 1 for the lower bound.
2014	static bool isContiguousArraySlice(fir::SliceOp sliceOp, bool checkWhole = true,
2015	mlir::Value origBox = nullptr,
2016	bool mayHaveNonDefaultLowerBounds = true) {
2017	if (sliceOp.getFields().empty() && sliceOp.getSubstr().empty()) {
2018	// TODO: generalize code for the triples analysis with
2019	// hlfir::designatePreservesContinuity, especially when
2020	// recognition of the whole dimension slices is added.
2021	auto triples = sliceOp.getTriples();
2022	assert((triples.size() % 3) == 0 && "invalid triples size");
2023
2024	// A slice with step=1 in the innermost dimension preserves
2025	// the continuity of the array in the innermost dimension.
2026	// If checkWhole is false, then check only the innermost slice triples.
2027	std::size_t checkUpTo = checkWhole ? triples.size() : 3;
2028	checkUpTo = std::min(checkUpTo, triples.size());
2029	for (std::size_t i = 0; i < checkUpTo; i += 3) {
2030	if (triples[i] != triples[i + 1]) {
2031	// This is a section of the dimension. Only allow it
2032	// to be the first triple, if the source of the slice
2033	// is a boxed array. If it is a raw pointer, then
2034	// the result will still be contiguous, as long as
2035	// the strides are all ones.
2036	// When origBox is not null, we must prove that the triple
2037	// covers the whole dimension and the stride is one,
2038	// before claiming contiguity for this dimension.
2039	if (i != 0 && origBox) {
2040	std::int64_t dim = i / 3;
2041	if (!isBoxLb(origBox, dim, triples[i],
2042	mayHaveNonDefaultLowerBounds) ||
2043	!isBoxUb(origBox, dim, triples[i + 1],
2044	mayHaveNonDefaultLowerBounds))
2045	return false;
2046	}
2047	auto constantStep = fir::getIntIfConstant(triples[i + 2]);
2048	if (!constantStep || *constantStep != 1)
2049	return false;
2050	}
2051	}
2052	return true;
2053	}
2054	return false;
2055	}
2056
2057	bool fir::isContiguousEmbox(fir::EmboxOp embox, bool checkWhole) {
2058	auto sliceArg = embox.getSlice();
2059	if (!sliceArg)
2060	return true;
2061
2062	if (auto sliceOp =
2063	mlir::dyn_cast_or_null<fir::SliceOp>(sliceArg.getDefiningOp()))
2064	return isContiguousArraySlice(sliceOp, checkWhole);
2065
2066	return false;
2067	}
2068
2069	//===----------------------------------------------------------------------===//
2070	// EmboxCharOp
2071	//===----------------------------------------------------------------------===//
2072
2073	llvm::LogicalResult fir::EmboxCharOp::verify() {
2074	auto eleTy = fir::dyn_cast_ptrEleTy(getMemref().getType());
2075	if (!mlir::dyn_cast_or_null<fir::CharacterType>(eleTy))
2076	return mlir::failure();
2077	return mlir::success();
2078	}
2079
2080	//===----------------------------------------------------------------------===//
2081	// EmboxProcOp
2082	//===----------------------------------------------------------------------===//
2083
2084	llvm::LogicalResult fir::EmboxProcOp::verify() {
2085	// host bindings (optional) must be a reference to a tuple
2086	if (auto h = getHost()) {
2087	if (auto r = mlir::dyn_cast<fir::ReferenceType>(h.getType()))
2088	if (mlir::isa<mlir::TupleType>(r.getEleTy()))
2089	return mlir::success();
2090	return mlir::failure();
2091	}
2092	return mlir::success();
2093	}
2094
2095	//===----------------------------------------------------------------------===//
2096	// TypeDescOp
2097	//===----------------------------------------------------------------------===//
2098
2099	void fir::TypeDescOp::build(mlir::OpBuilder &, mlir::OperationState &result,
2100	mlir::TypeAttr inty) {
2101	result.addAttribute("in_type", inty);
2102	result.addTypes(TypeDescType::get(inty.getValue()));
2103	}
2104
2105	mlir::ParseResult fir::TypeDescOp::parse(mlir::OpAsmParser &parser,
2106	mlir::OperationState &result) {
2107	mlir::Type intype;
2108	if (parser.parseType(intype))
2109	return mlir::failure();
2110	result.addAttribute("in_type", mlir::TypeAttr::get(intype));
2111	mlir::Type restype = fir::TypeDescType::get(intype);
2112	if (parser.addTypeToList(restype, result.types))
2113	return mlir::failure();
2114	return mlir::success();
2115	}
2116
2117	void fir::TypeDescOp::print(mlir::OpAsmPrinter &p) {
2118	p << ' ' << getOperation()->getAttr("in_type");
2119	p.printOptionalAttrDict(getOperation()->getAttrs(), {"in_type"});
2120	}
2121
2122	llvm::LogicalResult fir::TypeDescOp::verify() {
2123	mlir::Type resultTy = getType();
2124	if (auto tdesc = mlir::dyn_cast<fir::TypeDescType>(resultTy)) {
2125	if (tdesc.getOfTy() != getInType())
2126	return emitOpError("wrapped type mismatched");
2127	return mlir::success();
2128	}
2129	return emitOpError("must be !fir.tdesc type");
2130	}
2131
2132	//===----------------------------------------------------------------------===//
2133	// GlobalOp
2134	//===----------------------------------------------------------------------===//
2135
2136	mlir::Type fir::GlobalOp::resultType() {
2137	return wrapAllocaResultType(getType());
2138	}
2139
2140	mlir::ParseResult fir::GlobalOp::parse(mlir::OpAsmParser &parser,
2141	mlir::OperationState &result) {
2142	// Parse the optional linkage
2143	llvm::StringRef linkage;
2144	auto &builder = parser.getBuilder();
2145	if (mlir::succeeded(parser.parseOptionalKeyword(&linkage))) {
2146	if (fir::GlobalOp::verifyValidLinkage(linkage))
2147	return mlir::failure();
2148	mlir::StringAttr linkAttr = builder.getStringAttr(linkage);
2149	result.addAttribute(fir::GlobalOp::getLinkNameAttrName(result.name),
2150	linkAttr);
2151	}
2152
2153	// Parse the name as a symbol reference attribute.
2154	mlir::SymbolRefAttr nameAttr;
2155	if (parser.parseAttribute(nameAttr,
2156	fir::GlobalOp::getSymrefAttrName(result.name),
2157	result.attributes))
2158	return mlir::failure();
2159	result.addAttribute(mlir::SymbolTable::getSymbolAttrName(),
2160	nameAttr.getRootReference());
2161
2162	bool simpleInitializer = false;
2163	if (mlir::succeeded(parser.parseOptionalLParen())) {
2164	mlir::Attribute attr;
2165	if (parser.parseAttribute(attr, getInitValAttrName(result.name),
2166	result.attributes) ||
2167	parser.parseRParen())
2168	return mlir::failure();
2169	simpleInitializer = true;
2170	}
2171
2172	if (parser.parseOptionalAttrDict(result.attributes))
2173	return mlir::failure();
2174
2175	if (succeeded(
2176	parser.parseOptionalKeyword(getConstantAttrName(result.name)))) {
2177	// if "constant" keyword then mark this as a constant, not a variable
2178	result.addAttribute(getConstantAttrName(result.name),
2179	builder.getUnitAttr());
2180	}
2181
2182	if (succeeded(parser.parseOptionalKeyword(getTargetAttrName(result.name))))
2183	result.addAttribute(getTargetAttrName(result.name), builder.getUnitAttr());
2184
2185	mlir::Type globalType;
2186	if (parser.parseColonType(globalType))
2187	return mlir::failure();
2188
2189	result.addAttribute(fir::GlobalOp::getTypeAttrName(result.name),
2190	mlir::TypeAttr::get(globalType));
2191
2192	if (simpleInitializer) {
2193	result.addRegion();
2194	} else {
2195	// Parse the optional initializer body.
2196	auto parseResult =
2197	parser.parseOptionalRegion(*result.addRegion(), /*arguments=*/{});
2198	if (parseResult.has_value() && mlir::failed(*parseResult))
2199	return mlir::failure();
2200	}
2201	return mlir::success();
2202	}
2203
2204	void fir::GlobalOp::print(mlir::OpAsmPrinter &p) {
2205	if (getLinkName())
2206	p << ' ' << *getLinkName();
2207	p << ' ';
2208	p.printAttributeWithoutType(getSymrefAttr());
2209	if (auto val = getValueOrNull())
2210	p << '(' << val << ')';
2211	// Print all other attributes that are not pretty printed here.
2212	p.printOptionalAttrDict((*this)->getAttrs(), /*elideAttrs=*/{
2213	getSymNameAttrName(), getSymrefAttrName(),
2214	getTypeAttrName(), getConstantAttrName(),
2215	getTargetAttrName(), getLinkNameAttrName(),
2216	getInitValAttrName()});
2217	if (getOperation()->getAttr(getConstantAttrName()))
2218	p << " " << getConstantAttrName().strref();
2219	if (getOperation()->getAttr(getTargetAttrName()))
2220	p << " " << getTargetAttrName().strref();
2221	p << " : ";
2222	p.printType(getType());
2223	if (hasInitializationBody()) {
2224	p << ' ';
2225	p.printRegion(getOperation()->getRegion(0),
2226	/*printEntryBlockArgs=*/false,
2227	/*printBlockTerminators=*/true);
2228	}
2229	}
2230
2231	void fir::GlobalOp::appendInitialValue(mlir::Operation *op) {
2232	getBlock().getOperations().push_back(op);
2233	}
2234
2235	void fir::GlobalOp::build(mlir::OpBuilder &builder,
2236	mlir::OperationState &result, llvm::StringRef name,
2237	bool isConstant, bool isTarget, mlir::Type type,
2238	mlir::Attribute initialVal, mlir::StringAttr linkage,
2239	llvm::ArrayRef<mlir::NamedAttribute> attrs) {
2240	result.addRegion();
2241	result.addAttribute(getTypeAttrName(result.name), mlir::TypeAttr::get(type));
2242	result.addAttribute(mlir::SymbolTable::getSymbolAttrName(),
2243	builder.getStringAttr(name));
2244	result.addAttribute(getSymrefAttrName(result.name),
2245	mlir::SymbolRefAttr::get(builder.getContext(), name));
2246	if (isConstant)
2247	result.addAttribute(getConstantAttrName(result.name),
2248	builder.getUnitAttr());
2249	if (isTarget)
2250	result.addAttribute(getTargetAttrName(result.name), builder.getUnitAttr());
2251	if (initialVal)
2252	result.addAttribute(getInitValAttrName(result.name), initialVal);
2253	if (linkage)
2254	result.addAttribute(getLinkNameAttrName(result.name), linkage);
2255	result.attributes.append(attrs.begin(), attrs.end());
2256	}
2257
2258	void fir::GlobalOp::build(mlir::OpBuilder &builder,
2259	mlir::OperationState &result, llvm::StringRef name,
2260	mlir::Type type, mlir::Attribute initialVal,
2261	mlir::StringAttr linkage,
2262	llvm::ArrayRef<mlir::NamedAttribute> attrs) {
2263	build(builder, result, name, /*isConstant=*/false, /*isTarget=*/false, type,
2264	{}, linkage, attrs);
2265	}
2266
2267	void fir::GlobalOp::build(mlir::OpBuilder &builder,
2268	mlir::OperationState &result, llvm::StringRef name,
2269	bool isConstant, bool isTarget, mlir::Type type,
2270	mlir::StringAttr linkage,
2271	llvm::ArrayRef<mlir::NamedAttribute> attrs) {
2272	build(builder, result, name, isConstant, isTarget, type, {}, linkage, attrs);
2273	}
2274
2275	void fir::GlobalOp::build(mlir::OpBuilder &builder,
2276	mlir::OperationState &result, llvm::StringRef name,
2277	mlir::Type type, mlir::StringAttr linkage,
2278	llvm::ArrayRef<mlir::NamedAttribute> attrs) {
2279	build(builder, result, name, /*isConstant=*/false, /*isTarget=*/false, type,
2280	{}, linkage, attrs);
2281	}
2282
2283	void fir::GlobalOp::build(mlir::OpBuilder &builder,
2284	mlir::OperationState &result, llvm::StringRef name,
2285	bool isConstant, bool isTarget, mlir::Type type,
2286	llvm::ArrayRef<mlir::NamedAttribute> attrs) {
2287	build(builder, result, name, isConstant, isTarget, type, mlir::StringAttr{},
2288	attrs);
2289	}
2290
2291	void fir::GlobalOp::build(mlir::OpBuilder &builder,
2292	mlir::OperationState &result, llvm::StringRef name,
2293	mlir::Type type,
2294	llvm::ArrayRef<mlir::NamedAttribute> attrs) {
2295	build(builder, result, name, /*isConstant=*/false, /*isTarget=*/false, type,
2296	attrs);
2297	}
2298
2299	mlir::ParseResult fir::GlobalOp::verifyValidLinkage(llvm::StringRef linkage) {
2300	// Supporting only a subset of the LLVM linkage types for now
2301	static const char *validNames[] = {"common", "internal", "linkonce",
2302	"linkonce_odr", "weak"};
2303	return mlir::success(llvm::is_contained(validNames, linkage));
2304	}
2305
2306	//===----------------------------------------------------------------------===//
2307	// GlobalLenOp
2308	//===----------------------------------------------------------------------===//
2309
2310	mlir::ParseResult fir::GlobalLenOp::parse(mlir::OpAsmParser &parser,
2311	mlir::OperationState &result) {
2312	llvm::StringRef fieldName;
2313	if (failed(parser.parseOptionalKeyword(&fieldName))) {
2314	mlir::StringAttr fieldAttr;
2315	if (parser.parseAttribute(fieldAttr,
2316	fir::GlobalLenOp::getLenParamAttrName(),
2317	result.attributes))
2318	return mlir::failure();
2319	} else {
2320	result.addAttribute(fir::GlobalLenOp::getLenParamAttrName(),
2321	parser.getBuilder().getStringAttr(fieldName));
2322	}
2323	mlir::IntegerAttr constant;
2324	if (parser.parseComma() ||
2325	parser.parseAttribute(constant, fir::GlobalLenOp::getIntAttrName(),
2326	result.attributes))
2327	return mlir::failure();
2328	return mlir::success();
2329	}
2330
2331	void fir::GlobalLenOp::print(mlir::OpAsmPrinter &p) {
2332	p << ' ' << getOperation()->getAttr(fir::GlobalLenOp::getLenParamAttrName())
2333	<< ", " << getOperation()->getAttr(fir::GlobalLenOp::getIntAttrName());
2334	}
2335
2336	//===----------------------------------------------------------------------===//
2337	// FieldIndexOp
2338	//===----------------------------------------------------------------------===//
2339
2340	template <typename TY>
2341	mlir::ParseResult parseFieldLikeOp(mlir::OpAsmParser &parser,
2342	mlir::OperationState &result) {
2343	llvm::StringRef fieldName;
2344	auto &builder = parser.getBuilder();
2345	mlir::Type recty;
2346	if (parser.parseOptionalKeyword(keyword: &fieldName) || parser.parseComma() ||
2347	parser.parseType(result&: recty))
2348	return mlir::failure();
2349	result.addAttribute(fir::FieldIndexOp::getFieldAttrName(),
2350	builder.getStringAttr(fieldName));
2351	if (!mlir::dyn_cast<fir::RecordType>(recty))
2352	return mlir::failure();
2353	result.addAttribute(fir::FieldIndexOp::getTypeAttrName(),
2354	mlir::TypeAttr::get(recty));
2355	if (!parser.parseOptionalLParen()) {
2356	llvm::SmallVector<mlir::OpAsmParser::UnresolvedOperand> operands;
2357	llvm::SmallVector<mlir::Type> types;
2358	auto loc = parser.getNameLoc();
2359	if (parser.parseOperandList(result&: operands, delimiter: mlir::OpAsmParser::Delimiter::None) ||
2360	parser.parseColonTypeList(result&: types) || parser.parseRParen() ||
2361	parser.resolveOperands(operands, types, loc, result&: result.operands))
2362	return mlir::failure();
2363	}
2364	mlir::Type fieldType = TY::get(builder.getContext());
2365	if (parser.addTypeToList(type: fieldType, result&: result.types))
2366	return mlir::failure();
2367	return mlir::success();
2368	}
2369
2370	mlir::ParseResult fir::FieldIndexOp::parse(mlir::OpAsmParser &parser,
2371	mlir::OperationState &result) {
2372	return parseFieldLikeOp<fir::FieldType>(parser, result);
2373	}
2374
2375	template <typename OP>
2376	void printFieldLikeOp(mlir::OpAsmPrinter &p, OP &op) {
2377	p << ' '
2378	<< op.getOperation()
2379	->template getAttrOfType<mlir::StringAttr>(
2380	fir::FieldIndexOp::getFieldAttrName())
2381	.getValue()
2382	<< ", " << op.getOperation()->getAttr(fir::FieldIndexOp::getTypeAttrName());
2383	if (op.getNumOperands()) {
2384	p << '(';
2385	p.printOperands(op.getTypeparams());
2386	auto sep = ") : ";
2387	for (auto op : op.getTypeparams()) {
2388	p << sep;
2389	if (op)
2390	p.printType(type: op.getType());
2391	else
2392	p << "()";
2393	sep = ", ";
2394	}
2395	}
2396	}
2397
2398	void fir::FieldIndexOp::print(mlir::OpAsmPrinter &p) {
2399	printFieldLikeOp(p, *this);
2400	}
2401
2402	void fir::FieldIndexOp::build(mlir::OpBuilder &builder,
2403	mlir::OperationState &result,
2404	llvm::StringRef fieldName, mlir::Type recTy,
2405	mlir::ValueRange operands) {
2406	result.addAttribute(getFieldAttrName(), builder.getStringAttr(fieldName));
2407	result.addAttribute(getTypeAttrName(), mlir::TypeAttr::get(recTy));
2408	result.addOperands(operands);
2409	}
2410
2411	llvm::SmallVector<mlir::Attribute> fir::FieldIndexOp::getAttributes() {
2412	llvm::SmallVector<mlir::Attribute> attrs;
2413	attrs.push_back(getFieldIdAttr());
2414	attrs.push_back(getOnTypeAttr());
2415	return attrs;
2416	}
2417
2418	//===----------------------------------------------------------------------===//
2419	// InsertOnRangeOp
2420	//===----------------------------------------------------------------------===//
2421
2422	static mlir::ParseResult
2423	parseCustomRangeSubscript(mlir::OpAsmParser &parser,
2424	mlir::DenseIntElementsAttr &coord) {
2425	llvm::SmallVector<std::int64_t> lbounds;
2426	llvm::SmallVector<std::int64_t> ubounds;
2427	if (parser.parseKeyword(keyword: "from") ||
2428	parser.parseCommaSeparatedList(
2429	delimiter: mlir::AsmParser::Delimiter::Paren,
2430	parseElementFn: [&] { return parser.parseInteger(result&: lbounds.emplace_back(Args: 0)); }) ||
2431	parser.parseKeyword(keyword: "to") ||
2432	parser.parseCommaSeparatedList(delimiter: mlir::AsmParser::Delimiter::Paren, parseElementFn: [&] {
2433	return parser.parseInteger(result&: ubounds.emplace_back(Args: 0));
2434	}))
2435	return mlir::failure();
2436	llvm::SmallVector<std::int64_t> zippedBounds;
2437	for (auto zip : llvm::zip(t&: lbounds, u&: ubounds)) {
2438	zippedBounds.push_back(Elt: std::get<0>(t&: zip));
2439	zippedBounds.push_back(Elt: std::get<1>(t&: zip));
2440	}
2441	coord = mlir::Builder(parser.getContext()).getIndexTensorAttr(values: zippedBounds);
2442	return mlir::success();
2443	}
2444
2445	static void printCustomRangeSubscript(mlir::OpAsmPrinter &printer,
2446	fir::InsertOnRangeOp op,
2447	mlir::DenseIntElementsAttr coord) {
2448	printer << "from (";
2449	auto enumerate = llvm::enumerate(First: coord.getValues<std::int64_t>());
2450	// Even entries are the lower bounds.
2451	llvm::interleaveComma(
2452	c: make_filter_range(
2453	Range&: enumerate,
2454	Pred: [](auto indexed_value) { return indexed_value.index() % 2 == 0; }),
2455	os&: printer, each_fn: [&](auto indexed_value) { printer << indexed_value.value(); });
2456	printer << ") to (";
2457	// Odd entries are the upper bounds.
2458	llvm::interleaveComma(
2459	c: make_filter_range(
2460	Range&: enumerate,
2461	Pred: [](auto indexed_value) { return indexed_value.index() % 2 != 0; }),
2462	os&: printer, each_fn: [&](auto indexed_value) { printer << indexed_value.value(); });
2463	printer << ")";
2464	}
2465
2466	/// Range bounds must be nonnegative, and the range must not be empty.
2467	llvm::LogicalResult fir::InsertOnRangeOp::verify() {
2468	if (fir::hasDynamicSize(getSeq().getType()))
2469	return emitOpError("must have constant shape and size");
2470	mlir::DenseIntElementsAttr coorAttr = getCoor();
2471	if (coorAttr.size() < 2 || coorAttr.size() % 2 != 0)
2472	return emitOpError("has uneven number of values in ranges");
2473	bool rangeIsKnownToBeNonempty = false;
2474	for (auto i = coorAttr.getValues<std::int64_t>().end(),
2475	b = coorAttr.getValues<std::int64_t>().begin();
2476	i != b;) {
2477	int64_t ub = (*--i);
2478	int64_t lb = (*--i);
2479	if (lb < 0 || ub < 0)
2480	return emitOpError("negative range bound");
2481	if (rangeIsKnownToBeNonempty)
2482	continue;
2483	if (lb > ub)
2484	return emitOpError("empty range");
2485	rangeIsKnownToBeNonempty = lb < ub;
2486	}
2487	return mlir::success();
2488	}
2489
2490	bool fir::InsertOnRangeOp::isFullRange() {
2491	auto extents = getType().getShape();
2492	mlir::DenseIntElementsAttr indexes = getCoor();
2493	if (indexes.size() / 2 != static_cast<int64_t>(extents.size()))
2494	return false;
2495	auto cur_index = indexes.value_begin<int64_t>();
2496	for (unsigned i = 0; i < indexes.size(); i += 2) {
2497	if (*(cur_index++) != 0)
2498	return false;
2499	if (*(cur_index++) != extents[i / 2] - 1)
2500	return false;
2501	}
2502	return true;
2503	}
2504
2505	//===----------------------------------------------------------------------===//
2506	// InsertValueOp
2507	//===----------------------------------------------------------------------===//
2508
2509	static bool checkIsIntegerConstant(mlir::Attribute attr, std::int64_t conVal) {
2510	if (auto iattr = mlir::dyn_cast<mlir::IntegerAttr>(Val&: attr))
2511	return iattr.getInt() == conVal;
2512	return false;
2513	}
2514
2515	static bool isZero(mlir::Attribute a) { return checkIsIntegerConstant(attr: a, conVal: 0); }
2516	static bool isOne(mlir::Attribute a) { return checkIsIntegerConstant(attr: a, conVal: 1); }
2517
2518	// Undo some complex patterns created in the front-end and turn them back into
2519	// complex ops.
2520	template <typename FltOp, typename CpxOp>
2521	struct UndoComplexPattern : public mlir::RewritePattern {
2522	UndoComplexPattern(mlir::MLIRContext *ctx)
2523	: mlir::RewritePattern("fir.insert_value", 2, ctx) {}
2524
2525	llvm::LogicalResult
2526	matchAndRewrite(mlir::Operation *op,
2527	mlir::PatternRewriter &rewriter) const override {
2528	auto insval = mlir::dyn_cast_or_null<fir::InsertValueOp>(op);
2529	if (!insval || !mlir::isa<mlir::ComplexType>(insval.getType()))
2530	return mlir::failure();
2531	auto insval2 = mlir::dyn_cast_or_null<fir::InsertValueOp>(
2532	insval.getAdt().getDefiningOp());
2533	if (!insval2)
2534	return mlir::failure();
2535	auto binf = mlir::dyn_cast_or_null<FltOp>(insval.getVal().getDefiningOp());
2536	auto binf2 =
2537	mlir::dyn_cast_or_null<FltOp>(insval2.getVal().getDefiningOp());
2538	if (!binf || !binf2 || insval.getCoor().size() != 1 ||
2539	!isOne(insval.getCoor()[0]) || insval2.getCoor().size() != 1 ||
2540	!isZero(insval2.getCoor()[0]))
2541	return mlir::failure();
2542	auto eai = mlir::dyn_cast_or_null<fir::ExtractValueOp>(
2543	binf.getLhs().getDefiningOp());
2544	auto ebi = mlir::dyn_cast_or_null<fir::ExtractValueOp>(
2545	binf.getRhs().getDefiningOp());
2546	auto ear = mlir::dyn_cast_or_null<fir::ExtractValueOp>(
2547	binf2.getLhs().getDefiningOp());
2548	auto ebr = mlir::dyn_cast_or_null<fir::ExtractValueOp>(
2549	binf2.getRhs().getDefiningOp());
2550	if (!eai || !ebi || !ear || !ebr || ear.getAdt() != eai.getAdt() ||
2551	ebr.getAdt() != ebi.getAdt() || eai.getCoor().size() != 1 ||
2552	!isOne(eai.getCoor()[0]) || ebi.getCoor().size() != 1 ||
2553	!isOne(ebi.getCoor()[0]) || ear.getCoor().size() != 1 ||
2554	!isZero(ear.getCoor()[0]) || ebr.getCoor().size() != 1 ||
2555	!isZero(ebr.getCoor()[0]))
2556	return mlir::failure();
2557	rewriter.replaceOpWithNewOp<CpxOp>(op, ear.getAdt(), ebr.getAdt());
2558	return mlir::success();
2559	}
2560	};
2561
2562	void fir::InsertValueOp::getCanonicalizationPatterns(
2563	mlir::RewritePatternSet &results, mlir::MLIRContext *context) {
2564	results.insert<UndoComplexPattern<mlir::arith::AddFOp, fir::AddcOp>,
2565	UndoComplexPattern<mlir::arith::SubFOp, fir::SubcOp>>(context);
2566	}
2567
2568	//===----------------------------------------------------------------------===//
2569	// IterWhileOp
2570	//===----------------------------------------------------------------------===//
2571
2572	void fir::IterWhileOp::build(mlir::OpBuilder &builder,
2573	mlir::OperationState &result, mlir::Value lb,
2574	mlir::Value ub, mlir::Value step,
2575	mlir::Value iterate, bool finalCountValue,
2576	mlir::ValueRange iterArgs,
2577	llvm::ArrayRef<mlir::NamedAttribute> attributes) {
2578	result.addOperands({lb, ub, step, iterate});
2579	if (finalCountValue) {
2580	result.addTypes(builder.getIndexType());
2581	result.addAttribute(getFinalValueAttrNameStr(), builder.getUnitAttr());
2582	}
2583	result.addTypes(iterate.getType());
2584	result.addOperands(iterArgs);
2585	for (auto v : iterArgs)
2586	result.addTypes(v.getType());
2587	mlir::Region *bodyRegion = result.addRegion();
2588	bodyRegion->push_back(new mlir::Block{});
2589	bodyRegion->front().addArgument(builder.getIndexType(), result.location);
2590	bodyRegion->front().addArgument(iterate.getType(), result.location);
2591	bodyRegion->front().addArguments(
2592	iterArgs.getTypes(),
2593	llvm::SmallVector<mlir::Location>(iterArgs.size(), result.location));
2594	result.addAttributes(attributes);
2595	}
2596
2597	mlir::ParseResult fir::IterWhileOp::parse(mlir::OpAsmParser &parser,
2598	mlir::OperationState &result) {
2599	auto &builder = parser.getBuilder();
2600	mlir::OpAsmParser::Argument inductionVariable, iterateVar;
2601	mlir::OpAsmParser::UnresolvedOperand lb, ub, step, iterateInput;
2602	if (parser.parseLParen() || parser.parseArgument(inductionVariable) ||
2603	parser.parseEqual())
2604	return mlir::failure();
2605
2606	// Parse loop bounds.
2607	auto indexType = builder.getIndexType();
2608	auto i1Type = builder.getIntegerType(1);
2609	if (parser.parseOperand(lb) ||
2610	parser.resolveOperand(lb, indexType, result.operands) ||
2611	parser.parseKeyword("to") || parser.parseOperand(ub) ||
2612	parser.resolveOperand(ub, indexType, result.operands) ||
2613	parser.parseKeyword("step") || parser.parseOperand(step) ||
2614	parser.parseRParen() ||
2615	parser.resolveOperand(step, indexType, result.operands) ||
2616	parser.parseKeyword("and") || parser.parseLParen() ||
2617	parser.parseArgument(iterateVar) || parser.parseEqual() ||
2618	parser.parseOperand(iterateInput) || parser.parseRParen() ||
2619	parser.resolveOperand(iterateInput, i1Type, result.operands))
2620	return mlir::failure();
2621
2622	// Parse the initial iteration arguments.
2623	auto prependCount = false;
2624
2625	// Induction variable.
2626	llvm::SmallVector<mlir::OpAsmParser::Argument> regionArgs;
2627	regionArgs.push_back(inductionVariable);
2628	regionArgs.push_back(iterateVar);
2629
2630	if (succeeded(parser.parseOptionalKeyword("iter_args"))) {
2631	llvm::SmallVector<mlir::OpAsmParser::UnresolvedOperand> operands;
2632	llvm::SmallVector<mlir::Type> regionTypes;
2633	// Parse assignment list and results type list.
2634	if (parser.parseAssignmentList(regionArgs, operands) ||
2635	parser.parseArrowTypeList(regionTypes))
2636	return mlir::failure();
2637	if (regionTypes.size() == operands.size() + 2)
2638	prependCount = true;
2639	llvm::ArrayRef<mlir::Type> resTypes = regionTypes;
2640	resTypes = prependCount ? resTypes.drop_front(2) : resTypes;
2641	// Resolve input operands.
2642	for (auto operandType : llvm::zip(operands, resTypes))
2643	if (parser.resolveOperand(std::get<0>(operandType),
2644	std::get<1>(operandType), result.operands))
2645	return mlir::failure();
2646	if (prependCount) {
2647	result.addTypes(regionTypes);
2648	} else {
2649	result.addTypes(i1Type);
2650	result.addTypes(resTypes);
2651	}
2652	} else if (succeeded(parser.parseOptionalArrow())) {
2653	llvm::SmallVector<mlir::Type> typeList;
2654	if (parser.parseLParen() || parser.parseTypeList(typeList) ||
2655	parser.parseRParen())
2656	return mlir::failure();
2657	// Type list must be "(index, i1)".
2658	if (typeList.size() != 2 || !mlir::isa<mlir::IndexType>(typeList[0]) ||
2659	!typeList[1].isSignlessInteger(1))
2660	return mlir::failure();
2661	result.addTypes(typeList);
2662	prependCount = true;
2663	} else {
2664	result.addTypes(i1Type);
2665	}
2666
2667	if (parser.parseOptionalAttrDictWithKeyword(result.attributes))
2668	return mlir::failure();
2669
2670	llvm::SmallVector<mlir::Type> argTypes;
2671	// Induction variable (hidden)
2672	if (prependCount)
2673	result.addAttribute(IterWhileOp::getFinalValueAttrNameStr(),
2674	builder.getUnitAttr());
2675	else
2676	argTypes.push_back(indexType);
2677	// Loop carried variables (including iterate)
2678	argTypes.append(result.types.begin(), result.types.end());
2679	// Parse the body region.
2680	auto *body = result.addRegion();
2681	if (regionArgs.size() != argTypes.size())
2682	return parser.emitError(
2683	parser.getNameLoc(),
2684	"mismatch in number of loop-carried values and defined values");
2685
2686	for (size_t i = 0, e = regionArgs.size(); i != e; ++i)
2687	regionArgs[i].type = argTypes[i];
2688
2689	if (parser.parseRegion(*body, regionArgs))
2690	return mlir::failure();
2691
2692	fir::IterWhileOp::ensureTerminator(*body, builder, result.location);
2693	return mlir::success();
2694	}
2695
2696	llvm::LogicalResult fir::IterWhileOp::verify() {
2697	// Check that the body defines as single block argument for the induction
2698	// variable.
2699	auto *body = getBody();
2700	if (!body->getArgument(1).getType().isInteger(1))
2701	return emitOpError(
2702	"expected body second argument to be an index argument for "
2703	"the induction variable");
2704	if (!body->getArgument(0).getType().isIndex())
2705	return emitOpError(
2706	"expected body first argument to be an index argument for "
2707	"the induction variable");
2708
2709	auto opNumResults = getNumResults();
2710	if (getFinalValue()) {
2711	// Result type must be "(index, i1, ...)".
2712	if (!mlir::isa<mlir::IndexType>(getResult(0).getType()))
2713	return emitOpError("result #0 expected to be index");
2714	if (!getResult(1).getType().isSignlessInteger(1))
2715	return emitOpError("result #1 expected to be i1");
2716	opNumResults--;
2717	} else {
2718	// iterate_while always returns the early exit induction value.
2719	// Result type must be "(i1, ...)"
2720	if (!getResult(0).getType().isSignlessInteger(1))
2721	return emitOpError("result #0 expected to be i1");
2722	}
2723	if (opNumResults == 0)
2724	return mlir::failure();
2725	if (getNumIterOperands() != opNumResults)
2726	return emitOpError(
2727	"mismatch in number of loop-carried values and defined values");
2728	if (getNumRegionIterArgs() != opNumResults)
2729	return emitOpError(
2730	"mismatch in number of basic block args and defined values");
2731	auto iterOperands = getIterOperands();
2732	auto iterArgs = getRegionIterArgs();
2733	auto opResults = getFinalValue() ? getResults().drop_front() : getResults();
2734	unsigned i = 0u;
2735	for (auto e : llvm::zip(iterOperands, iterArgs, opResults)) {
2736	if (std::get<0>(e).getType() != std::get<2>(e).getType())
2737	return emitOpError() << "types mismatch between " << i
2738	<< "th iter operand and defined value";
2739	if (std::get<1>(e).getType() != std::get<2>(e).getType())
2740	return emitOpError() << "types mismatch between " << i
2741	<< "th iter region arg and defined value";
2742
2743	i++;
2744	}
2745	return mlir::success();
2746	}
2747
2748	void fir::IterWhileOp::print(mlir::OpAsmPrinter &p) {
2749	p << " (" << getInductionVar() << " = " << getLowerBound() << " to "
2750	<< getUpperBound() << " step " << getStep() << ") and (";
2751	assert(hasIterOperands());
2752	auto regionArgs = getRegionIterArgs();
2753	auto operands = getIterOperands();
2754	p << regionArgs.front() << " = " << *operands.begin() << ")";
2755	if (regionArgs.size() > 1) {
2756	p << " iter_args(";
2757	llvm::interleaveComma(
2758	llvm::zip(regionArgs.drop_front(), operands.drop_front()), p,
2759	[&](auto it) { p << std::get<0>(it) << " = " << std::get<1>(it); });
2760	p << ") -> (";
2761	llvm::interleaveComma(
2762	llvm::drop_begin(getResultTypes(), getFinalValue() ? 0 : 1), p);
2763	p << ")";
2764	} else if (getFinalValue()) {
2765	p << " -> (" << getResultTypes() << ')';
2766	}
2767	p.printOptionalAttrDictWithKeyword((*this)->getAttrs(),
2768	{getFinalValueAttrNameStr()});
2769	p << ' ';
2770	p.printRegion(getRegion(), /*printEntryBlockArgs=*/false,
2771	/*printBlockTerminators=*/true);
2772	}
2773
2774	llvm::SmallVector<mlir::Region *> fir::IterWhileOp::getLoopRegions() {
2775	return {&getRegion()};
2776	}
2777
2778	mlir::BlockArgument fir::IterWhileOp::iterArgToBlockArg(mlir::Value iterArg) {
2779	for (auto i : llvm::enumerate(getInitArgs()))
2780	if (iterArg == i.value())
2781	return getRegion().front().getArgument(i.index() + 1);
2782	return {};
2783	}
2784
2785	void fir::IterWhileOp::resultToSourceOps(
2786	llvm::SmallVectorImpl<mlir::Value> &results, unsigned resultNum) {
2787	auto oper = getFinalValue() ? resultNum + 1 : resultNum;
2788	auto *term = getRegion().front().getTerminator();
2789	if (oper < term->getNumOperands())
2790	results.push_back(term->getOperand(oper));
2791	}
2792
2793	mlir::Value fir::IterWhileOp::blockArgToSourceOp(unsigned blockArgNum) {
2794	if (blockArgNum > 0 && blockArgNum <= getInitArgs().size())
2795	return getInitArgs()[blockArgNum - 1];
2796	return {};
2797	}
2798
2799	std::optional<llvm::MutableArrayRef<mlir::OpOperand>>
2800	fir::IterWhileOp::getYieldedValuesMutable() {
2801	auto *term = getRegion().front().getTerminator();
2802	return getFinalValue() ? term->getOpOperands().drop_front()
2803	: term->getOpOperands();
2804	}
2805
2806	//===----------------------------------------------------------------------===//
2807	// LenParamIndexOp
2808	//===----------------------------------------------------------------------===//
2809
2810	mlir::ParseResult fir::LenParamIndexOp::parse(mlir::OpAsmParser &parser,
2811	mlir::OperationState &result) {
2812	return parseFieldLikeOp<fir::LenType>(parser, result);
2813	}
2814
2815	void fir::LenParamIndexOp::print(mlir::OpAsmPrinter &p) {
2816	printFieldLikeOp(p, *this);
2817	}
2818
2819	void fir::LenParamIndexOp::build(mlir::OpBuilder &builder,
2820	mlir::OperationState &result,
2821	llvm::StringRef fieldName, mlir::Type recTy,
2822	mlir::ValueRange operands) {
2823	result.addAttribute(getFieldAttrName(), builder.getStringAttr(fieldName));
2824	result.addAttribute(getTypeAttrName(), mlir::TypeAttr::get(recTy));
2825	result.addOperands(operands);
2826	}
2827
2828	llvm::SmallVector<mlir::Attribute> fir::LenParamIndexOp::getAttributes() {
2829	llvm::SmallVector<mlir::Attribute> attrs;
2830	attrs.push_back(getFieldIdAttr());
2831	attrs.push_back(getOnTypeAttr());
2832	return attrs;
2833	}
2834
2835	//===----------------------------------------------------------------------===//
2836	// LoadOp
2837	//===----------------------------------------------------------------------===//
2838
2839	void fir::LoadOp::build(mlir::OpBuilder &builder, mlir::OperationState &result,
2840	mlir::Value refVal) {
2841	if (!refVal) {
2842	mlir::emitError(result.location, "LoadOp has null argument");
2843	return;
2844	}
2845	auto eleTy = fir::dyn_cast_ptrEleTy(refVal.getType());
2846	if (!eleTy) {
2847	mlir::emitError(result.location, "not a memory reference type");
2848	return;
2849	}
2850	build(builder, result, eleTy, refVal);
2851	}
2852
2853	void fir::LoadOp::build(mlir::OpBuilder &builder, mlir::OperationState &result,
2854	mlir::Type resTy, mlir::Value refVal) {
2855
2856	if (!refVal) {
2857	mlir::emitError(result.location, "LoadOp has null argument");
2858	return;
2859	}
2860	result.addOperands(refVal);
2861	result.addTypes(resTy);
2862	}
2863
2864	mlir::ParseResult fir::LoadOp::getElementOf(mlir::Type &ele, mlir::Type ref) {
2865	if ((ele = fir::dyn_cast_ptrEleTy(ref)))
2866	return mlir::success();
2867	return mlir::failure();
2868	}
2869
2870	mlir::ParseResult fir::LoadOp::parse(mlir::OpAsmParser &parser,
2871	mlir::OperationState &result) {
2872	mlir::Type type;
2873	mlir::OpAsmParser::UnresolvedOperand oper;
2874	if (parser.parseOperand(oper) ||
2875	parser.parseOptionalAttrDict(result.attributes) ||
2876	parser.parseColonType(type) ||
2877	parser.resolveOperand(oper, type, result.operands))
2878	return mlir::failure();
2879	mlir::Type eleTy;
2880	if (fir::LoadOp::getElementOf(eleTy, type) ||
2881	parser.addTypeToList(eleTy, result.types))
2882	return mlir::failure();
2883	return mlir::success();
2884	}
2885
2886	void fir::LoadOp::print(mlir::OpAsmPrinter &p) {
2887	p << ' ';
2888	p.printOperand(getMemref());
2889	p.printOptionalAttrDict(getOperation()->getAttrs(), {});
2890	p << " : " << getMemref().getType();
2891	}
2892
2893	void fir::LoadOp::getEffects(
2894	llvm::SmallVectorImpl<
2895	mlir::SideEffects::EffectInstance<mlir::MemoryEffects::Effect>>
2896	&effects) {
2897	effects.emplace_back(mlir::MemoryEffects::Read::get(), &getMemrefMutable(),
2898	mlir::SideEffects::DefaultResource::get());
2899	addVolatileMemoryEffects({getMemref().getType()}, effects);
2900	}
2901
2902	//===----------------------------------------------------------------------===//
2903	// DoLoopOp
2904	//===----------------------------------------------------------------------===//
2905
2906	void fir::DoLoopOp::build(mlir::OpBuilder &builder,
2907	mlir::OperationState &result, mlir::Value lb,
2908	mlir::Value ub, mlir::Value step, bool unordered,
2909	bool finalCountValue, mlir::ValueRange iterArgs,
2910	mlir::ValueRange reduceOperands,
2911	llvm::ArrayRef<mlir::Attribute> reduceAttrs,
2912	llvm::ArrayRef<mlir::NamedAttribute> attributes) {
2913	result.addOperands({lb, ub, step});
2914	result.addOperands(reduceOperands);
2915	result.addOperands(iterArgs);
2916	result.addAttribute(getOperandSegmentSizeAttr(),
2917	builder.getDenseI32ArrayAttr(
2918	{1, 1, 1, static_cast<int32_t>(reduceOperands.size()),
2919	static_cast<int32_t>(iterArgs.size())}));
2920	if (finalCountValue) {
2921	result.addTypes(builder.getIndexType());
2922	result.addAttribute(getFinalValueAttrName(result.name),
2923	builder.getUnitAttr());
2924	}
2925	for (auto v : iterArgs)
2926	result.addTypes(v.getType());
2927	mlir::Region *bodyRegion = result.addRegion();
2928	bodyRegion->push_back(new mlir::Block{});
2929	if (iterArgs.empty() && !finalCountValue)
2930	fir::DoLoopOp::ensureTerminator(*bodyRegion, builder, result.location);
2931	bodyRegion->front().addArgument(builder.getIndexType(), result.location);
2932	bodyRegion->front().addArguments(
2933	iterArgs.getTypes(),
2934	llvm::SmallVector<mlir::Location>(iterArgs.size(), result.location));
2935	if (unordered)
2936	result.addAttribute(getUnorderedAttrName(result.name),
2937	builder.getUnitAttr());
2938	if (!reduceAttrs.empty())
2939	result.addAttribute(getReduceAttrsAttrName(result.name),
2940	builder.getArrayAttr(reduceAttrs));
2941	result.addAttributes(attributes);
2942	}
2943
2944	mlir::ParseResult fir::DoLoopOp::parse(mlir::OpAsmParser &parser,
2945	mlir::OperationState &result) {
2946	auto &builder = parser.getBuilder();
2947	mlir::OpAsmParser::Argument inductionVariable;
2948	mlir::OpAsmParser::UnresolvedOperand lb, ub, step;
2949	// Parse the induction variable followed by '='.
2950	if (parser.parseArgument(inductionVariable) || parser.parseEqual())
2951	return mlir::failure();
2952
2953	// Parse loop bounds.
2954	auto indexType = builder.getIndexType();
2955	if (parser.parseOperand(lb) ||
2956	parser.resolveOperand(lb, indexType, result.operands) ||
2957	parser.parseKeyword("to") || parser.parseOperand(ub) ||
2958	parser.resolveOperand(ub, indexType, result.operands) ||
2959	parser.parseKeyword("step") || parser.parseOperand(step) ||
2960	parser.resolveOperand(step, indexType, result.operands))
2961	return mlir::failure();
2962
2963	if (mlir::succeeded(parser.parseOptionalKeyword("unordered")))
2964	result.addAttribute("unordered", builder.getUnitAttr());
2965
2966	// Parse the reduction arguments.
2967	llvm::SmallVector<mlir::OpAsmParser::UnresolvedOperand> reduceOperands;
2968	llvm::SmallVector<mlir::Type> reduceArgTypes;
2969	if (succeeded(parser.parseOptionalKeyword("reduce"))) {
2970	// Parse reduction attributes and variables.
2971	llvm::SmallVector<ReduceAttr> attributes;
2972	if (failed(parser.parseCommaSeparatedList(
2973	mlir::AsmParser::Delimiter::Paren, [&]() {
2974	if (parser.parseAttribute(attributes.emplace_back()) ||
2975	parser.parseArrow() ||
2976	parser.parseOperand(reduceOperands.emplace_back()) ||
2977	parser.parseColonType(reduceArgTypes.emplace_back()))
2978	return mlir::failure();
2979	return mlir::success();
2980	})))
2981	return mlir::failure();
2982	// Resolve input operands.
2983	for (auto operand_type : llvm::zip(reduceOperands, reduceArgTypes))
2984	if (parser.resolveOperand(std::get<0>(operand_type),
2985	std::get<1>(operand_type), result.operands))
2986	return mlir::failure();
2987	llvm::SmallVector<mlir::Attribute> arrayAttr(attributes.begin(),
2988	attributes.end());
2989	result.addAttribute(getReduceAttrsAttrName(result.name),
2990	builder.getArrayAttr(arrayAttr));
2991	}
2992
2993	// Parse the optional initial iteration arguments.
2994	llvm::SmallVector<mlir::OpAsmParser::Argument> regionArgs;
2995	llvm::SmallVector<mlir::OpAsmParser::UnresolvedOperand> iterOperands;
2996	llvm::SmallVector<mlir::Type> argTypes;
2997	bool prependCount = false;
2998	regionArgs.push_back(inductionVariable);
2999
3000	if (succeeded(parser.parseOptionalKeyword("iter_args"))) {
3001	// Parse assignment list and results type list.
3002	if (parser.parseAssignmentList(regionArgs, iterOperands) ||
3003	parser.parseArrowTypeList(result.types))
3004	return mlir::failure();
3005	if (result.types.size() == iterOperands.size() + 1)
3006	prependCount = true;
3007	// Resolve input operands.
3008	llvm::ArrayRef<mlir::Type> resTypes = result.types;
3009	for (auto operand_type : llvm::zip(
3010	iterOperands, prependCount ? resTypes.drop_front() : resTypes))
3011	if (parser.resolveOperand(std::get<0>(operand_type),
3012	std::get<1>(operand_type), result.operands))
3013	return mlir::failure();
3014	} else if (succeeded(parser.parseOptionalArrow())) {
3015	if (parser.parseKeyword("index"))
3016	return mlir::failure();
3017	result.types.push_back(indexType);
3018	prependCount = true;
3019	}
3020
3021	// Set the operandSegmentSizes attribute
3022	result.addAttribute(getOperandSegmentSizeAttr(),
3023	builder.getDenseI32ArrayAttr(
3024	{1, 1, 1, static_cast<int32_t>(reduceOperands.size()),
3025	static_cast<int32_t>(iterOperands.size())}));
3026
3027	if (parser.parseOptionalAttrDictWithKeyword(result.attributes))
3028	return mlir::failure();
3029
3030	// Induction variable.
3031	if (prependCount)
3032	result.addAttribute(DoLoopOp::getFinalValueAttrName(result.name),
3033	builder.getUnitAttr());
3034	else
3035	argTypes.push_back(indexType);
3036	// Loop carried variables
3037	argTypes.append(result.types.begin(), result.types.end());
3038	// Parse the body region.
3039	auto *body = result.addRegion();
3040	if (regionArgs.size() != argTypes.size())
3041	return parser.emitError(
3042	parser.getNameLoc(),
3043	"mismatch in number of loop-carried values and defined values");
3044	for (size_t i = 0, e = regionArgs.size(); i != e; ++i)
3045	regionArgs[i].type = argTypes[i];
3046
3047	if (parser.parseRegion(*body, regionArgs))
3048	return mlir::failure();
3049
3050	DoLoopOp::ensureTerminator(*body, builder, result.location);
3051
3052	return mlir::success();
3053	}
3054
3055	fir::DoLoopOp fir::getForInductionVarOwner(mlir::Value val) {
3056	auto ivArg = mlir::dyn_cast<mlir::BlockArgument>(val);
3057	if (!ivArg)
3058	return {};
3059	assert(ivArg.getOwner() && "unlinked block argument");
3060	auto *containingInst = ivArg.getOwner()->getParentOp();
3061	return mlir::dyn_cast_or_null<fir::DoLoopOp>(containingInst);
3062	}
3063
3064	// Lifted from loop.loop
3065	llvm::LogicalResult fir::DoLoopOp::verify() {
3066	// Check that the body defines as single block argument for the induction
3067	// variable.
3068	auto *body = getBody();
3069	if (!body->getArgument(0).getType().isIndex())
3070	return emitOpError(
3071	"expected body first argument to be an index argument for "
3072	"the induction variable");
3073
3074	auto opNumResults = getNumResults();
3075	if (opNumResults == 0)
3076	return mlir::success();
3077
3078	if (getFinalValue()) {
3079	if (getUnordered())
3080	return emitOpError("unordered loop has no final value");
3081	opNumResults--;
3082	}
3083	if (getNumIterOperands() != opNumResults)
3084	return emitOpError(
3085	"mismatch in number of loop-carried values and defined values");
3086	if (getNumRegionIterArgs() != opNumResults)
3087	return emitOpError(
3088	"mismatch in number of basic block args and defined values");
3089	auto iterOperands = getIterOperands();
3090	auto iterArgs = getRegionIterArgs();
3091	auto opResults = getFinalValue() ? getResults().drop_front() : getResults();
3092	unsigned i = 0u;
3093	for (auto e : llvm::zip(iterOperands, iterArgs, opResults)) {
3094	if (std::get<0>(e).getType() != std::get<2>(e).getType())
3095	return emitOpError() << "types mismatch between " << i
3096	<< "th iter operand and defined value";
3097	if (std::get<1>(e).getType() != std::get<2>(e).getType())
3098	return emitOpError() << "types mismatch between " << i
3099	<< "th iter region arg and defined value";
3100
3101	i++;
3102	}
3103	auto reduceAttrs = getReduceAttrsAttr();
3104	if (getNumReduceOperands() != (reduceAttrs ? reduceAttrs.size() : 0))
3105	return emitOpError(
3106	"mismatch in number of reduction variables and reduction attributes");
3107	return mlir::success();
3108	}
3109
3110	void fir::DoLoopOp::print(mlir::OpAsmPrinter &p) {
3111	bool printBlockTerminators = false;
3112	p << ' ' << getInductionVar() << " = " << getLowerBound() << " to "
3113	<< getUpperBound() << " step " << getStep();
3114	if (getUnordered())
3115	p << " unordered";
3116	if (hasReduceOperands()) {
3117	p << " reduce(";
3118	auto attrs = getReduceAttrsAttr();
3119	auto operands = getReduceOperands();
3120	llvm::interleaveComma(llvm::zip(attrs, operands), p, [&](auto it) {
3121	p << std::get<0>(it) << " -> " << std::get<1>(it) << " : "
3122	<< std::get<1>(it).getType();
3123	});
3124	p << ')';
3125	printBlockTerminators = true;
3126	}
3127	if (hasIterOperands()) {
3128	p << " iter_args(";
3129	auto regionArgs = getRegionIterArgs();
3130	auto operands = getIterOperands();
3131	llvm::interleaveComma(llvm::zip(regionArgs, operands), p, [&](auto it) {
3132	p << std::get<0>(it) << " = " << std::get<1>(it);
3133	});
3134	p << ") -> (" << getResultTypes() << ')';
3135	printBlockTerminators = true;
3136	} else if (getFinalValue()) {
3137	p << " -> " << getResultTypes();
3138	printBlockTerminators = true;
3139	}
3140	p.printOptionalAttrDictWithKeyword(
3141	(*this)->getAttrs(),
3142	{"unordered", "finalValue", "reduceAttrs", "operandSegmentSizes"});
3143	p << ' ';
3144	p.printRegion(getRegion(), /*printEntryBlockArgs=*/false,
3145	printBlockTerminators);
3146	}
3147
3148	llvm::SmallVector<mlir::Region *> fir::DoLoopOp::getLoopRegions() {
3149	return {&getRegion()};
3150	}
3151
3152	/// Translate a value passed as an iter_arg to the corresponding block
3153	/// argument in the body of the loop.
3154	mlir::BlockArgument fir::DoLoopOp::iterArgToBlockArg(mlir::Value iterArg) {
3155	for (auto i : llvm::enumerate(getInitArgs()))
3156	if (iterArg == i.value())
3157	return getRegion().front().getArgument(i.index() + 1);
3158	return {};
3159	}
3160
3161	/// Translate the result vector (by index number) to the corresponding value
3162	/// to the `fir.result` Op.
3163	void fir::DoLoopOp::resultToSourceOps(
3164	llvm::SmallVectorImpl<mlir::Value> &results, unsigned resultNum) {
3165	auto oper = getFinalValue() ? resultNum + 1 : resultNum;
3166	auto *term = getRegion().front().getTerminator();
3167	if (oper < term->getNumOperands())
3168	results.push_back(term->getOperand(oper));
3169	}
3170
3171	/// Translate the block argument (by index number) to the corresponding value
3172	/// passed as an iter_arg to the parent DoLoopOp.
3173	mlir::Value fir::DoLoopOp::blockArgToSourceOp(unsigned blockArgNum) {
3174	if (blockArgNum > 0 && blockArgNum <= getInitArgs().size())
3175	return getInitArgs()[blockArgNum - 1];
3176	return {};
3177	}
3178
3179	std::optional<llvm::MutableArrayRef<mlir::OpOperand>>
3180	fir::DoLoopOp::getYieldedValuesMutable() {
3181	auto *term = getRegion().front().getTerminator();
3182	return getFinalValue() ? term->getOpOperands().drop_front()
3183	: term->getOpOperands();
3184	}
3185
3186	//===----------------------------------------------------------------------===//
3187	// DTEntryOp
3188	//===----------------------------------------------------------------------===//
3189
3190	mlir::ParseResult fir::DTEntryOp::parse(mlir::OpAsmParser &parser,
3191	mlir::OperationState &result) {
3192	llvm::StringRef methodName;
3193	// allow `methodName` or `"methodName"`
3194	if (failed(parser.parseOptionalKeyword(&methodName))) {
3195	mlir::StringAttr methodAttr;
3196	if (parser.parseAttribute(methodAttr, getMethodAttrName(result.name),
3197	result.attributes))
3198	return mlir::failure();
3199	} else {
3200	result.addAttribute(getMethodAttrName(result.name),
3201	parser.getBuilder().getStringAttr(methodName));
3202	}
3203	mlir::SymbolRefAttr calleeAttr;
3204	if (parser.parseComma() ||
3205	parser.parseAttribute(calleeAttr, fir::DTEntryOp::getProcAttrNameStr(),
3206	result.attributes))
3207	return mlir::failure();
3208	return mlir::success();
3209	}
3210
3211	void fir::DTEntryOp::print(mlir::OpAsmPrinter &p) {
3212	p << ' ' << getMethodAttr() << ", " << getProcAttr();
3213	}
3214
3215	//===----------------------------------------------------------------------===//
3216	// ReboxOp
3217	//===----------------------------------------------------------------------===//
3218
3219	/// Get the scalar type related to a fir.box type.
3220	/// Example: return f32 for !fir.box<!fir.heap<!fir.array<?x?xf32>>.
3221	static mlir::Type getBoxScalarEleTy(mlir::Type boxTy) {
3222	auto eleTy = fir::dyn_cast_ptrOrBoxEleTy(boxTy);
3223	if (auto seqTy = mlir::dyn_cast<fir::SequenceType>(eleTy))
3224	return seqTy.getEleTy();
3225	return eleTy;
3226	}
3227
3228	/// Test if \p t1 and \p t2 are compatible character types (if they can
3229	/// represent the same type at runtime).
3230	static bool areCompatibleCharacterTypes(mlir::Type t1, mlir::Type t2) {
3231	auto c1 = mlir::dyn_cast<fir::CharacterType>(t1);
3232	auto c2 = mlir::dyn_cast<fir::CharacterType>(t2);
3233	if (!c1 || !c2)
3234	return false;
3235	if (c1.hasDynamicLen() || c2.hasDynamicLen())
3236	return true;
3237	return c1.getLen() == c2.getLen();
3238	}
3239
3240	llvm::LogicalResult fir::ReboxOp::verify() {
3241	auto inputBoxTy = getBox().getType();
3242	if (fir::isa_unknown_size_box(inputBoxTy))
3243	return emitOpError("box operand must not have unknown rank or type");
3244	auto outBoxTy = getType();
3245	if (fir::isa_unknown_size_box(outBoxTy))
3246	return emitOpError("result type must not have unknown rank or type");
3247	auto inputRank = fir::getBoxRank(inputBoxTy);
3248	auto inputEleTy = getBoxScalarEleTy(inputBoxTy);
3249	auto outRank = fir::getBoxRank(outBoxTy);
3250	auto outEleTy = getBoxScalarEleTy(outBoxTy);
3251
3252	if (auto sliceVal = getSlice()) {
3253	// Slicing case
3254	if (mlir::cast<fir::SliceType>(sliceVal.getType()).getRank() != inputRank)
3255	return emitOpError("slice operand rank must match box operand rank");
3256	if (auto shapeVal = getShape()) {
3257	if (auto shiftTy = mlir::dyn_cast<fir::ShiftType>(shapeVal.getType())) {
3258	if (shiftTy.getRank() != inputRank)
3259	return emitOpError("shape operand and input box ranks must match "
3260	"when there is a slice");
3261	} else {
3262	return emitOpError("shape operand must absent or be a fir.shift "
3263	"when there is a slice");
3264	}
3265	}
3266	if (auto sliceOp = sliceVal.getDefiningOp()) {
3267	auto slicedRank = mlir::cast<fir::SliceOp>(sliceOp).getOutRank();
3268	if (slicedRank != outRank)
3269	return emitOpError("result type rank and rank after applying slice "
3270	"operand must match");
3271	}
3272	} else {
3273	// Reshaping case
3274	unsigned shapeRank = inputRank;
3275	if (auto shapeVal = getShape()) {
3276	auto ty = shapeVal.getType();
3277	if (auto shapeTy = mlir::dyn_cast<fir::ShapeType>(ty)) {
3278	shapeRank = shapeTy.getRank();
3279	} else if (auto shapeShiftTy = mlir::dyn_cast<fir::ShapeShiftType>(ty)) {
3280	shapeRank = shapeShiftTy.getRank();
3281	} else {
3282	auto shiftTy = mlir::cast<fir::ShiftType>(ty);
3283	shapeRank = shiftTy.getRank();
3284	if (shapeRank != inputRank)
3285	return emitOpError("shape operand and input box ranks must match "
3286	"when the shape is a fir.shift");
3287	}
3288	}
3289	if (shapeRank != outRank)
3290	return emitOpError("result type and shape operand ranks must match");
3291	}
3292
3293	if (inputEleTy != outEleTy) {
3294	// TODO: check that outBoxTy is a parent type of inputBoxTy for derived
3295	// types.
3296	// Character input and output types with constant length may be different if
3297	// there is a substring in the slice, otherwise, they must match. If any of
3298	// the types is a character with dynamic length, the other type can be any
3299	// character type.
3300	const bool typeCanMismatch =
3301	mlir::isa<fir::RecordType>(inputEleTy) ||
3302	mlir::isa<mlir::NoneType>(outEleTy) ||
3303	(mlir::isa<mlir::NoneType>(inputEleTy) &&
3304	mlir::isa<fir::RecordType>(outEleTy)) ||
3305	(getSlice() && mlir::isa<fir::CharacterType>(inputEleTy)) ||
3306	(getSlice() && fir::isa_complex(inputEleTy) &&
3307	mlir::isa<mlir::FloatType>(outEleTy)) ||
3308	areCompatibleCharacterTypes(inputEleTy, outEleTy);
3309	if (!typeCanMismatch)
3310	return emitOpError(
3311	"op input and output element types must match for intrinsic types");
3312	}
3313	return mlir::success();
3314	}
3315
3316	//===----------------------------------------------------------------------===//
3317	// ReboxAssumedRankOp
3318	//===----------------------------------------------------------------------===//
3319
3320	static bool areCompatibleAssumedRankElementType(mlir::Type inputEleTy,
3321	mlir::Type outEleTy) {
3322	if (inputEleTy == outEleTy)
3323	return true;
3324	// Output is unlimited polymorphic -> output dynamic type is the same as input
3325	// type.
3326	if (mlir::isa<mlir::NoneType>(Val: outEleTy))
3327	return true;
3328	// Output/Input are derived types. Assuming input extends output type, output
3329	// dynamic type is the output static type, unless output is polymorphic.
3330	if (mlir::isa<fir::RecordType>(inputEleTy) &&
3331	mlir::isa<fir::RecordType>(outEleTy))
3332	return true;
3333	if (areCompatibleCharacterTypes(t1: inputEleTy, t2: outEleTy))
3334	return true;
3335	return false;
3336	}
3337
3338	llvm::LogicalResult fir::ReboxAssumedRankOp::verify() {
3339	mlir::Type inputType = getBox().getType();
3340	if (!mlir::isa<fir::BaseBoxType>(inputType) && !fir::isBoxAddress(inputType))
3341	return emitOpError("input must be a box or box address");
3342	mlir::Type inputEleTy =
3343	mlir::cast<fir::BaseBoxType>(fir::unwrapRefType(inputType))
3344	.unwrapInnerType();
3345	mlir::Type outEleTy =
3346	mlir::cast<fir::BaseBoxType>(getType()).unwrapInnerType();
3347	if (!areCompatibleAssumedRankElementType(inputEleTy, outEleTy))
3348	return emitOpError("input and output element types are incompatible");
3349	return mlir::success();
3350	}
3351
3352	void fir::ReboxAssumedRankOp::getEffects(
3353	llvm::SmallVectorImpl<
3354	mlir::SideEffects::EffectInstance<mlir::MemoryEffects::Effect>>
3355	&effects) {
3356	mlir::OpOperand &inputBox = getBoxMutable();
3357	if (fir::isBoxAddress(inputBox.get().getType()))
3358	effects.emplace_back(mlir::MemoryEffects::Read::get(), &inputBox,
3359	mlir::SideEffects::DefaultResource::get());
3360	}
3361
3362	//===----------------------------------------------------------------------===//
3363	// ResultOp
3364	//===----------------------------------------------------------------------===//
3365
3366	llvm::LogicalResult fir::ResultOp::verify() {
3367	auto *parentOp = (*this)->getParentOp();
3368	auto results = parentOp->getResults();
3369	auto operands = (*this)->getOperands();
3370
3371	if (parentOp->getNumResults() != getNumOperands())
3372	return emitOpError() << "parent of result must have same arity";
3373	for (auto e : llvm::zip(results, operands))
3374	if (std::get<0>(e).getType() != std::get<1>(e).getType())
3375	return emitOpError() << "types mismatch between result op and its parent";
3376	return mlir::success();
3377	}
3378
3379	//===----------------------------------------------------------------------===//
3380	// SaveResultOp
3381	//===----------------------------------------------------------------------===//
3382
3383	llvm::LogicalResult fir::SaveResultOp::verify() {
3384	auto resultType = getValue().getType();
3385	if (resultType != fir::dyn_cast_ptrEleTy(getMemref().getType()))
3386	return emitOpError("value type must match memory reference type");
3387	if (fir::isa_unknown_size_box(resultType))
3388	return emitOpError("cannot save !fir.box of unknown rank or type");
3389
3390	if (mlir::isa<fir::BoxType>(resultType)) {
3391	if (getShape() || !getTypeparams().empty())
3392	return emitOpError(
3393	"must not have shape or length operands if the value is a fir.box");
3394	return mlir::success();
3395	}
3396
3397	// fir.record or fir.array case.
3398	unsigned shapeTyRank = 0;
3399	if (auto shapeVal = getShape()) {
3400	auto shapeTy = shapeVal.getType();
3401	if (auto s = mlir::dyn_cast<fir::ShapeType>(shapeTy))
3402	shapeTyRank = s.getRank();
3403	else
3404	shapeTyRank = mlir::cast<fir::ShapeShiftType>(shapeTy).getRank();
3405	}
3406
3407	auto eleTy = resultType;
3408	if (auto seqTy = mlir::dyn_cast<fir::SequenceType>(resultType)) {
3409	if (seqTy.getDimension() != shapeTyRank)
3410	emitOpError("shape operand must be provided and have the value rank "
3411	"when the value is a fir.array");
3412	eleTy = seqTy.getEleTy();
3413	} else {
3414	if (shapeTyRank != 0)
3415	emitOpError(
3416	"shape operand should only be provided if the value is a fir.array");
3417	}
3418
3419	if (auto recTy = mlir::dyn_cast<fir::RecordType>(eleTy)) {
3420	if (recTy.getNumLenParams() != getTypeparams().size())
3421	emitOpError("length parameters number must match with the value type "
3422	"length parameters");
3423	} else if (auto charTy = mlir::dyn_cast<fir::CharacterType>(eleTy)) {
3424	if (getTypeparams().size() > 1)
3425	emitOpError("no more than one length parameter must be provided for "
3426	"character value");
3427	} else {
3428	if (!getTypeparams().empty())
3429	emitOpError("length parameters must not be provided for this value type");
3430	}
3431
3432	return mlir::success();
3433	}
3434
3435	//===----------------------------------------------------------------------===//
3436	// IntegralSwitchTerminator
3437	//===----------------------------------------------------------------------===//
3438	static constexpr llvm::StringRef getCompareOffsetAttr() {
3439	return "compare_operand_offsets";
3440	}
3441
3442	static constexpr llvm::StringRef getTargetOffsetAttr() {
3443	return "target_operand_offsets";
3444	}
3445
3446	template <typename OpT>
3447	static llvm::LogicalResult verifyIntegralSwitchTerminator(OpT op) {
3448	if (!mlir::isa<mlir::IntegerType, mlir::IndexType, fir::IntegerType>(
3449	op.getSelector().getType()))
3450	return op.emitOpError("must be an integer");
3451	auto cases =
3452	op->template getAttrOfType<mlir::ArrayAttr>(op.getCasesAttr()).getValue();
3453	auto count = op.getNumDest();
3454	if (count == 0)
3455	return op.emitOpError("must have at least one successor");
3456	if (op.getNumConditions() != count)
3457	return op.emitOpError("number of cases and targets don't match");
3458	if (op.targetOffsetSize() != count)
3459	return op.emitOpError("incorrect number of successor operand groups");
3460	for (decltype(count) i = 0; i != count; ++i) {
3461	if (!mlir::isa<mlir::IntegerAttr, mlir::UnitAttr>(cases[i]))
3462	return op.emitOpError("invalid case alternative");
3463	}
3464	return mlir::success();
3465	}
3466
3467	static mlir::ParseResult parseIntegralSwitchTerminator(
3468	mlir::OpAsmParser &parser, mlir::OperationState &result,
3469	llvm::StringRef casesAttr, llvm::StringRef operandSegmentAttr) {
3470	mlir::OpAsmParser::UnresolvedOperand selector;
3471	mlir::Type type;
3472	if (fir::parseSelector(parser, result, selector, type))
3473	return mlir::failure();
3474
3475	llvm::SmallVector<mlir::Attribute> ivalues;
3476	llvm::SmallVector<mlir::Block *> dests;
3477	llvm::SmallVector<llvm::SmallVector<mlir::Value>> destArgs;
3478	while (true) {
3479	mlir::Attribute ivalue; // Integer or Unit
3480	mlir::Block *dest;
3481	llvm::SmallVector<mlir::Value> destArg;
3482	mlir::NamedAttrList temp;
3483	if (parser.parseAttribute(result&: ivalue, attrName: "i", attrs&: temp) || parser.parseComma() ||
3484	parser.parseSuccessorAndUseList(dest, operands&: destArg))
3485	return mlir::failure();
3486	ivalues.push_back(Elt: ivalue);
3487	dests.push_back(Elt: dest);
3488	destArgs.push_back(Elt: destArg);
3489	if (!parser.parseOptionalRSquare())
3490	break;
3491	if (parser.parseComma())
3492	return mlir::failure();
3493	}
3494	auto &bld = parser.getBuilder();
3495	result.addAttribute(name: casesAttr, attr: bld.getArrayAttr(value: ivalues));
3496	llvm::SmallVector<int32_t> argOffs;
3497	int32_t sumArgs = 0;
3498	const auto count = dests.size();
3499	for (std::remove_const_t<decltype(count)> i = 0; i != count; ++i) {
3500	result.addSuccessors(successor: dests[i]);
3501	result.addOperands(newOperands: destArgs[i]);
3502	auto argSize = destArgs[i].size();
3503	argOffs.push_back(Elt: argSize);
3504	sumArgs += argSize;
3505	}
3506	result.addAttribute(name: operandSegmentAttr,
3507	attr: bld.getDenseI32ArrayAttr(values: {1, 0, sumArgs}));
3508	result.addAttribute(name: getTargetOffsetAttr(), attr: bld.getDenseI32ArrayAttr(values: argOffs));
3509	return mlir::success();
3510	}
3511
3512	template <typename OpT>
3513	static void printIntegralSwitchTerminator(OpT op, mlir::OpAsmPrinter &p) {
3514	p << ' ';
3515	p.printOperand(op.getSelector());
3516	p << " : " << op.getSelector().getType() << " [";
3517	auto cases =
3518	op->template getAttrOfType<mlir::ArrayAttr>(op.getCasesAttr()).getValue();
3519	auto count = op.getNumConditions();
3520	for (decltype(count) i = 0; i != count; ++i) {
3521	if (i)
3522	p << ", ";
3523	auto &attr = cases[i];
3524	if (auto intAttr = mlir::dyn_cast_or_null<mlir::IntegerAttr>(attr))
3525	p << intAttr.getValue();
3526	else
3527	p.printAttribute(attr);
3528	p << ", ";
3529	op.printSuccessorAtIndex(p, i);
3530	}
3531	p << ']';
3532	p.printOptionalAttrDict(
3533	attrs: op->getAttrs(), elidedAttrs: {op.getCasesAttr(), getCompareOffsetAttr(),
3534	getTargetOffsetAttr(), op.getOperandSegmentSizeAttr()});
3535	}
3536
3537	//===----------------------------------------------------------------------===//
3538	// SelectOp
3539	//===----------------------------------------------------------------------===//
3540
3541	llvm::LogicalResult fir::SelectOp::verify() {
3542	return verifyIntegralSwitchTerminator(*this);
3543	}
3544
3545	mlir::ParseResult fir::SelectOp::parse(mlir::OpAsmParser &parser,
3546	mlir::OperationState &result) {
3547	return parseIntegralSwitchTerminator(parser, result, getCasesAttr(),
3548	getOperandSegmentSizeAttr());
3549	}
3550
3551	void fir::SelectOp::print(mlir::OpAsmPrinter &p) {
3552	printIntegralSwitchTerminator(*this, p);
3553	}
3554
3555	template <typename A, typename... AdditionalArgs>
3556	static A getSubOperands(unsigned pos, A allArgs, mlir::DenseI32ArrayAttr ranges,
3557	AdditionalArgs &&...additionalArgs) {
3558	unsigned start = 0;
3559	for (unsigned i = 0; i < pos; ++i)
3560	start += ranges[i];
3561	return allArgs.slice(start, ranges[pos],
3562	std::forward<AdditionalArgs>(additionalArgs)...);
3563	}
3564
3565	static mlir::MutableOperandRange
3566	getMutableSuccessorOperands(unsigned pos, mlir::MutableOperandRange operands,
3567	llvm::StringRef offsetAttr) {
3568	mlir::Operation *owner = operands.getOwner();
3569	mlir::NamedAttribute targetOffsetAttr =
3570	*owner->getAttrDictionary().getNamed(name: offsetAttr);
3571	return getSubOperands(
3572	pos, allArgs: operands,
3573	ranges: mlir::cast<mlir::DenseI32ArrayAttr>(Val: targetOffsetAttr.getValue()),
3574	additionalArgs: mlir::MutableOperandRange::OperandSegment(pos, targetOffsetAttr));
3575	}
3576
3577	std::optional<mlir::OperandRange> fir::SelectOp::getCompareOperands(unsigned) {
3578	return {};
3579	}
3580
3581	std::optional<llvm::ArrayRef<mlir::Value>>
3582	fir::SelectOp::getCompareOperands(llvm::ArrayRef<mlir::Value>, unsigned) {
3583	return {};
3584	}
3585
3586	mlir::SuccessorOperands fir::SelectOp::getSuccessorOperands(unsigned oper) {
3587	return mlir::SuccessorOperands(::getMutableSuccessorOperands(
3588	oper, getTargetArgsMutable(), getTargetOffsetAttr()));
3589	}
3590
3591	std::optional<llvm::ArrayRef<mlir::Value>>
3592	fir::SelectOp::getSuccessorOperands(llvm::ArrayRef<mlir::Value> operands,
3593	unsigned oper) {
3594	auto a =
3595	(*this)->getAttrOfType<mlir::DenseI32ArrayAttr>(getTargetOffsetAttr());
3596	auto segments = (*this)->getAttrOfType<mlir::DenseI32ArrayAttr>(
3597	getOperandSegmentSizeAttr());
3598	return {getSubOperands(oper, getSubOperands(2, operands, segments), a)};
3599	}
3600
3601	std::optional<mlir::ValueRange>
3602	fir::SelectOp::getSuccessorOperands(mlir::ValueRange operands, unsigned oper) {
3603	auto a =
3604	(*this)->getAttrOfType<mlir::DenseI32ArrayAttr>(getTargetOffsetAttr());
3605	auto segments = (*this)->getAttrOfType<mlir::DenseI32ArrayAttr>(
3606	getOperandSegmentSizeAttr());
3607	return {getSubOperands(oper, getSubOperands(2, operands, segments), a)};
3608	}
3609
3610	unsigned fir::SelectOp::targetOffsetSize() {
3611	return (*this)
3612	->getAttrOfType<mlir::DenseI32ArrayAttr>(getTargetOffsetAttr())
3613	.size();
3614	}
3615
3616	//===----------------------------------------------------------------------===//
3617	// SelectCaseOp
3618	//===----------------------------------------------------------------------===//
3619
3620	std::optional<mlir::OperandRange>
3621	fir::SelectCaseOp::getCompareOperands(unsigned cond) {
3622	auto a =
3623	(*this)->getAttrOfType<mlir::DenseI32ArrayAttr>(getCompareOffsetAttr());
3624	return {getSubOperands(cond, getCompareArgs(), a)};
3625	}
3626
3627	std::optional<llvm::ArrayRef<mlir::Value>>
3628	fir::SelectCaseOp::getCompareOperands(llvm::ArrayRef<mlir::Value> operands,
3629	unsigned cond) {
3630	auto a =
3631	(*this)->getAttrOfType<mlir::DenseI32ArrayAttr>(getCompareOffsetAttr());
3632	auto segments = (*this)->getAttrOfType<mlir::DenseI32ArrayAttr>(
3633	getOperandSegmentSizeAttr());
3634	return {getSubOperands(cond, getSubOperands(1, operands, segments), a)};
3635	}
3636
3637	std::optional<mlir::ValueRange>
3638	fir::SelectCaseOp::getCompareOperands(mlir::ValueRange operands,
3639	unsigned cond) {
3640	auto a =
3641	(*this)->getAttrOfType<mlir::DenseI32ArrayAttr>(getCompareOffsetAttr());
3642	auto segments = (*this)->getAttrOfType<mlir::DenseI32ArrayAttr>(
3643	getOperandSegmentSizeAttr());
3644	return {getSubOperands(cond, getSubOperands(1, operands, segments), a)};
3645	}
3646
3647	mlir::SuccessorOperands fir::SelectCaseOp::getSuccessorOperands(unsigned oper) {
3648	return mlir::SuccessorOperands(::getMutableSuccessorOperands(
3649	oper, getTargetArgsMutable(), getTargetOffsetAttr()));
3650	}
3651
3652	std::optional<llvm::ArrayRef<mlir::Value>>
3653	fir::SelectCaseOp::getSuccessorOperands(llvm::ArrayRef<mlir::Value> operands,
3654	unsigned oper) {
3655	auto a =
3656	(*this)->getAttrOfType<mlir::DenseI32ArrayAttr>(getTargetOffsetAttr());
3657	auto segments = (*this)->getAttrOfType<mlir::DenseI32ArrayAttr>(
3658	getOperandSegmentSizeAttr());
3659	return {getSubOperands(oper, getSubOperands(2, operands, segments), a)};
3660	}
3661
3662	std::optional<mlir::ValueRange>
3663	fir::SelectCaseOp::getSuccessorOperands(mlir::ValueRange operands,
3664	unsigned oper) {
3665	auto a =
3666	(*this)->getAttrOfType<mlir::DenseI32ArrayAttr>(getTargetOffsetAttr());
3667	auto segments = (*this)->getAttrOfType<mlir::DenseI32ArrayAttr>(
3668	getOperandSegmentSizeAttr());
3669	return {getSubOperands(oper, getSubOperands(2, operands, segments), a)};
3670	}
3671
3672	// parser for fir.select_case Op
3673	mlir::ParseResult fir::SelectCaseOp::parse(mlir::OpAsmParser &parser,
3674	mlir::OperationState &result) {
3675	mlir::OpAsmParser::UnresolvedOperand selector;
3676	mlir::Type type;
3677	if (fir::parseSelector(parser, result, selector, type))
3678	return mlir::failure();
3679
3680	llvm::SmallVector<mlir::Attribute> attrs;
3681	llvm::SmallVector<mlir::OpAsmParser::UnresolvedOperand> opers;
3682	llvm::SmallVector<mlir::Block *> dests;
3683	llvm::SmallVector<llvm::SmallVector<mlir::Value>> destArgs;
3684	llvm::SmallVector<std::int32_t> argOffs;
3685	std::int32_t offSize = 0;
3686	while (true) {
3687	mlir::Attribute attr;
3688	mlir::Block *dest;
3689	llvm::SmallVector<mlir::Value> destArg;
3690	mlir::NamedAttrList temp;
3691	if (parser.parseAttribute(attr, "a", temp) || isValidCaseAttr(attr) ||
3692	parser.parseComma())
3693	return mlir::failure();
3694	attrs.push_back(attr);
3695	if (mlir::dyn_cast_or_null<mlir::UnitAttr>(attr)) {
3696	argOffs.push_back(0);
3697	} else if (mlir::dyn_cast_or_null<fir::ClosedIntervalAttr>(attr)) {
3698	mlir::OpAsmParser::UnresolvedOperand oper1;
3699	mlir::OpAsmParser::UnresolvedOperand oper2;
3700	if (parser.parseOperand(oper1) || parser.parseComma() ||
3701	parser.parseOperand(oper2) || parser.parseComma())
3702	return mlir::failure();
3703	opers.push_back(oper1);
3704	opers.push_back(oper2);
3705	argOffs.push_back(2);
3706	offSize += 2;
3707	} else {
3708	mlir::OpAsmParser::UnresolvedOperand oper;
3709	if (parser.parseOperand(oper) || parser.parseComma())
3710	return mlir::failure();
3711	opers.push_back(oper);
3712	argOffs.push_back(1);
3713	++offSize;
3714	}
3715	if (parser.parseSuccessorAndUseList(dest, destArg))
3716	return mlir::failure();
3717	dests.push_back(dest);
3718	destArgs.push_back(destArg);
3719	if (mlir::succeeded(parser.parseOptionalRSquare()))
3720	break;
3721	if (parser.parseComma())
3722	return mlir::failure();
3723	}
3724	result.addAttribute(fir::SelectCaseOp::getCasesAttr(),
3725	parser.getBuilder().getArrayAttr(attrs));
3726	if (parser.resolveOperands(opers, type, result.operands))
3727	return mlir::failure();
3728	llvm::SmallVector<int32_t> targOffs;
3729	int32_t toffSize = 0;
3730	const auto count = dests.size();
3731	for (std::remove_const_t<decltype(count)> i = 0; i != count; ++i) {
3732	result.addSuccessors(dests[i]);
3733	result.addOperands(destArgs[i]);
3734	auto argSize = destArgs[i].size();
3735	targOffs.push_back(argSize);
3736	toffSize += argSize;
3737	}
3738	auto &bld = parser.getBuilder();
3739	result.addAttribute(fir::SelectCaseOp::getOperandSegmentSizeAttr(),
3740	bld.getDenseI32ArrayAttr({1, offSize, toffSize}));
3741	result.addAttribute(getCompareOffsetAttr(),
3742	bld.getDenseI32ArrayAttr(argOffs));
3743	result.addAttribute(getTargetOffsetAttr(),
3744	bld.getDenseI32ArrayAttr(targOffs));
3745	return mlir::success();
3746	}
3747
3748	void fir::SelectCaseOp::print(mlir::OpAsmPrinter &p) {
3749	p << ' ';
3750	p.printOperand(getSelector());
3751	p << " : " << getSelector().getType() << " [";
3752	auto cases =
3753	getOperation()->getAttrOfType<mlir::ArrayAttr>(getCasesAttr()).getValue();
3754	auto count = getNumConditions();
3755	for (decltype(count) i = 0; i != count; ++i) {
3756	if (i)
3757	p << ", ";
3758	p << cases[i] << ", ";
3759	if (!mlir::isa<mlir::UnitAttr>(cases[i])) {
3760	auto caseArgs = *getCompareOperands(i);
3761	p.printOperand(*caseArgs.begin());
3762	p << ", ";
3763	if (mlir::isa<fir::ClosedIntervalAttr>(cases[i])) {
3764	p.printOperand(*(++caseArgs.begin()));
3765	p << ", ";
3766	}
3767	}
3768	printSuccessorAtIndex(p, i);
3769	}
3770	p << ']';
3771	p.printOptionalAttrDict(getOperation()->getAttrs(),
3772	{getCasesAttr(), getCompareOffsetAttr(),
3773	getTargetOffsetAttr(), getOperandSegmentSizeAttr()});
3774	}
3775
3776	unsigned fir::SelectCaseOp::compareOffsetSize() {
3777	return (*this)
3778	->getAttrOfType<mlir::DenseI32ArrayAttr>(getCompareOffsetAttr())
3779	.size();
3780	}
3781
3782	unsigned fir::SelectCaseOp::targetOffsetSize() {
3783	return (*this)
3784	->getAttrOfType<mlir::DenseI32ArrayAttr>(getTargetOffsetAttr())
3785	.size();
3786	}
3787
3788	void fir::SelectCaseOp::build(mlir::OpBuilder &builder,
3789	mlir::OperationState &result,
3790	mlir::Value selector,
3791	llvm::ArrayRef<mlir::Attribute> compareAttrs,
3792	llvm::ArrayRef<mlir::ValueRange> cmpOperands,
3793	llvm::ArrayRef<mlir::Block *> destinations,
3794	llvm::ArrayRef<mlir::ValueRange> destOperands,
3795	llvm::ArrayRef<mlir::NamedAttribute> attributes) {
3796	result.addOperands(selector);
3797	result.addAttribute(getCasesAttr(), builder.getArrayAttr(compareAttrs));
3798	llvm::SmallVector<int32_t> operOffs;
3799	int32_t operSize = 0;
3800	for (auto attr : compareAttrs) {
3801	if (mlir::isa<fir::ClosedIntervalAttr>(attr)) {
3802	operOffs.push_back(2);
3803	operSize += 2;
3804	} else if (mlir::isa<mlir::UnitAttr>(attr)) {
3805	operOffs.push_back(0);
3806	} else {
3807	operOffs.push_back(1);
3808	++operSize;
3809	}
3810	}
3811	for (auto ops : cmpOperands)
3812	result.addOperands(ops);
3813	result.addAttribute(getCompareOffsetAttr(),
3814	builder.getDenseI32ArrayAttr(operOffs));
3815	const auto count = destinations.size();
3816	for (auto d : destinations)
3817	result.addSuccessors(d);
3818	const auto opCount = destOperands.size();
3819	llvm::SmallVector<std::int32_t> argOffs;
3820	std::int32_t sumArgs = 0;
3821	for (std::remove_const_t<decltype(count)> i = 0; i != count; ++i) {
3822	if (i < opCount) {
3823	result.addOperands(destOperands[i]);
3824	const auto argSz = destOperands[i].size();
3825	argOffs.push_back(argSz);
3826	sumArgs += argSz;
3827	} else {
3828	argOffs.push_back(0);
3829	}
3830	}
3831	result.addAttribute(getOperandSegmentSizeAttr(),
3832	builder.getDenseI32ArrayAttr({1, operSize, sumArgs}));
3833	result.addAttribute(getTargetOffsetAttr(),
3834	builder.getDenseI32ArrayAttr(argOffs));
3835	result.addAttributes(attributes);
3836	}
3837
3838	/// This builder has a slightly simplified interface in that the list of
3839	/// operands need not be partitioned by the builder. Instead the operands are
3840	/// partitioned here, before being passed to the default builder. This
3841	/// partitioning is unchecked, so can go awry on bad input.
3842	void fir::SelectCaseOp::build(mlir::OpBuilder &builder,
3843	mlir::OperationState &result,
3844	mlir::Value selector,
3845	llvm::ArrayRef<mlir::Attribute> compareAttrs,
3846	llvm::ArrayRef<mlir::Value> cmpOpList,
3847	llvm::ArrayRef<mlir::Block *> destinations,
3848	llvm::ArrayRef<mlir::ValueRange> destOperands,
3849	llvm::ArrayRef<mlir::NamedAttribute> attributes) {
3850	llvm::SmallVector<mlir::ValueRange> cmpOpers;
3851	auto iter = cmpOpList.begin();
3852	for (auto &attr : compareAttrs) {
3853	if (mlir::isa<fir::ClosedIntervalAttr>(attr)) {
3854	cmpOpers.push_back(mlir::ValueRange({iter, iter + 2}));
3855	iter += 2;
3856	} else if (mlir::isa<mlir::UnitAttr>(attr)) {
3857	cmpOpers.push_back(mlir::ValueRange{});
3858	} else {
3859	cmpOpers.push_back(mlir::ValueRange({iter, iter + 1}));
3860	++iter;
3861	}
3862	}
3863	build(builder, result, selector, compareAttrs, cmpOpers, destinations,
3864	destOperands, attributes);
3865	}
3866
3867	llvm::LogicalResult fir::SelectCaseOp::verify() {
3868	if (!mlir::isa<mlir::IntegerType, mlir::IndexType, fir::IntegerType,
3869	fir::LogicalType, fir::CharacterType>(getSelector().getType()))
3870	return emitOpError("must be an integer, character, or logical");
3871	auto cases =
3872	getOperation()->getAttrOfType<mlir::ArrayAttr>(getCasesAttr()).getValue();
3873	auto count = getNumDest();
3874	if (count == 0)
3875	return emitOpError("must have at least one successor");
3876	if (getNumConditions() != count)
3877	return emitOpError("number of conditions and successors don't match");
3878	if (compareOffsetSize() != count)
3879	return emitOpError("incorrect number of compare operand groups");
3880	if (targetOffsetSize() != count)
3881	return emitOpError("incorrect number of successor operand groups");
3882	for (decltype(count) i = 0; i != count; ++i) {
3883	auto &attr = cases[i];
3884	if (!(mlir::isa<fir::PointIntervalAttr>(attr) ||
3885	mlir::isa<fir::LowerBoundAttr>(attr) ||
3886	mlir::isa<fir::UpperBoundAttr>(attr) ||
3887	mlir::isa<fir::ClosedIntervalAttr>(attr) ||
3888	mlir::isa<mlir::UnitAttr>(attr)))
3889	return emitOpError("incorrect select case attribute type");
3890	}
3891	return mlir::success();
3892	}
3893
3894	//===----------------------------------------------------------------------===//
3895	// SelectRankOp
3896	//===----------------------------------------------------------------------===//
3897
3898	llvm::LogicalResult fir::SelectRankOp::verify() {
3899	return verifyIntegralSwitchTerminator(*this);
3900	}
3901
3902	mlir::ParseResult fir::SelectRankOp::parse(mlir::OpAsmParser &parser,
3903	mlir::OperationState &result) {
3904	return parseIntegralSwitchTerminator(parser, result, getCasesAttr(),
3905	getOperandSegmentSizeAttr());
3906	}
3907
3908	void fir::SelectRankOp::print(mlir::OpAsmPrinter &p) {
3909	printIntegralSwitchTerminator(*this, p);
3910	}
3911
3912	std::optional<mlir::OperandRange>
3913	fir::SelectRankOp::getCompareOperands(unsigned) {
3914	return {};
3915	}
3916
3917	std::optional<llvm::ArrayRef<mlir::Value>>
3918	fir::SelectRankOp::getCompareOperands(llvm::ArrayRef<mlir::Value>, unsigned) {
3919	return {};
3920	}
3921
3922	mlir::SuccessorOperands fir::SelectRankOp::getSuccessorOperands(unsigned oper) {
3923	return mlir::SuccessorOperands(::getMutableSuccessorOperands(
3924	oper, getTargetArgsMutable(), getTargetOffsetAttr()));
3925	}
3926
3927	std::optional<llvm::ArrayRef<mlir::Value>>
3928	fir::SelectRankOp::getSuccessorOperands(llvm::ArrayRef<mlir::Value> operands,
3929	unsigned oper) {
3930	auto a =
3931	(*this)->getAttrOfType<mlir::DenseI32ArrayAttr>(getTargetOffsetAttr());
3932	auto segments = (*this)->getAttrOfType<mlir::DenseI32ArrayAttr>(
3933	getOperandSegmentSizeAttr());
3934	return {getSubOperands(oper, getSubOperands(2, operands, segments), a)};
3935	}
3936
3937	std::optional<mlir::ValueRange>
3938	fir::SelectRankOp::getSuccessorOperands(mlir::ValueRange operands,
3939	unsigned oper) {
3940	auto a =
3941	(*this)->getAttrOfType<mlir::DenseI32ArrayAttr>(getTargetOffsetAttr());
3942	auto segments = (*this)->getAttrOfType<mlir::DenseI32ArrayAttr>(
3943	getOperandSegmentSizeAttr());
3944	return {getSubOperands(oper, getSubOperands(2, operands, segments), a)};
3945	}
3946
3947	unsigned fir::SelectRankOp::targetOffsetSize() {
3948	return (*this)
3949	->getAttrOfType<mlir::DenseI32ArrayAttr>(getTargetOffsetAttr())
3950	.size();
3951	}
3952
3953	//===----------------------------------------------------------------------===//
3954	// SelectTypeOp
3955	//===----------------------------------------------------------------------===//
3956
3957	std::optional<mlir::OperandRange>
3958	fir::SelectTypeOp::getCompareOperands(unsigned) {
3959	return {};
3960	}
3961
3962	std::optional<llvm::ArrayRef<mlir::Value>>
3963	fir::SelectTypeOp::getCompareOperands(llvm::ArrayRef<mlir::Value>, unsigned) {
3964	return {};
3965	}
3966
3967	mlir::SuccessorOperands fir::SelectTypeOp::getSuccessorOperands(unsigned oper) {
3968	return mlir::SuccessorOperands(::getMutableSuccessorOperands(
3969	oper, getTargetArgsMutable(), getTargetOffsetAttr()));
3970	}
3971
3972	std::optional<llvm::ArrayRef<mlir::Value>>
3973	fir::SelectTypeOp::getSuccessorOperands(llvm::ArrayRef<mlir::Value> operands,
3974	unsigned oper) {
3975	auto a =
3976	(*this)->getAttrOfType<mlir::DenseI32ArrayAttr>(getTargetOffsetAttr());
3977	auto segments = (*this)->getAttrOfType<mlir::DenseI32ArrayAttr>(
3978	getOperandSegmentSizeAttr());
3979	return {getSubOperands(oper, getSubOperands(2, operands, segments), a)};
3980	}
3981
3982	std::optional<mlir::ValueRange>
3983	fir::SelectTypeOp::getSuccessorOperands(mlir::ValueRange operands,
3984	unsigned oper) {
3985	auto a =
3986	(*this)->getAttrOfType<mlir::DenseI32ArrayAttr>(getTargetOffsetAttr());
3987	auto segments = (*this)->getAttrOfType<mlir::DenseI32ArrayAttr>(
3988	getOperandSegmentSizeAttr());
3989	return {getSubOperands(oper, getSubOperands(2, operands, segments), a)};
3990	}
3991
3992	mlir::ParseResult fir::SelectTypeOp::parse(mlir::OpAsmParser &parser,
3993	mlir::OperationState &result) {
3994	mlir::OpAsmParser::UnresolvedOperand selector;
3995	mlir::Type type;
3996	if (fir::parseSelector(parser, result, selector, type))
3997	return mlir::failure();
3998
3999	llvm::SmallVector<mlir::Attribute> attrs;
4000	llvm::SmallVector<mlir::Block *> dests;
4001	llvm::SmallVector<llvm::SmallVector<mlir::Value>> destArgs;
4002	while (true) {
4003	mlir::Attribute attr;
4004	mlir::Block *dest;
4005	llvm::SmallVector<mlir::Value> destArg;
4006	mlir::NamedAttrList temp;
4007	if (parser.parseAttribute(attr, "a", temp) || parser.parseComma() ||
4008	parser.parseSuccessorAndUseList(dest, destArg))
4009	return mlir::failure();
4010	attrs.push_back(attr);
4011	dests.push_back(dest);
4012	destArgs.push_back(destArg);
4013	if (mlir::succeeded(parser.parseOptionalRSquare()))
4014	break;
4015	if (parser.parseComma())
4016	return mlir::failure();
4017	}
4018	auto &bld = parser.getBuilder();
4019	result.addAttribute(fir::SelectTypeOp::getCasesAttr(),
4020	bld.getArrayAttr(attrs));
4021	llvm::SmallVector<int32_t> argOffs;
4022	int32_t offSize = 0;
4023	const auto count = dests.size();
4024	for (std::remove_const_t<decltype(count)> i = 0; i != count; ++i) {
4025	result.addSuccessors(dests[i]);
4026	result.addOperands(destArgs[i]);
4027	auto argSize = destArgs[i].size();
4028	argOffs.push_back(argSize);
4029	offSize += argSize;
4030	}
4031	result.addAttribute(fir::SelectTypeOp::getOperandSegmentSizeAttr(),
4032	bld.getDenseI32ArrayAttr({1, 0, offSize}));
4033	result.addAttribute(getTargetOffsetAttr(), bld.getDenseI32ArrayAttr(argOffs));
4034	return mlir::success();
4035	}
4036
4037	unsigned fir::SelectTypeOp::targetOffsetSize() {
4038	return (*this)
4039	->getAttrOfType<mlir::DenseI32ArrayAttr>(getTargetOffsetAttr())
4040	.size();
4041	}
4042
4043	void fir::SelectTypeOp::print(mlir::OpAsmPrinter &p) {
4044	p << ' ';
4045	p.printOperand(getSelector());
4046	p << " : " << getSelector().getType() << " [";
4047	auto cases =
4048	getOperation()->getAttrOfType<mlir::ArrayAttr>(getCasesAttr()).getValue();
4049	auto count = getNumConditions();
4050	for (decltype(count) i = 0; i != count; ++i) {
4051	if (i)
4052	p << ", ";
4053	p << cases[i] << ", ";
4054	printSuccessorAtIndex(p, i);
4055	}
4056	p << ']';
4057	p.printOptionalAttrDict(getOperation()->getAttrs(),
4058	{getCasesAttr(), getCompareOffsetAttr(),
4059	getTargetOffsetAttr(),
4060	fir::SelectTypeOp::getOperandSegmentSizeAttr()});
4061	}
4062
4063	llvm::LogicalResult fir::SelectTypeOp::verify() {
4064	if (!mlir::isa<fir::BaseBoxType>(getSelector().getType()))
4065	return emitOpError("must be a fir.class or fir.box type");
4066	if (auto boxType = mlir::dyn_cast<fir::BoxType>(getSelector().getType()))
4067	if (!mlir::isa<mlir::NoneType>(boxType.getEleTy()))
4068	return emitOpError("selector must be polymorphic");
4069	auto typeGuardAttr = getCases();
4070	for (unsigned idx = 0; idx < typeGuardAttr.size(); ++idx)
4071	if (mlir::isa<mlir::UnitAttr>(typeGuardAttr[idx]) &&
4072	idx != typeGuardAttr.size() - 1)
4073	return emitOpError("default must be the last attribute");
4074	auto count = getNumDest();
4075	if (count == 0)
4076	return emitOpError("must have at least one successor");
4077	if (getNumConditions() != count)
4078	return emitOpError("number of conditions and successors don't match");
4079	if (targetOffsetSize() != count)
4080	return emitOpError("incorrect number of successor operand groups");
4081	for (unsigned i = 0; i != count; ++i) {
4082	if (!mlir::isa<fir::ExactTypeAttr, fir::SubclassAttr, mlir::UnitAttr>(
4083	typeGuardAttr[i]))
4084	return emitOpError("invalid type-case alternative");
4085	}
4086	return mlir::success();
4087	}
4088
4089	void fir::SelectTypeOp::build(mlir::OpBuilder &builder,
4090	mlir::OperationState &result,
4091	mlir::Value selector,
4092	llvm::ArrayRef<mlir::Attribute> typeOperands,
4093	llvm::ArrayRef<mlir::Block *> destinations,
4094	llvm::ArrayRef<mlir::ValueRange> destOperands,
4095	llvm::ArrayRef<mlir::NamedAttribute> attributes) {
4096	result.addOperands(selector);
4097	result.addAttribute(getCasesAttr(), builder.getArrayAttr(typeOperands));
4098	const auto count = destinations.size();
4099	for (mlir::Block *dest : destinations)
4100	result.addSuccessors(dest);
4101	const auto opCount = destOperands.size();
4102	llvm::SmallVector<int32_t> argOffs;
4103	int32_t sumArgs = 0;
4104	for (std::remove_const_t<decltype(count)> i = 0; i != count; ++i) {
4105	if (i < opCount) {
4106	result.addOperands(destOperands[i]);
4107	const auto argSz = destOperands[i].size();
4108	argOffs.push_back(argSz);
4109	sumArgs += argSz;
4110	} else {
4111	argOffs.push_back(0);
4112	}
4113	}
4114	result.addAttribute(getOperandSegmentSizeAttr(),
4115	builder.getDenseI32ArrayAttr({1, 0, sumArgs}));
4116	result.addAttribute(getTargetOffsetAttr(),
4117	builder.getDenseI32ArrayAttr(argOffs));
4118	result.addAttributes(attributes);
4119	}
4120
4121	//===----------------------------------------------------------------------===//
4122	// ShapeOp
4123	//===----------------------------------------------------------------------===//
4124
4125	llvm::LogicalResult fir::ShapeOp::verify() {
4126	auto size = getExtents().size();
4127	auto shapeTy = mlir::dyn_cast<fir::ShapeType>(getType());
4128	assert(shapeTy && "must be a shape type");
4129	if (shapeTy.getRank() != size)
4130	return emitOpError("shape type rank mismatch");
4131	return mlir::success();
4132	}
4133
4134	void fir::ShapeOp::build(mlir::OpBuilder &builder, mlir::OperationState &result,
4135	mlir::ValueRange extents) {
4136	auto type = fir::ShapeType::get(builder.getContext(), extents.size());
4137	build(builder, result, type, extents);
4138	}
4139
4140	//===----------------------------------------------------------------------===//
4141	// ShapeShiftOp
4142	//===----------------------------------------------------------------------===//
4143
4144	llvm::LogicalResult fir::ShapeShiftOp::verify() {
4145	auto size = getPairs().size();
4146	if (size < 2 || size > 16 * 2)
4147	return emitOpError("incorrect number of args");
4148	if (size % 2 != 0)
4149	return emitOpError("requires a multiple of 2 args");
4150	auto shapeTy = mlir::dyn_cast<fir::ShapeShiftType>(getType());
4151	assert(shapeTy && "must be a shape shift type");
4152	if (shapeTy.getRank() * 2 != size)
4153	return emitOpError("shape type rank mismatch");
4154	return mlir::success();
4155	}
4156
4157	//===----------------------------------------------------------------------===//
4158	// ShiftOp
4159	//===----------------------------------------------------------------------===//
4160
4161	llvm::LogicalResult fir::ShiftOp::verify() {
4162	auto size = getOrigins().size();
4163	auto shiftTy = mlir::dyn_cast<fir::ShiftType>(getType());
4164	assert(shiftTy && "must be a shift type");
4165	if (shiftTy.getRank() != size)
4166	return emitOpError("shift type rank mismatch");
4167	return mlir::success();
4168	}
4169
4170	//===----------------------------------------------------------------------===//
4171	// SliceOp
4172	//===----------------------------------------------------------------------===//
4173
4174	void fir::SliceOp::build(mlir::OpBuilder &builder, mlir::OperationState &result,
4175	mlir::ValueRange trips, mlir::ValueRange path,
4176	mlir::ValueRange substr) {
4177	const auto rank = trips.size() / 3;
4178	auto sliceTy = fir::SliceType::get(builder.getContext(), rank);
4179	build(builder, result, sliceTy, trips, path, substr);
4180	}
4181
4182	/// Return the output rank of a slice op. The output rank must be between 1 and
4183	/// the rank of the array being sliced (inclusive).
4184	unsigned fir::SliceOp::getOutputRank(mlir::ValueRange triples) {
4185	unsigned rank = 0;
4186	if (!triples.empty()) {
4187	for (unsigned i = 1, end = triples.size(); i < end; i += 3) {
4188	auto *op = triples[i].getDefiningOp();
4189	if (!mlir::isa_and_nonnull<fir::UndefOp>(op))
4190	++rank;
4191	}
4192	assert(rank > 0);
4193	}
4194	return rank;
4195	}
4196
4197	llvm::LogicalResult fir::SliceOp::verify() {
4198	auto size = getTriples().size();
4199	if (size < 3 || size > 16 * 3)
4200	return emitOpError("incorrect number of args for triple");
4201	if (size % 3 != 0)
4202	return emitOpError("requires a multiple of 3 args");
4203	auto sliceTy = mlir::dyn_cast<fir::SliceType>(getType());
4204	assert(sliceTy && "must be a slice type");
4205	if (sliceTy.getRank() * 3 != size)
4206	return emitOpError("slice type rank mismatch");
4207	return mlir::success();
4208	}
4209
4210	//===----------------------------------------------------------------------===//
4211	// StoreOp
4212	//===----------------------------------------------------------------------===//
4213
4214	mlir::Type fir::StoreOp::elementType(mlir::Type refType) {
4215	return fir::dyn_cast_ptrEleTy(refType);
4216	}
4217
4218	mlir::ParseResult fir::StoreOp::parse(mlir::OpAsmParser &parser,
4219	mlir::OperationState &result) {
4220	mlir::Type type;
4221	mlir::OpAsmParser::UnresolvedOperand oper;
4222	mlir::OpAsmParser::UnresolvedOperand store;
4223	if (parser.parseOperand(oper) || parser.parseKeyword("to") ||
4224	parser.parseOperand(store) ||
4225	parser.parseOptionalAttrDict(result.attributes) ||
4226	parser.parseColonType(type) ||
4227	parser.resolveOperand(oper, fir::StoreOp::elementType(type),
4228	result.operands) ||
4229	parser.resolveOperand(store, type, result.operands))
4230	return mlir::failure();
4231	return mlir::success();
4232	}
4233
4234	void fir::StoreOp::print(mlir::OpAsmPrinter &p) {
4235	p << ' ';
4236	p.printOperand(getValue());
4237	p << " to ";
4238	p.printOperand(getMemref());
4239	p.printOptionalAttrDict(getOperation()->getAttrs(), {});
4240	p << " : " << getMemref().getType();
4241	}
4242
4243	llvm::LogicalResult fir::StoreOp::verify() {
4244	if (getValue().getType() != fir::dyn_cast_ptrEleTy(getMemref().getType()))
4245	return emitOpError("store value type must match memory reference type");
4246	return mlir::success();
4247	}
4248
4249	void fir::StoreOp::build(mlir::OpBuilder &builder, mlir::OperationState &result,
4250	mlir::Value value, mlir::Value memref) {
4251	build(builder, result, value, memref, {});
4252	}
4253
4254	void fir::StoreOp::getEffects(
4255	llvm::SmallVectorImpl<
4256	mlir::SideEffects::EffectInstance<mlir::MemoryEffects::Effect>>
4257	&effects) {
4258	effects.emplace_back(mlir::MemoryEffects::Write::get(), &getMemrefMutable(),
4259	mlir::SideEffects::DefaultResource::get());
4260	addVolatileMemoryEffects({getMemref().getType()}, effects);
4261	}
4262
4263	//===----------------------------------------------------------------------===//
4264	// CopyOp
4265	//===----------------------------------------------------------------------===//
4266
4267	void fir::CopyOp::build(mlir::OpBuilder &builder, mlir::OperationState &result,
4268	mlir::Value source, mlir::Value destination,
4269	bool noOverlap) {
4270	mlir::UnitAttr noOverlapAttr =
4271	noOverlap ? builder.getUnitAttr() : mlir::UnitAttr{};
4272	build(builder, result, source, destination, noOverlapAttr);
4273	}
4274
4275	llvm::LogicalResult fir::CopyOp::verify() {
4276	mlir::Type sourceType = fir::unwrapRefType(getSource().getType());
4277	mlir::Type destinationType = fir::unwrapRefType(getDestination().getType());
4278	if (sourceType != destinationType)
4279	return emitOpError("source and destination must have the same value type");
4280	return mlir::success();
4281	}
4282
4283	void fir::CopyOp::getEffects(
4284	llvm::SmallVectorImpl<
4285	mlir::SideEffects::EffectInstance<mlir::MemoryEffects::Effect>>
4286	&effects) {
4287	effects.emplace_back(mlir::MemoryEffects::Read::get(), &getSourceMutable(),
4288	mlir::SideEffects::DefaultResource::get());
4289	effects.emplace_back(mlir::MemoryEffects::Write::get(),
4290	&getDestinationMutable(),
4291	mlir::SideEffects::DefaultResource::get());
4292	addVolatileMemoryEffects({getDestination().getType(), getSource().getType()},
4293	effects);
4294	}
4295
4296	//===----------------------------------------------------------------------===//
4297	// StringLitOp
4298	//===----------------------------------------------------------------------===//
4299
4300	inline fir::CharacterType::KindTy stringLitOpGetKind(fir::StringLitOp op) {
4301	auto eleTy = mlir::cast<fir::SequenceType>(op.getType()).getElementType();
4302	return mlir::cast<fir::CharacterType>(eleTy).getFKind();
4303	}
4304
4305	bool fir::StringLitOp::isWideValue() { return stringLitOpGetKind(*this) != 1; }
4306
4307	static mlir::NamedAttribute
4308	mkNamedIntegerAttr(mlir::OpBuilder &builder, llvm::StringRef name, int64_t v) {
4309	assert(v > 0);
4310	return builder.getNamedAttr(
4311	name, val: builder.getIntegerAttr(type: builder.getIntegerType(width: 64), value: v));
4312	}
4313
4314	void fir::StringLitOp::build(mlir::OpBuilder &builder,
4315	mlir::OperationState &result,
4316	fir::CharacterType inType, llvm::StringRef val,
4317	std::optional<int64_t> len) {
4318	auto valAttr = builder.getNamedAttr(value(), builder.getStringAttr(val));
4319	int64_t length = len ? *len : inType.getLen();
4320	auto lenAttr = mkNamedIntegerAttr(builder, size(), length);
4321	result.addAttributes({valAttr, lenAttr});
4322	result.addTypes(inType);
4323	}
4324
4325	template <typename C>
4326	static mlir::ArrayAttr convertToArrayAttr(mlir::OpBuilder &builder,
4327	llvm::ArrayRef<C> xlist) {
4328	llvm::SmallVector<mlir::Attribute> attrs;
4329	auto ty = builder.getIntegerType(width: 8 * sizeof(C));
4330	for (auto ch : xlist)
4331	attrs.push_back(Elt: builder.getIntegerAttr(ty, ch));
4332	return builder.getArrayAttr(value: attrs);
4333	}
4334
4335	void fir::StringLitOp::build(mlir::OpBuilder &builder,
4336	mlir::OperationState &result,
4337	fir::CharacterType inType,
4338	llvm::ArrayRef<char> vlist,
4339	std::optional<std::int64_t> len) {
4340	auto valAttr =
4341	builder.getNamedAttr(xlist(), convertToArrayAttr(builder, vlist));
4342	std::int64_t length = len ? *len : inType.getLen();
4343	auto lenAttr = mkNamedIntegerAttr(builder, size(), length);
4344	result.addAttributes({valAttr, lenAttr});
4345	result.addTypes(inType);
4346	}
4347
4348	void fir::StringLitOp::build(mlir::OpBuilder &builder,
4349	mlir::OperationState &result,
4350	fir::CharacterType inType,
4351	llvm::ArrayRef<char16_t> vlist,
4352	std::optional<std::int64_t> len) {
4353	auto valAttr =
4354	builder.getNamedAttr(xlist(), convertToArrayAttr(builder, vlist));
4355	std::int64_t length = len ? *len : inType.getLen();
4356	auto lenAttr = mkNamedIntegerAttr(builder, size(), length);
4357	result.addAttributes({valAttr, lenAttr});
4358	result.addTypes(inType);
4359	}
4360
4361	void fir::StringLitOp::build(mlir::OpBuilder &builder,
4362	mlir::OperationState &result,
4363	fir::CharacterType inType,
4364	llvm::ArrayRef<char32_t> vlist,
4365	std::optional<std::int64_t> len) {
4366	auto valAttr =
4367	builder.getNamedAttr(xlist(), convertToArrayAttr(builder, vlist));
4368	std::int64_t length = len ? *len : inType.getLen();
4369	auto lenAttr = mkNamedIntegerAttr(builder, size(), length);
4370	result.addAttributes({valAttr, lenAttr});
4371	result.addTypes(inType);
4372	}
4373
4374	mlir::ParseResult fir::StringLitOp::parse(mlir::OpAsmParser &parser,
4375	mlir::OperationState &result) {
4376	auto &builder = parser.getBuilder();
4377	mlir::Attribute val;
4378	mlir::NamedAttrList attrs;
4379	llvm::SMLoc trailingTypeLoc;
4380	if (parser.parseAttribute(val, "fake", attrs))
4381	return mlir::failure();
4382	if (auto v = mlir::dyn_cast<mlir::StringAttr>(val))
4383	result.attributes.push_back(
4384	builder.getNamedAttr(fir::StringLitOp::value(), v));
4385	else if (auto v = mlir::dyn_cast<mlir::DenseElementsAttr>(val))
4386	result.attributes.push_back(
4387	builder.getNamedAttr(fir::StringLitOp::xlist(), v));
4388	else if (auto v = mlir::dyn_cast<mlir::ArrayAttr>(val))
4389	result.attributes.push_back(
4390	builder.getNamedAttr(fir::StringLitOp::xlist(), v));
4391	else
4392	return parser.emitError(parser.getCurrentLocation(),
4393	"found an invalid constant");
4394	mlir::IntegerAttr sz;
4395	mlir::Type type;
4396	if (parser.parseLParen() ||
4397	parser.parseAttribute(sz, fir::StringLitOp::size(), result.attributes) ||
4398	parser.parseRParen() || parser.getCurrentLocation(&trailingTypeLoc) ||
4399	parser.parseColonType(type))
4400	return mlir::failure();
4401	auto charTy = mlir::dyn_cast<fir::CharacterType>(type);
4402	if (!charTy)
4403	return parser.emitError(trailingTypeLoc, "must have character type");
4404	type = fir::CharacterType::get(builder.getContext(), charTy.getFKind(),
4405	sz.getInt());
4406	if (!type || parser.addTypesToList(type, result.types))
4407	return mlir::failure();
4408	return mlir::success();
4409	}
4410
4411	void fir::StringLitOp::print(mlir::OpAsmPrinter &p) {
4412	p << ' ' << getValue() << '(';
4413	p << mlir::cast<mlir::IntegerAttr>(getSize()).getValue() << ") : ";
4414	p.printType(getType());
4415	}
4416
4417	llvm::LogicalResult fir::StringLitOp::verify() {
4418	if (mlir::cast<mlir::IntegerAttr>(getSize()).getValue().isNegative())
4419	return emitOpError("size must be non-negative");
4420	if (auto xl = getOperation()->getAttr(fir::StringLitOp::xlist())) {
4421	if (auto xList = mlir::dyn_cast<mlir::ArrayAttr>(xl)) {
4422	for (auto a : xList)
4423	if (!mlir::isa<mlir::IntegerAttr>(a))
4424	return emitOpError("values in initializer must be integers");
4425	} else if (mlir::isa<mlir::DenseElementsAttr>(xl)) {
4426	// do nothing
4427	} else {
4428	return emitOpError("has unexpected attribute");
4429	}
4430	}
4431	return mlir::success();
4432	}
4433
4434	//===----------------------------------------------------------------------===//
4435	// UnboxProcOp
4436	//===----------------------------------------------------------------------===//
4437
4438	llvm::LogicalResult fir::UnboxProcOp::verify() {
4439	if (auto eleTy = fir::dyn_cast_ptrEleTy(getRefTuple().getType()))
4440	if (mlir::isa<mlir::TupleType>(eleTy))
4441	return mlir::success();
4442	return emitOpError("second output argument has bad type");
4443	}
4444
4445	//===----------------------------------------------------------------------===//
4446	// IfOp
4447	//===----------------------------------------------------------------------===//
4448
4449	void fir::IfOp::build(mlir::OpBuilder &builder, mlir::OperationState &result,
4450	mlir::Value cond, bool withElseRegion) {
4451	build(builder, result, std::nullopt, cond, withElseRegion);
4452	}
4453
4454	void fir::IfOp::build(mlir::OpBuilder &builder, mlir::OperationState &result,
4455	mlir::TypeRange resultTypes, mlir::Value cond,
4456	bool withElseRegion) {
4457	result.addOperands(cond);
4458	result.addTypes(resultTypes);
4459
4460	mlir::Region *thenRegion = result.addRegion();
4461	thenRegion->push_back(new mlir::Block());
4462	if (resultTypes.empty())
4463	IfOp::ensureTerminator(*thenRegion, builder, result.location);
4464
4465	mlir::Region *elseRegion = result.addRegion();
4466	if (withElseRegion) {
4467	elseRegion->push_back(new mlir::Block());
4468	if (resultTypes.empty())
4469	IfOp::ensureTerminator(*elseRegion, builder, result.location);
4470	}
4471	}
4472
4473	// These 3 functions copied from scf.if implementation.
4474
4475	/// Given the region at `index`, or the parent operation if `index` is None,
4476	/// return the successor regions. These are the regions that may be selected
4477	/// during the flow of control.
4478	void fir::IfOp::getSuccessorRegions(
4479	mlir::RegionBranchPoint point,
4480	llvm::SmallVectorImpl<mlir::RegionSuccessor> &regions) {
4481	// The `then` and the `else` region branch back to the parent operation.
4482	if (!point.isParent()) {
4483	regions.push_back(mlir::RegionSuccessor(getResults()));
4484	return;
4485	}
4486
4487	// Don't consider the else region if it is empty.
4488	regions.push_back(mlir::RegionSuccessor(&getThenRegion()));
4489
4490	// Don't consider the else region if it is empty.
4491	mlir::Region *elseRegion = &this->getElseRegion();
4492	if (elseRegion->empty())
4493	regions.push_back(mlir::RegionSuccessor());
4494	else
4495	regions.push_back(mlir::RegionSuccessor(elseRegion));
4496	}
4497
4498	void fir::IfOp::getEntrySuccessorRegions(
4499	llvm::ArrayRef<mlir::Attribute> operands,
4500	llvm::SmallVectorImpl<mlir::RegionSuccessor> &regions) {
4501	FoldAdaptor adaptor(operands);
4502	auto boolAttr =
4503	mlir::dyn_cast_or_null<mlir::BoolAttr>(adaptor.getCondition());
4504	if (!boolAttr || boolAttr.getValue())
4505	regions.emplace_back(&getThenRegion());
4506
4507	// If the else region is empty, execution continues after the parent op.
4508	if (!boolAttr || !boolAttr.getValue()) {
4509	if (!getElseRegion().empty())
4510	regions.emplace_back(&getElseRegion());
4511	else
4512	regions.emplace_back(getResults());
4513	}
4514	}
4515
4516	void fir::IfOp::getRegionInvocationBounds(
4517	llvm::ArrayRef<mlir::Attribute> operands,
4518	llvm::SmallVectorImpl<mlir::InvocationBounds> &invocationBounds) {
4519	if (auto cond = mlir::dyn_cast_or_null<mlir::BoolAttr>(operands[0])) {
4520	// If the condition is known, then one region is known to be executed once
4521	// and the other zero times.
4522	invocationBounds.emplace_back(0, cond.getValue() ? 1 : 0);
4523	invocationBounds.emplace_back(0, cond.getValue() ? 0 : 1);
4524	} else {
4525	// Non-constant condition. Each region may be executed 0 or 1 times.
4526	invocationBounds.assign(2, {0, 1});
4527	}
4528	}
4529
4530	mlir::ParseResult fir::IfOp::parse(mlir::OpAsmParser &parser,
4531	mlir::OperationState &result) {
4532	result.regions.reserve(2);
4533	mlir::Region *thenRegion = result.addRegion();
4534	mlir::Region *elseRegion = result.addRegion();
4535
4536	auto &builder = parser.getBuilder();
4537	mlir::OpAsmParser::UnresolvedOperand cond;
4538	mlir::Type i1Type = builder.getIntegerType(1);
4539	if (parser.parseOperand(cond) ||
4540	parser.resolveOperand(cond, i1Type, result.operands))
4541	return mlir::failure();
4542
4543	if (mlir::succeeded(
4544	parser.parseOptionalKeyword(getWeightsAttrAssemblyName()))) {
4545	if (parser.parseLParen())
4546	return mlir::failure();
4547	mlir::DenseI32ArrayAttr weights;
4548	if (parser.parseCustomAttributeWithFallback(weights, mlir::Type{}))
4549	return mlir::failure();
4550	if (weights)
4551	result.addAttribute(getRegionWeightsAttrName(result.name), weights);
4552	if (parser.parseRParen())
4553	return mlir::failure();
4554	}
4555
4556	if (parser.parseOptionalArrowTypeList(result.types))
4557	return mlir::failure();
4558
4559	if (parser.parseRegion(*thenRegion, {}, {}))
4560	return mlir::failure();
4561	fir::IfOp::ensureTerminator(*thenRegion, parser.getBuilder(),
4562	result.location);
4563
4564	if (mlir::succeeded(parser.parseOptionalKeyword("else"))) {
4565	if (parser.parseRegion(*elseRegion, {}, {}))
4566	return mlir::failure();
4567	fir::IfOp::ensureTerminator(*elseRegion, parser.getBuilder(),
4568	result.location);
4569	}
4570
4571	// Parse the optional attribute list.
4572	if (parser.parseOptionalAttrDict(result.attributes))
4573	return mlir::failure();
4574	return mlir::success();
4575	}
4576
4577	llvm::LogicalResult fir::IfOp::verify() {
4578	if (getNumResults() != 0 && getElseRegion().empty())
4579	return emitOpError("must have an else block if defining values");
4580
4581	return mlir::success();
4582	}
4583
4584	void fir::IfOp::print(mlir::OpAsmPrinter &p) {
4585	bool printBlockTerminators = false;
4586	p << ' ' << getCondition();
4587	if (auto weights = getRegionWeightsAttr()) {
4588	p << ' ' << getWeightsAttrAssemblyName() << '(';
4589	p.printStrippedAttrOrType(weights);
4590	p << ')';
4591	}
4592	if (!getResults().empty()) {
4593	p << " -> (" << getResultTypes() << ')';
4594	printBlockTerminators = true;
4595	}
4596	p << ' ';
4597	p.printRegion(getThenRegion(), /*printEntryBlockArgs=*/false,
4598	printBlockTerminators);
4599
4600	// Print the 'else' regions if it exists and has a block.
4601	auto &otherReg = getElseRegion();
4602	if (!otherReg.empty()) {
4603	p << " else ";
4604	p.printRegion(otherReg, /*printEntryBlockArgs=*/false,
4605	printBlockTerminators);
4606	}
4607	p.printOptionalAttrDict((*this)->getAttrs(),
4608	/*elideAttrs=*/{getRegionWeightsAttrName()});
4609	}
4610
4611	void fir::IfOp::resultToSourceOps(llvm::SmallVectorImpl<mlir::Value> &results,
4612	unsigned resultNum) {
4613	auto *term = getThenRegion().front().getTerminator();
4614	if (resultNum < term->getNumOperands())
4615	results.push_back(term->getOperand(resultNum));
4616	term = getElseRegion().front().getTerminator();
4617	if (resultNum < term->getNumOperands())
4618	results.push_back(term->getOperand(resultNum));
4619	}
4620
4621	//===----------------------------------------------------------------------===//
4622	// BoxOffsetOp
4623	//===----------------------------------------------------------------------===//
4624
4625	llvm::LogicalResult fir::BoxOffsetOp::verify() {
4626	auto boxType = mlir::dyn_cast_or_null<fir::BaseBoxType>(
4627	fir::dyn_cast_ptrEleTy(getBoxRef().getType()));
4628	mlir::Type boxCharType;
4629	if (!boxType) {
4630	boxCharType = mlir::dyn_cast_or_null<fir::BoxCharType>(
4631	fir::dyn_cast_ptrEleTy(getBoxRef().getType()));
4632	if (!boxCharType)
4633	return emitOpError("box_ref operand must have !fir.ref<!fir.box<T>> or "
4634	"!fir.ref<!fir.boxchar<k>> type");
4635	if (getField() == fir::BoxFieldAttr::derived_type)
4636	return emitOpError("cannot address derived_type field of a fir.boxchar");
4637	}
4638	if (getField() != fir::BoxFieldAttr::base_addr &&
4639	getField() != fir::BoxFieldAttr::derived_type)
4640	return emitOpError("cannot address provided field");
4641	if (getField() == fir::BoxFieldAttr::derived_type) {
4642	if (!fir::boxHasAddendum(boxType))
4643	return emitOpError("can only address derived_type field of derived type "
4644	"or unlimited polymorphic fir.box");
4645	}
4646	return mlir::success();
4647	}
4648
4649	void fir::BoxOffsetOp::build(mlir::OpBuilder &builder,
4650	mlir::OperationState &result, mlir::Value boxRef,
4651	fir::BoxFieldAttr field) {
4652	mlir::Type valueType =
4653	fir::unwrapPassByRefType(fir::unwrapRefType(boxRef.getType()));
4654	mlir::Type resultType = valueType;
4655	if (field == fir::BoxFieldAttr::base_addr)
4656	resultType = fir::LLVMPointerType::get(fir::ReferenceType::get(valueType));
4657	else if (field == fir::BoxFieldAttr::derived_type)
4658	resultType = fir::LLVMPointerType::get(
4659	fir::TypeDescType::get(fir::unwrapSequenceType(valueType)));
4660	build(builder, result, {resultType}, boxRef, field);
4661	}
4662
4663	//===----------------------------------------------------------------------===//
4664
4665	mlir::ParseResult fir::isValidCaseAttr(mlir::Attribute attr) {
4666	if (mlir::isa<mlir::UnitAttr, fir::ClosedIntervalAttr, fir::PointIntervalAttr,
4667	fir::LowerBoundAttr, fir::UpperBoundAttr>(attr))
4668	return mlir::success();
4669	return mlir::failure();
4670	}
4671
4672	unsigned fir::getCaseArgumentOffset(llvm::ArrayRef<mlir::Attribute> cases,
4673	unsigned dest) {
4674	unsigned o = 0;
4675	for (unsigned i = 0; i < dest; ++i) {
4676	auto &attr = cases[i];
4677	if (!mlir::dyn_cast_or_null<mlir::UnitAttr>(attr)) {
4678	++o;
4679	if (mlir::dyn_cast_or_null<fir::ClosedIntervalAttr>(attr))
4680	++o;
4681	}
4682	}
4683	return o;
4684	}
4685
4686	mlir::ParseResult
4687	fir::parseSelector(mlir::OpAsmParser &parser, mlir::OperationState &result,
4688	mlir::OpAsmParser::UnresolvedOperand &selector,
4689	mlir::Type &type) {
4690	if (parser.parseOperand(selector) || parser.parseColonType(type) ||
4691	parser.resolveOperand(selector, type, result.operands) ||
4692	parser.parseLSquare())
4693	return mlir::failure();
4694	return mlir::success();
4695	}
4696
4697	mlir::func::FuncOp fir::createFuncOp(mlir::Location loc, mlir::ModuleOp module,
4698	llvm::StringRef name,
4699	mlir::FunctionType type,
4700	llvm::ArrayRef<mlir::NamedAttribute> attrs,
4701	const mlir::SymbolTable *symbolTable) {
4702	if (symbolTable)
4703	if (auto f = symbolTable->lookup<mlir::func::FuncOp>(name)) {
4704	#ifdef EXPENSIVE_CHECKS
4705	assert(f == module.lookupSymbol<mlir::func::FuncOp>(name) &&
4706	"symbolTable and module out of sync");
4707	#endif
4708	return f;
4709	}
4710	if (auto f = module.lookupSymbol<mlir::func::FuncOp>(name))
4711	return f;
4712	mlir::OpBuilder modBuilder(module.getBodyRegion());
4713	modBuilder.setInsertionPointToEnd(module.getBody());
4714	auto result = modBuilder.create<mlir::func::FuncOp>(loc, name, type, attrs);
4715	result.setVisibility(mlir::SymbolTable::Visibility::Private);
4716	return result;
4717	}
4718
4719	fir::GlobalOp fir::createGlobalOp(mlir::Location loc, mlir::ModuleOp module,
4720	llvm::StringRef name, mlir::Type type,
4721	llvm::ArrayRef<mlir::NamedAttribute> attrs,
4722	const mlir::SymbolTable *symbolTable) {
4723	if (symbolTable)
4724	if (auto g = symbolTable->lookup<fir::GlobalOp>(name)) {
4725	#ifdef EXPENSIVE_CHECKS
4726	assert(g == module.lookupSymbol<fir::GlobalOp>(name) &&
4727	"symbolTable and module out of sync");
4728	#endif
4729	return g;
4730	}
4731	if (auto g = module.lookupSymbol<fir::GlobalOp>(name))
4732	return g;
4733	mlir::OpBuilder modBuilder(module.getBodyRegion());
4734	auto result = modBuilder.create<fir::GlobalOp>(loc, name, type, attrs);
4735	result.setVisibility(mlir::SymbolTable::Visibility::Private);
4736	return result;
4737	}
4738
4739	bool fir::hasHostAssociationArgument(mlir::func::FuncOp func) {
4740	if (auto allArgAttrs = func.getAllArgAttrs())
4741	for (auto attr : allArgAttrs)
4742	if (auto dict = mlir::dyn_cast_or_null<mlir::DictionaryAttr>(attr))
4743	if (dict.get(fir::getHostAssocAttrName()))
4744	return true;
4745	return false;
4746	}
4747
4748	// Test if value's definition has the specified set of
4749	// attributeNames. The value's definition is one of the operations
4750	// that are able to carry the Fortran variable attributes, e.g.
4751	// fir.alloca or fir.allocmem. Function arguments may also represent
4752	// value definitions and carry relevant attributes.
4753	//
4754	// If it is not possible to reach the limited set of definition
4755	// entities from the given value, then the function will return
4756	// std::nullopt. Otherwise, the definition is known and the return
4757	// value is computed as:
4758	// * if checkAny is true, then the function will return true
4759	// iff any of the attributeNames attributes is set on the definition.
4760	// * if checkAny is false, then the function will return true
4761	// iff all of the attributeNames attributes are set on the definition.
4762	static std::optional<bool>
4763	valueCheckFirAttributes(mlir::Value value,
4764	llvm::ArrayRef<llvm::StringRef> attributeNames,
4765	bool checkAny) {
4766	auto testAttributeSets = [&](llvm::ArrayRef<mlir::NamedAttribute> setAttrs,
4767	llvm::ArrayRef<llvm::StringRef> checkAttrs) {
4768	if (checkAny) {
4769	// Return true iff any of checkAttrs attributes is present
4770	// in setAttrs set.
4771	for (llvm::StringRef checkAttrName : checkAttrs)
4772	if (llvm::any_of(Range&: setAttrs, P: [&](mlir::NamedAttribute setAttr) {
4773	return setAttr.getName() == checkAttrName;
4774	}))
4775	return true;
4776
4777	return false;
4778	}
4779
4780	// Return true iff all attributes from checkAttrs are present
4781	// in setAttrs set.
4782	for (mlir::StringRef checkAttrName : checkAttrs)
4783	if (llvm::none_of(Range&: setAttrs, P: [&](mlir::NamedAttribute setAttr) {
4784	return setAttr.getName() == checkAttrName;
4785	}))
4786	return false;
4787
4788	return true;
4789	};
4790	// If this is a fir.box that was loaded, the fir attributes will be on the
4791	// related fir.ref<fir.box> creation.
4792	if (mlir::isa<fir::BoxType>(value.getType()))
4793	if (auto definingOp = value.getDefiningOp())
4794	if (auto loadOp = mlir::dyn_cast<fir::LoadOp>(definingOp))
4795	value = loadOp.getMemref();
4796	// If this is a function argument, look in the argument attributes.
4797	if (auto blockArg = mlir::dyn_cast<mlir::BlockArgument>(Val&: value)) {
4798	if (blockArg.getOwner() && blockArg.getOwner()->isEntryBlock())
4799	if (auto funcOp = mlir::dyn_cast<mlir::func::FuncOp>(
4800	Val: blockArg.getOwner()->getParentOp()))
4801	return testAttributeSets(
4802	mlir::cast<mlir::FunctionOpInterface>(Val&: *funcOp).getArgAttrs(
4803	index: blockArg.getArgNumber()),
4804	attributeNames);
4805
4806	// If it is not a function argument, the attributes are unknown.
4807	return std::nullopt;
4808	}
4809
4810	if (auto definingOp = value.getDefiningOp()) {
4811	// If this is an allocated value, look at the allocation attributes.
4812	if (mlir::isa<fir::AllocMemOp>(definingOp) ||
4813	mlir::isa<fir::AllocaOp>(definingOp))
4814	return testAttributeSets(definingOp->getAttrs(), attributeNames);
4815	// If this is an imported global, look at AddrOfOp and GlobalOp attributes.
4816	// Both operations are looked at because use/host associated variable (the
4817	// AddrOfOp) can have ASYNCHRONOUS/VOLATILE attributes even if the ultimate
4818	// entity (the globalOp) does not have them.
4819	if (auto addressOfOp = mlir::dyn_cast<fir::AddrOfOp>(definingOp)) {
4820	if (testAttributeSets(addressOfOp->getAttrs(), attributeNames))
4821	return true;
4822	if (auto module = definingOp->getParentOfType<mlir::ModuleOp>())
4823	if (auto globalOp =
4824	module.lookupSymbol<fir::GlobalOp>(addressOfOp.getSymbol()))
4825	return testAttributeSets(globalOp->getAttrs(), attributeNames);
4826	}
4827	}
4828	// TODO: Construct associated entities attributes. Decide where the fir
4829	// attributes must be placed/looked for in this case.
4830	return std::nullopt;
4831	}
4832
4833	bool fir::valueMayHaveFirAttributes(
4834	mlir::Value value, llvm::ArrayRef<llvm::StringRef> attributeNames) {
4835	std::optional<bool> mayHaveAttr =
4836	valueCheckFirAttributes(value, attributeNames, /*checkAny=*/true);
4837	return mayHaveAttr.value_or(true);
4838	}
4839
4840	bool fir::valueHasFirAttribute(mlir::Value value,
4841	llvm::StringRef attributeName) {
4842	std::optional<bool> mayHaveAttr =
4843	valueCheckFirAttributes(value, {attributeName}, /*checkAny=*/false);
4844	return mayHaveAttr.value_or(false);
4845	}
4846
4847	bool fir::anyFuncArgsHaveAttr(mlir::func::FuncOp func, llvm::StringRef attr) {
4848	for (unsigned i = 0, end = func.getNumArguments(); i < end; ++i)
4849	if (func.getArgAttr(i, attr))
4850	return true;
4851	return false;
4852	}
4853
4854	std::optional<std::int64_t> fir::getIntIfConstant(mlir::Value value) {
4855	if (auto *definingOp = value.getDefiningOp()) {
4856	if (auto cst = mlir::dyn_cast<mlir::arith::ConstantOp>(definingOp))
4857	if (auto intAttr = mlir::dyn_cast<mlir::IntegerAttr>(cst.getValue()))
4858	return intAttr.getInt();
4859	if (auto llConstOp = mlir::dyn_cast<mlir::LLVM::ConstantOp>(definingOp))
4860	if (auto attr = mlir::dyn_cast<mlir::IntegerAttr>(llConstOp.getValue()))
4861	return attr.getValue().getSExtValue();
4862	}
4863	return {};
4864	}
4865
4866	bool fir::isDummyArgument(mlir::Value v) {
4867	auto blockArg{mlir::dyn_cast<mlir::BlockArgument>(v)};
4868	if (!blockArg) {
4869	auto defOp = v.getDefiningOp();
4870	if (defOp) {
4871	if (auto declareOp = mlir::dyn_cast<fir::DeclareOp>(defOp))
4872	if (declareOp.getDummyScope())
4873	return true;
4874	}
4875	return false;
4876	}
4877
4878	auto *owner{blockArg.getOwner()};
4879	return owner->isEntryBlock() &&
4880	mlir::isa<mlir::FunctionOpInterface>(owner->getParentOp());
4881	}
4882
4883	mlir::Type fir::applyPathToType(mlir::Type eleTy, mlir::ValueRange path) {
4884	for (auto i = path.begin(), end = path.end(); eleTy && i < end;) {
4885	eleTy = llvm::TypeSwitch<mlir::Type, mlir::Type>(eleTy)
4886	.Case<fir::RecordType>([&](fir::RecordType ty) {
4887	if (auto *op = (*i++).getDefiningOp()) {
4888	if (auto off = mlir::dyn_cast<fir::FieldIndexOp>(op))
4889	return ty.getType(off.getFieldName());
4890	if (auto off = mlir::dyn_cast<mlir::arith::ConstantOp>(op))
4891	return ty.getType(fir::toInt(off));
4892	}
4893	return mlir::Type{};
4894	})
4895	.Case<fir::SequenceType>([&](fir::SequenceType ty) {
4896	bool valid = true;
4897	const auto rank = ty.getDimension();
4898	for (std::remove_const_t<decltype(rank)> ii = 0;
4899	valid && ii < rank; ++ii)
4900	valid = i < end && fir::isa_integer((*i++).getType());
4901	return valid ? ty.getEleTy() : mlir::Type{};
4902	})
4903	.Case<mlir::TupleType>([&](mlir::TupleType ty) {
4904	if (auto *op = (*i++).getDefiningOp())
4905	if (auto off = mlir::dyn_cast<mlir::arith::ConstantOp>(op))
4906	return ty.getType(fir::toInt(off));
4907	return mlir::Type{};
4908	})
4909	.Case<mlir::ComplexType>([&](mlir::ComplexType ty) {
4910	if (fir::isa_integer((*i++).getType()))
4911	return ty.getElementType();
4912	return mlir::Type{};
4913	})
4914	.Default([&](const auto &) { return mlir::Type{}; });
4915	}
4916	return eleTy;
4917	}
4918
4919	bool fir::reboxPreservesContinuity(fir::ReboxOp rebox,
4920	bool mayHaveNonDefaultLowerBounds,
4921	bool checkWhole) {
4922	// If slicing is not involved, then the rebox does not affect
4923	// the continuity of the array.
4924	auto sliceArg = rebox.getSlice();
4925	if (!sliceArg)
4926	return true;
4927
4928	if (auto sliceOp =
4929	mlir::dyn_cast_or_null<fir::SliceOp>(sliceArg.getDefiningOp()))
4930	return isContiguousArraySlice(sliceOp, checkWhole, rebox.getBox(),
4931	mayHaveNonDefaultLowerBounds);
4932
4933	return false;
4934	}
4935
4936	std::optional<int64_t> fir::getAllocaByteSize(fir::AllocaOp alloca,
4937	const mlir::DataLayout &dl,
4938	const fir::KindMapping &kindMap) {
4939	mlir::Type type = alloca.getInType();
4940	// TODO: should use the constant operands when all info is not available in
4941	// the type.
4942	if (!alloca.isDynamic())
4943	if (auto sizeAndAlignment =
4944	getTypeSizeAndAlignment(alloca.getLoc(), type, dl, kindMap))
4945	return sizeAndAlignment->first;
4946	return std::nullopt;
4947	}
4948
4949	//===----------------------------------------------------------------------===//
4950	// DeclareOp
4951	//===----------------------------------------------------------------------===//
4952
4953	llvm::LogicalResult fir::DeclareOp::verify() {
4954	auto fortranVar =
4955	mlir::cast<fir::FortranVariableOpInterface>(this->getOperation());
4956	return fortranVar.verifyDeclareLikeOpImpl(getMemref());
4957	}
4958
4959	//===----------------------------------------------------------------------===//
4960	// PackArrayOp
4961	//===----------------------------------------------------------------------===//
4962
4963	llvm::LogicalResult fir::PackArrayOp::verify() {
4964	mlir::Type arrayType = getArray().getType();
4965	if (!validTypeParams(arrayType, getTypeparams(), /*allowParamsForBox=*/true))
4966	return emitOpError("invalid type parameters");
4967
4968	if (getInnermost() && fir::getBoxRank(arrayType) == 1)
4969	return emitOpError(
4970	"'innermost' is invalid for 1D arrays, use 'whole' instead");
4971	return mlir::success();
4972	}
4973
4974	void fir::PackArrayOp::getEffects(
4975	llvm::SmallVectorImpl<
4976	mlir::SideEffects::EffectInstance<mlir::MemoryEffects::Effect>>
4977	&effects) {
4978	if (getStack())
4979	effects.emplace_back(
4980	mlir::MemoryEffects::Allocate::get(),
4981	mlir::SideEffects::AutomaticAllocationScopeResource::get());
4982	else
4983	effects.emplace_back(mlir::MemoryEffects::Allocate::get(),
4984	mlir::SideEffects::DefaultResource::get());
4985
4986	if (!getNoCopy())
4987	effects.emplace_back(mlir::MemoryEffects::Read::get(),
4988	mlir::SideEffects::DefaultResource::get());
4989	}
4990
4991	static mlir::ParseResult
4992	parsePackArrayConstraints(mlir::OpAsmParser &parser, mlir::IntegerAttr &maxSize,
4993	mlir::IntegerAttr &maxElementSize,
4994	mlir::IntegerAttr &minStride) {
4995	mlir::OperationName opName = mlir::OperationName(
4996	fir::PackArrayOp::getOperationName(), parser.getContext());
4997	struct {
4998	llvm::StringRef name;
4999	mlir::IntegerAttr &ref;
5000	} attributes[] = {
5001	{fir::PackArrayOp::getMaxSizeAttrName(opName), maxSize},
5002	{fir::PackArrayOp::getMaxElementSizeAttrName(opName), maxElementSize},
5003	{fir::PackArrayOp::getMinStrideAttrName(opName), minStride}};
5004
5005	mlir::NamedAttrList parsedAttrs;
5006	if (succeeded(Result: parser.parseOptionalAttrDict(result&: parsedAttrs))) {
5007	for (auto parsedAttr : parsedAttrs) {
5008	for (auto opAttr : attributes) {
5009	if (parsedAttr.getName() == opAttr.name)
5010	opAttr.ref = mlir::cast<mlir::IntegerAttr>(parsedAttr.getValue());
5011	}
5012	}
5013	return mlir::success();
5014	}
5015	return mlir::failure();
5016	}
5017
5018	static void printPackArrayConstraints(mlir::OpAsmPrinter &p,
5019	fir::PackArrayOp &op,
5020	const mlir::IntegerAttr &maxSize,
5021	const mlir::IntegerAttr &maxElementSize,
5022	const mlir::IntegerAttr &minStride) {
5023	llvm::SmallVector<mlir::NamedAttribute> attributes;
5024	if (maxSize)
5025	attributes.emplace_back(op.getMaxSizeAttrName(), maxSize);
5026	if (maxElementSize)
5027	attributes.emplace_back(op.getMaxElementSizeAttrName(), maxElementSize);
5028	if (minStride)
5029	attributes.emplace_back(op.getMinStrideAttrName(), minStride);
5030
5031	p.printOptionalAttrDict(attrs: attributes);
5032	}
5033
5034	//===----------------------------------------------------------------------===//
5035	// UnpackArrayOp
5036	//===----------------------------------------------------------------------===//
5037
5038	llvm::LogicalResult fir::UnpackArrayOp::verify() {
5039	if (auto packOp = getTemp().getDefiningOp<fir::PackArrayOp>())
5040	if (getStack() != packOp.getStack())
5041	return emitOpError() << "the pack operation uses different memory for "
5042	"the temporary (stack vs heap): "
5043	<< *packOp.getOperation() << "\n";
5044	return mlir::success();
5045	}
5046
5047	void fir::UnpackArrayOp::getEffects(
5048	llvm::SmallVectorImpl<
5049	mlir::SideEffects::EffectInstance<mlir::MemoryEffects::Effect>>
5050	&effects) {
5051	if (getStack())
5052	effects.emplace_back(
5053	mlir::MemoryEffects::Free::get(),
5054	mlir::SideEffects::AutomaticAllocationScopeResource::get());
5055	else
5056	effects.emplace_back(mlir::MemoryEffects::Free::get(),
5057	mlir::SideEffects::DefaultResource::get());
5058
5059	if (!getNoCopy())
5060	effects.emplace_back(mlir::MemoryEffects::Write::get(),
5061	mlir::SideEffects::DefaultResource::get());
5062	}
5063
5064	//===----------------------------------------------------------------------===//
5065	// IsContiguousBoxOp
5066	//===----------------------------------------------------------------------===//
5067
5068	namespace {
5069	struct SimplifyIsContiguousBoxOp
5070	: public mlir::OpRewritePattern<fir::IsContiguousBoxOp> {
5071	using mlir::OpRewritePattern<fir::IsContiguousBoxOp>::OpRewritePattern;
5072	mlir::LogicalResult
5073	matchAndRewrite(fir::IsContiguousBoxOp op,
5074	mlir::PatternRewriter &rewriter) const override;
5075	};
5076	} // namespace
5077
5078	mlir::LogicalResult SimplifyIsContiguousBoxOp::matchAndRewrite(
5079	fir::IsContiguousBoxOp op, mlir::PatternRewriter &rewriter) const {
5080	auto boxType = mlir::cast<fir::BaseBoxType>(op.getBox().getType());
5081	// Nothing to do for assumed-rank arrays and !fir.box<none>.
5082	if (boxType.isAssumedRank() || fir::isBoxNone(boxType))
5083	return mlir::failure();
5084
5085	if (fir::getBoxRank(boxType) == 0) {
5086	// Scalars are always contiguous.
5087	mlir::Type i1Type = rewriter.getI1Type();
5088	rewriter.replaceOpWithNewOp<mlir::arith::ConstantOp>(
5089	op, i1Type, rewriter.getIntegerAttr(i1Type, 1));
5090	return mlir::success();
5091	}
5092
5093	// TODO: support more patterns, e.g. a result of fir.embox without
5094	// the slice is contiguous. We can add fir::isSimplyContiguous(box)
5095	// that walks def-use to figure it out.
5096	return mlir::failure();
5097	}
5098
5099	void fir::IsContiguousBoxOp::getCanonicalizationPatterns(
5100	mlir::RewritePatternSet &patterns, mlir::MLIRContext *context) {
5101	patterns.add<SimplifyIsContiguousBoxOp>(context);
5102	}
5103
5104	//===----------------------------------------------------------------------===//
5105	// BoxTotalElementsOp
5106	//===----------------------------------------------------------------------===//
5107
5108	namespace {
5109	struct SimplifyBoxTotalElementsOp
5110	: public mlir::OpRewritePattern<fir::BoxTotalElementsOp> {
5111	using mlir::OpRewritePattern<fir::BoxTotalElementsOp>::OpRewritePattern;
5112	mlir::LogicalResult
5113	matchAndRewrite(fir::BoxTotalElementsOp op,
5114	mlir::PatternRewriter &rewriter) const override;
5115	};
5116	} // namespace
5117
5118	mlir::LogicalResult SimplifyBoxTotalElementsOp::matchAndRewrite(
5119	fir::BoxTotalElementsOp op, mlir::PatternRewriter &rewriter) const {
5120	auto boxType = mlir::cast<fir::BaseBoxType>(op.getBox().getType());
5121	// Nothing to do for assumed-rank arrays and !fir.box<none>.
5122	if (boxType.isAssumedRank() || fir::isBoxNone(boxType))
5123	return mlir::failure();
5124
5125	if (fir::getBoxRank(boxType) == 0) {
5126	// Scalar: 1 element.
5127	rewriter.replaceOpWithNewOp<mlir::arith::ConstantOp>(
5128	op, op.getType(), rewriter.getIntegerAttr(op.getType(), 1));
5129	return mlir::success();
5130	}
5131
5132	// TODO: support more cases, e.g. !fir.box<!fir.array<10xi32>>.
5133	return mlir::failure();
5134	}
5135
5136	void fir::BoxTotalElementsOp::getCanonicalizationPatterns(
5137	mlir::RewritePatternSet &patterns, mlir::MLIRContext *context) {
5138	patterns.add<SimplifyBoxTotalElementsOp>(context);
5139	}
5140
5141	//===----------------------------------------------------------------------===//
5142	// LocalitySpecifierOp
5143	//===----------------------------------------------------------------------===//
5144
5145	// TODO This is a copy of omp::PrivateClauseOp::verifiyRegions(). Once we find a
5146	// solution to merge both ops into one this duplication will not be needed. See:
5147	// https://discourse.llvm.org/t/dialect-for-data-locality-sharing-specifiers-clauses-in-openmp-openacc-and-do-concurrent/86108.
5148	llvm::LogicalResult fir::LocalitySpecifierOp::verifyRegions() {
5149	mlir::Type argType = getArgType();
5150	auto verifyTerminator = [&](mlir::Operation *terminator,
5151	bool yieldsValue) -> llvm::LogicalResult {
5152	if (!terminator->getBlock()->getSuccessors().empty())
5153	return llvm::success();
5154
5155	if (!llvm::isa<fir::YieldOp>(terminator))
5156	return mlir::emitError(terminator->getLoc())
5157	<< "expected exit block terminator to be an `fir.yield` op.";
5158
5159	YieldOp yieldOp = llvm::cast<YieldOp>(terminator);
5160	mlir::TypeRange yieldedTypes = yieldOp.getResults().getTypes();
5161
5162	if (!yieldsValue) {
5163	if (yieldedTypes.empty())
5164	return llvm::success();
5165
5166	return mlir::emitError(terminator->getLoc())
5167	<< "Did not expect any values to be yielded.";
5168	}
5169
5170	if (yieldedTypes.size() == 1 && yieldedTypes.front() == argType)
5171	return llvm::success();
5172
5173	auto error = mlir::emitError(yieldOp.getLoc())
5174	<< "Invalid yielded value. Expected type: " << argType
5175	<< ", got: ";
5176
5177	if (yieldedTypes.empty())
5178	error << "None";
5179	else
5180	error << yieldedTypes;
5181
5182	return error;
5183	};
5184
5185	auto verifyRegion = [&](mlir::Region &region, unsigned expectedNumArgs,
5186	llvm::StringRef regionName,
5187	bool yieldsValue) -> llvm::LogicalResult {
5188	assert(!region.empty());
5189
5190	if (region.getNumArguments() != expectedNumArgs)
5191	return mlir::emitError(region.getLoc())
5192	<< "`" << regionName << "`: "
5193	<< "expected " << expectedNumArgs
5194	<< " region arguments, got: " << region.getNumArguments();
5195
5196	for (mlir::Block &block : region) {
5197	// MLIR will verify the absence of the terminator for us.
5198	if (!block.mightHaveTerminator())
5199	continue;
5200
5201	if (failed(verifyTerminator(block.getTerminator(), yieldsValue)))
5202	return llvm::failure();
5203	}
5204
5205	return llvm::success();
5206	};
5207
5208	// Ensure all of the region arguments have the same type
5209	for (mlir::Region *region : getRegions())
5210	for (mlir::Type ty : region->getArgumentTypes())
5211	if (ty != argType)
5212	return emitError() << "Region argument type mismatch: got " << ty
5213	<< " expected " << argType << ".";
5214
5215	mlir::Region &initRegion = getInitRegion();
5216	if (!initRegion.empty() &&
5217	failed(verifyRegion(getInitRegion(), /*expectedNumArgs=*/2, "init",
5218	/*yieldsValue=*/true)))
5219	return llvm::failure();
5220
5221	LocalitySpecifierType dsType = getLocalitySpecifierType();
5222
5223	if (dsType == LocalitySpecifierType::Local && !getCopyRegion().empty())
5224	return emitError("`local` specifiers do not require a `copy` region.");
5225
5226	if (dsType == LocalitySpecifierType::LocalInit && getCopyRegion().empty())
5227	return emitError(
5228	"`local_init` specifiers require at least a `copy` region.");
5229
5230	if (dsType == LocalitySpecifierType::LocalInit &&
5231	failed(verifyRegion(getCopyRegion(), /*expectedNumArgs=*/2, "copy",
5232	/*yieldsValue=*/true)))
5233	return llvm::failure();
5234
5235	if (!getDeallocRegion().empty() &&
5236	failed(verifyRegion(getDeallocRegion(), /*expectedNumArgs=*/1, "dealloc",
5237	/*yieldsValue=*/false)))
5238	return llvm::failure();
5239
5240	return llvm::success();
5241	}
5242
5243	// TODO This is a copy of omp::DeclareReductionOp::verifiyRegions(). Once we
5244	// find a solution to merge both ops into one this duplication will not be
5245	// needed.
5246	mlir::LogicalResult fir::DeclareReductionOp::verifyRegions() {
5247	if (!getAllocRegion().empty()) {
5248	for (YieldOp yieldOp : getAllocRegion().getOps<YieldOp>()) {
5249	if (yieldOp.getResults().size() != 1 ||
5250	yieldOp.getResults().getTypes()[0] != getType())
5251	return emitOpError() << "expects alloc region to yield a value "
5252	"of the reduction type";
5253	}
5254	}
5255
5256	if (getInitializerRegion().empty())
5257	return emitOpError() << "expects non-empty initializer region";
5258	mlir::Block &initializerEntryBlock = getInitializerRegion().front();
5259
5260	if (initializerEntryBlock.getNumArguments() == 1) {
5261	if (!getAllocRegion().empty())
5262	return emitOpError() << "expects two arguments to the initializer region "
5263	"when an allocation region is used";
5264	} else if (initializerEntryBlock.getNumArguments() == 2) {
5265	if (getAllocRegion().empty())
5266	return emitOpError() << "expects one argument to the initializer region "
5267	"when no allocation region is used";
5268	} else {
5269	return emitOpError()
5270	<< "expects one or two arguments to the initializer region";
5271	}
5272
5273	for (mlir::Value arg : initializerEntryBlock.getArguments())
5274	if (arg.getType() != getType())
5275	return emitOpError() << "expects initializer region argument to match "
5276	"the reduction type";
5277
5278	for (YieldOp yieldOp : getInitializerRegion().getOps<YieldOp>()) {
5279	if (yieldOp.getResults().size() != 1 ||
5280	yieldOp.getResults().getTypes()[0] != getType())
5281	return emitOpError() << "expects initializer region to yield a value "
5282	"of the reduction type";
5283	}
5284
5285	if (getReductionRegion().empty())
5286	return emitOpError() << "expects non-empty reduction region";
5287	mlir::Block &reductionEntryBlock = getReductionRegion().front();
5288	if (reductionEntryBlock.getNumArguments() != 2 ||
5289	reductionEntryBlock.getArgumentTypes()[0] !=
5290	reductionEntryBlock.getArgumentTypes()[1] ||
5291	reductionEntryBlock.getArgumentTypes()[0] != getType())
5292	return emitOpError() << "expects reduction region with two arguments of "
5293	"the reduction type";
5294	for (YieldOp yieldOp : getReductionRegion().getOps<YieldOp>()) {
5295	if (yieldOp.getResults().size() != 1 ||
5296	yieldOp.getResults().getTypes()[0] != getType())
5297	return emitOpError() << "expects reduction region to yield a value "
5298	"of the reduction type";
5299	}
5300
5301	if (!getAtomicReductionRegion().empty()) {
5302	mlir::Block &atomicReductionEntryBlock = getAtomicReductionRegion().front();
5303	if (atomicReductionEntryBlock.getNumArguments() != 2 ||
5304	atomicReductionEntryBlock.getArgumentTypes()[0] !=
5305	atomicReductionEntryBlock.getArgumentTypes()[1])
5306	return emitOpError() << "expects atomic reduction region with two "
5307	"arguments of the same type";
5308	}
5309
5310	if (getCleanupRegion().empty())
5311	return mlir::success();
5312	mlir::Block &cleanupEntryBlock = getCleanupRegion().front();
5313	if (cleanupEntryBlock.getNumArguments() != 1 ||
5314	cleanupEntryBlock.getArgument(0).getType() != getType())
5315	return emitOpError() << "expects cleanup region with one argument "
5316	"of the reduction type";
5317
5318	return mlir::success();
5319	}
5320
5321	//===----------------------------------------------------------------------===//
5322	// DoConcurrentOp
5323	//===----------------------------------------------------------------------===//
5324
5325	llvm::LogicalResult fir::DoConcurrentOp::verify() {
5326	mlir::Block *body = getBody();
5327
5328	if (body->empty())
5329	return emitOpError("body cannot be empty");
5330
5331	if (!body->mightHaveTerminator() ||
5332	!mlir::isa<fir::DoConcurrentLoopOp>(body->getTerminator()))
5333	return emitOpError("must be terminated by 'fir.do_concurrent.loop'");
5334
5335	return mlir::success();
5336	}
5337
5338	//===----------------------------------------------------------------------===//
5339	// DoConcurrentLoopOp
5340	//===----------------------------------------------------------------------===//
5341
5342	static mlir::ParseResult parseSpecifierList(
5343	mlir::OpAsmParser &parser, mlir::OperationState &result,
5344	llvm::StringRef specifierKeyword, llvm::StringRef symsAttrName,
5345	llvm::SmallVectorImpl<mlir::OpAsmParser::Argument> &regionArgs,
5346	llvm::SmallVectorImpl<mlir::Type> &regionArgTypes,
5347	int32_t &numSpecifierOperands, bool isReduce = false) {
5348	auto &builder = parser.getBuilder();
5349	llvm::SmallVector<mlir::OpAsmParser::UnresolvedOperand> specifierOperands;
5350
5351	if (failed(Result: parser.parseOptionalKeyword(keyword: specifierKeyword)))
5352	return mlir::success();
5353
5354	std::size_t oldArgTypesSize = regionArgTypes.size();
5355	if (failed(Result: parser.parseLParen()))
5356	return mlir::failure();
5357
5358	llvm::SmallVector<bool> isByRefVec;
5359	llvm::SmallVector<mlir::SymbolRefAttr> spceifierSymbolVec;
5360	llvm::SmallVector<fir::ReduceAttr> attributes;
5361
5362	if (failed(Result: parser.parseCommaSeparatedList(parseElementFn: [&]() {
5363	if (isReduce)
5364	isByRefVec.push_back(
5365	Elt: parser.parseOptionalKeyword(keyword: "byref").succeeded());
5366
5367	if (failed(Result: parser.parseAttribute(result&: spceifierSymbolVec.emplace_back())))
5368	return mlir::failure();
5369
5370	if (isReduce &&
5371	failed(parser.parseAttribute(attributes.emplace_back())))
5372	return mlir::failure();
5373
5374	if (parser.parseOperand(result&: specifierOperands.emplace_back()) ||
5375	parser.parseArrow() ||
5376	parser.parseArgument(result&: regionArgs.emplace_back()))
5377	return mlir::failure();
5378
5379	return mlir::success();
5380	})))
5381	return mlir::failure();
5382
5383	if (failed(Result: parser.parseColon()))
5384	return mlir::failure();
5385
5386	if (failed(Result: parser.parseCommaSeparatedList(parseElementFn: [&]() {
5387	if (failed(Result: parser.parseType(result&: regionArgTypes.emplace_back())))
5388	return mlir::failure();
5389
5390	return mlir::success();
5391	})))
5392	return mlir::failure();
5393
5394	if (regionArgs.size() != regionArgTypes.size())
5395	return parser.emitError(loc: parser.getNameLoc(), message: "mismatch in number of " +
5396	specifierKeyword.str() +
5397	" arg and types");
5398
5399	if (failed(Result: parser.parseRParen()))
5400	return mlir::failure();
5401
5402	for (auto operandType :
5403	llvm::zip_equal(t&: specifierOperands,
5404	u: llvm::drop_begin(RangeOrContainer&: regionArgTypes, N: oldArgTypesSize)))
5405	if (parser.resolveOperand(operand: std::get<0>(t&: operandType),
5406	type: std::get<1>(t&: operandType), result&: result.operands))
5407	return mlir::failure();
5408
5409	if (isReduce)
5410	result.addAttribute(
5411	fir::DoConcurrentLoopOp::getReduceByrefAttrName(result.name),
5412	isByRefVec.empty()
5413	? nullptr
5414	: mlir::DenseBoolArrayAttr::get(builder.getContext(), isByRefVec));
5415
5416	llvm::SmallVector<mlir::Attribute> symbolAttrs(spceifierSymbolVec.begin(),
5417	spceifierSymbolVec.end());
5418	result.addAttribute(name: symsAttrName, attr: builder.getArrayAttr(value: symbolAttrs));
5419
5420	if (isReduce) {
5421	llvm::SmallVector<mlir::Attribute> arrayAttr(attributes.begin(),
5422	attributes.end());
5423	result.addAttribute(
5424	fir::DoConcurrentLoopOp::getReduceAttrsAttrName(result.name),
5425	builder.getArrayAttr(arrayAttr));
5426	}
5427
5428	numSpecifierOperands = specifierOperands.size();
5429
5430	return mlir::success();
5431	}
5432
5433	mlir::ParseResult fir::DoConcurrentLoopOp::parse(mlir::OpAsmParser &parser,
5434	mlir::OperationState &result) {
5435	auto &builder = parser.getBuilder();
5436	// Parse an opening `(` followed by induction variables followed by `)`
5437	llvm::SmallVector<mlir::OpAsmParser::Argument, 4> regionArgs;
5438
5439	if (parser.parseArgumentList(regionArgs, mlir::OpAsmParser::Delimiter::Paren))
5440	return mlir::failure();
5441
5442	llvm::SmallVector<mlir::Type> argTypes(regionArgs.size(),
5443	builder.getIndexType());
5444
5445	// Parse loop bounds.
5446	llvm::SmallVector<mlir::OpAsmParser::UnresolvedOperand, 4> lower;
5447	if (parser.parseEqual() ||
5448	parser.parseOperandList(lower, regionArgs.size(),
5449	mlir::OpAsmParser::Delimiter::Paren) ||
5450	parser.resolveOperands(lower, builder.getIndexType(), result.operands))
5451	return mlir::failure();
5452
5453	llvm::SmallVector<mlir::OpAsmParser::UnresolvedOperand, 4> upper;
5454	if (parser.parseKeyword("to") ||
5455	parser.parseOperandList(upper, regionArgs.size(),
5456	mlir::OpAsmParser::Delimiter::Paren) ||
5457	parser.resolveOperands(upper, builder.getIndexType(), result.operands))
5458	return mlir::failure();
5459
5460	// Parse step values.
5461	llvm::SmallVector<mlir::OpAsmParser::UnresolvedOperand, 4> steps;
5462	if (parser.parseKeyword("step") ||
5463	parser.parseOperandList(steps, regionArgs.size(),
5464	mlir::OpAsmParser::Delimiter::Paren) ||
5465	parser.resolveOperands(steps, builder.getIndexType(), result.operands))
5466	return mlir::failure();
5467
5468	int32_t numLocalOperands = 0;
5469	if (failed(parseSpecifierList(parser, result, "local",
5470	getLocalSymsAttrName(result.name), regionArgs,
5471	argTypes, numLocalOperands)))
5472	return mlir::failure();
5473
5474	int32_t numReduceOperands = 0;
5475	if (failed(parseSpecifierList(
5476	parser, result, "reduce", getReduceSymsAttrName(result.name),
5477	regionArgs, argTypes, numReduceOperands, /*isReduce=*/true)))
5478	return mlir::failure();
5479
5480	// Set `operandSegmentSizes` attribute.
5481	result.addAttribute(
5482	DoConcurrentLoopOp::getOperandSegmentSizeAttr(),
5483	builder.getDenseI32ArrayAttr({static_cast<int32_t>(lower.size()),
5484	static_cast<int32_t>(upper.size()),
5485	static_cast<int32_t>(steps.size()),
5486	static_cast<int32_t>(numLocalOperands),
5487	static_cast<int32_t>(numReduceOperands)}));
5488
5489	// Now parse the body.
5490	for (auto [arg, type] : llvm::zip_equal(regionArgs, argTypes))
5491	arg.type = type;
5492
5493	mlir::Region *body = result.addRegion();
5494	if (parser.parseRegion(*body, regionArgs))
5495	return mlir::failure();
5496
5497	// Parse attributes.
5498	if (parser.parseOptionalAttrDict(result.attributes))
5499	return mlir::failure();
5500
5501	return mlir::success();
5502	}
5503
5504	void fir::DoConcurrentLoopOp::print(mlir::OpAsmPrinter &p) {
5505	p << " (" << getBody()->getArguments().slice(0, getNumInductionVars())
5506	<< ") = (" << getLowerBound() << ") to (" << getUpperBound() << ") step ("
5507	<< getStep() << ")";
5508
5509	if (!getLocalVars().empty()) {
5510	p << " local(";
5511	llvm::interleaveComma(llvm::zip_equal(getLocalSymsAttr(), getLocalVars(),
5512	getRegionLocalArgs()),
5513	p, [&](auto it) {
5514	p << std::get<0>(it) << " " << std::get<1>(it)
5515	<< " -> " << std::get<2>(it);
5516	});
5517	p << " : ";
5518	llvm::interleaveComma(getLocalVars(), p,
5519	[&](auto it) { p << it.getType(); });
5520	p << ")";
5521	}
5522
5523	if (!getReduceVars().empty()) {
5524	p << " reduce(";
5525	llvm::interleaveComma(
5526	llvm::zip_equal(getReduceByrefAttr().asArrayRef(), getReduceSymsAttr(),
5527	getReduceAttrsAttr(), getReduceVars(),
5528	getRegionReduceArgs()),
5529	p, [&](auto it) {
5530	if (std::get<0>(it))
5531	p << "byref ";
5532
5533	p << std::get<1>(it) << " " << std::get<2>(it) << " "
5534	<< std::get<3>(it) << " -> " << std::get<4>(it);
5535	});
5536	p << " : ";
5537	llvm::interleaveComma(getReduceVars(), p,
5538	[&](auto it) { p << it.getType(); });
5539	p << ")";
5540	}
5541
5542	p << ' ';
5543	p.printRegion(getRegion(), /*printEntryBlockArgs=*/false);
5544	p.printOptionalAttrDict(
5545	(*this)->getAttrs(),
5546	/*elidedAttrs=*/{DoConcurrentLoopOp::getOperandSegmentSizeAttr(),
5547	DoConcurrentLoopOp::getLocalSymsAttrName(),
5548	DoConcurrentLoopOp::getReduceSymsAttrName(),
5549	DoConcurrentLoopOp::getReduceAttrsAttrName(),
5550	DoConcurrentLoopOp::getReduceByrefAttrName()});
5551	}
5552
5553	llvm::SmallVector<mlir::Region *> fir::DoConcurrentLoopOp::getLoopRegions() {
5554	return {&getRegion()};
5555	}
5556
5557	llvm::LogicalResult fir::DoConcurrentLoopOp::verify() {
5558	mlir::Operation::operand_range lbValues = getLowerBound();
5559	mlir::Operation::operand_range ubValues = getUpperBound();
5560	mlir::Operation::operand_range stepValues = getStep();
5561	mlir::Operation::operand_range localVars = getLocalVars();
5562	mlir::Operation::operand_range reduceVars = getReduceVars();
5563
5564	if (lbValues.empty())
5565	return emitOpError(
5566	"needs at least one tuple element for lowerBound, upperBound and step");
5567
5568	if (lbValues.size() != ubValues.size() ||
5569	ubValues.size() != stepValues.size())
5570	return emitOpError("different number of tuple elements for lowerBound, "
5571	"upperBound or step");
5572
5573	// Check that the body defines the same number of block arguments as the
5574	// number of tuple elements in step.
5575	mlir::Block *body = getBody();
5576	unsigned numIndVarArgs =
5577	body->getNumArguments() - localVars.size() - reduceVars.size();
5578
5579	if (numIndVarArgs != stepValues.size())
5580	return emitOpError() << "expects the same number of induction variables: "
5581	<< body->getNumArguments()
5582	<< " as bound and step values: " << stepValues.size();
5583	for (auto arg : body->getArguments().slice(0, numIndVarArgs))
5584	if (!arg.getType().isIndex())
5585	return emitOpError(
5586	"expects arguments for the induction variable to be of index type");
5587
5588	auto reduceAttrs = getReduceAttrsAttr();
5589	if (getNumReduceOperands() != (reduceAttrs ? reduceAttrs.size() : 0))
5590	return emitOpError(
5591	"mismatch in number of reduction variables and reduction attributes");
5592
5593	return mlir::success();
5594	}
5595
5596	std::optional<llvm::SmallVector<mlir::Value>>
5597	fir::DoConcurrentLoopOp::getLoopInductionVars() {
5598	return llvm::SmallVector<mlir::Value>{
5599	getBody()->getArguments().slice(0, getLowerBound().size())};
5600	}
5601
5602	//===----------------------------------------------------------------------===//
5603	// FIROpsDialect
5604	//===----------------------------------------------------------------------===//
5605
5606	void fir::FIROpsDialect::registerOpExternalInterfaces() {
5607	// Attach default declare target interfaces to operations which can be marked
5608	// as declare target.
5609	fir::GlobalOp::attachInterface<
5610	mlir::omp::DeclareTargetDefaultModel<fir::GlobalOp>>(*getContext());
5611	}
5612
5613	// Tablegen operators
5614
5615	#define GET_OP_CLASSES
5616	#include "flang/Optimizer/Dialect/FIROps.cpp.inc"
5617