1	//===- LLVMDialect.cpp - LLVM IR Ops and Dialect registration -------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file defines the types and operation details for the LLVM IR dialect in
10	// MLIR, and the LLVM IR dialect. It also registers the dialect.
11	//
12	//===----------------------------------------------------------------------===//
13
14	#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
15	#include "TypeDetail.h"
16	#include "mlir/Dialect/LLVMIR/LLVMAttrs.h"
17	#include "mlir/Dialect/LLVMIR/LLVMInterfaces.h"
18	#include "mlir/Dialect/LLVMIR/LLVMTypes.h"
19	#include "mlir/IR/Attributes.h"
20	#include "mlir/IR/Builders.h"
21	#include "mlir/IR/BuiltinOps.h"
22	#include "mlir/IR/BuiltinTypes.h"
23	#include "mlir/IR/DialectImplementation.h"
24	#include "mlir/IR/MLIRContext.h"
25	#include "mlir/IR/Matchers.h"
26	#include "mlir/Interfaces/FunctionImplementation.h"
27	#include "mlir/Transforms/InliningUtils.h"
28
29	#include "llvm/ADT/SCCIterator.h"
30	#include "llvm/ADT/TypeSwitch.h"
31	#include "llvm/AsmParser/Parser.h"
32	#include "llvm/Bitcode/BitcodeReader.h"
33	#include "llvm/Bitcode/BitcodeWriter.h"
34	#include "llvm/IR/Attributes.h"
35	#include "llvm/IR/Function.h"
36	#include "llvm/IR/Type.h"
37	#include "llvm/Support/Error.h"
38	#include "llvm/Support/Mutex.h"
39	#include "llvm/Support/SourceMgr.h"
40
41	#include <numeric>
42	#include <optional>
43
44	using namespace mlir;
45	using namespace mlir::LLVM;
46	using mlir::LLVM::cconv::getMaxEnumValForCConv;
47	using mlir::LLVM::linkage::getMaxEnumValForLinkage;
48	using mlir::LLVM::tailcallkind::getMaxEnumValForTailCallKind;
49
50	#include "mlir/Dialect/LLVMIR/LLVMOpsDialect.cpp.inc"
51
52	//===----------------------------------------------------------------------===//
53	// Attribute Helpers
54	//===----------------------------------------------------------------------===//
55
56	static constexpr const char kElemTypeAttrName[] = "elem_type";
57
58	static auto processFMFAttr(ArrayRef<NamedAttribute> attrs) {
59	SmallVector<NamedAttribute, 8> filteredAttrs(
60	llvm::make_filter_range(Range&: attrs, Pred: [&](NamedAttribute attr) {
61	if (attr.getName() == "fastmathFlags") {
62	auto defAttr =
63	FastmathFlagsAttr::get(attr.getValue().getContext(), {});
64	return defAttr != attr.getValue();
65	}
66	return true;
67	}));
68	return filteredAttrs;
69	}
70
71	/// Verifies `symbol`'s use in `op` to ensure the symbol is a valid and
72	/// fully defined llvm.func.
73	static LogicalResult verifySymbolAttrUse(FlatSymbolRefAttr symbol,
74	Operation *op,
75	SymbolTableCollection &symbolTable) {
76	StringRef name = symbol.getValue();
77	auto func =
78	symbolTable.lookupNearestSymbolFrom<LLVMFuncOp>(op, symbol.getAttr());
79	if (!func)
80	return op->emitOpError(message: "'")
81	<< name << "' does not reference a valid LLVM function";
82	if (func.isExternal())
83	return op->emitOpError(message: "'") << name << "' does not have a definition";
84	return success();
85	}
86
87	/// Returns a boolean type that has the same shape as `type`. It supports both
88	/// fixed size vectors as well as scalable vectors.
89	static Type getI1SameShape(Type type) {
90	Type i1Type = IntegerType::get(type.getContext(), 1);
91	if (LLVM::isCompatibleVectorType(type))
92	return LLVM::getVectorType(elementType: i1Type, numElements: LLVM::getVectorNumElements(type));
93	return i1Type;
94	}
95
96	// Parses one of the keywords provided in the list `keywords` and returns the
97	// position of the parsed keyword in the list. If none of the keywords from the
98	// list is parsed, returns -1.
99	static int parseOptionalKeywordAlternative(OpAsmParser &parser,
100	ArrayRef<StringRef> keywords) {
101	for (const auto &en : llvm::enumerate(First&: keywords)) {
102	if (succeeded(Result: parser.parseOptionalKeyword(keyword: en.value())))
103	return en.index();
104	}
105	return -1;
106	}
107
108	namespace {
109	template <typename Ty>
110	struct EnumTraits {};
111
112	#define REGISTER_ENUM_TYPE(Ty) \
113	template <> \
114	struct EnumTraits<Ty> { \
115	static StringRef stringify(Ty value) { return stringify##Ty(value); } \
116	static unsigned getMaxEnumVal() { return getMaxEnumValFor##Ty(); } \
117	}
118
119	REGISTER_ENUM_TYPE(Linkage);
120	REGISTER_ENUM_TYPE(UnnamedAddr);
121	REGISTER_ENUM_TYPE(CConv);
122	REGISTER_ENUM_TYPE(TailCallKind);
123	REGISTER_ENUM_TYPE(Visibility);
124	} // namespace
125
126	/// Parse an enum from the keyword, or default to the provided default value.
127	/// The return type is the enum type by default, unless overridden with the
128	/// second template argument.
129	template <typename EnumTy, typename RetTy = EnumTy>
130	static RetTy parseOptionalLLVMKeyword(OpAsmParser &parser,
131	OperationState &result,
132	EnumTy defaultValue) {
133	SmallVector<StringRef, 10> names;
134	for (unsigned i = 0, e = EnumTraits<EnumTy>::getMaxEnumVal(); i <= e; ++i)
135	names.push_back(Elt: EnumTraits<EnumTy>::stringify(static_cast<EnumTy>(i)));
136
137	int index = parseOptionalKeywordAlternative(parser, keywords: names);
138	if (index == -1)
139	return static_cast<RetTy>(defaultValue);
140	return static_cast<RetTy>(index);
141	}
142
143	//===----------------------------------------------------------------------===//
144	// Operand bundle helpers.
145	//===----------------------------------------------------------------------===//
146
147	static void printOneOpBundle(OpAsmPrinter &p, OperandRange operands,
148	TypeRange operandTypes, StringRef tag) {
149	p.printString(string: tag);
150	p << "(";
151
152	if (!operands.empty()) {
153	p.printOperands(container: operands);
154	p << " : ";
155	llvm::interleaveComma(c: operandTypes, os&: p);
156	}
157
158	p << ")";
159	}
160
161	static void printOpBundles(OpAsmPrinter &p, Operation *op,
162	OperandRangeRange opBundleOperands,
163	TypeRangeRange opBundleOperandTypes,
164	std::optional<ArrayAttr> opBundleTags) {
165	if (opBundleOperands.empty())
166	return;
167	assert(opBundleTags && "expect operand bundle tags");
168
169	p << "[";
170	llvm::interleaveComma(
171	llvm::zip(opBundleOperands, opBundleOperandTypes, *opBundleTags), p,
172	[&p](auto bundle) {
173	auto bundleTag = cast<StringAttr>(std::get<2>(bundle)).getValue();
174	printOneOpBundle(p, std::get<0>(bundle), std::get<1>(bundle),
175	bundleTag);
176	});
177	p << "]";
178	}
179
180	static ParseResult parseOneOpBundle(
181	OpAsmParser &p,
182	SmallVector<SmallVector<OpAsmParser::UnresolvedOperand>> &opBundleOperands,
183	SmallVector<SmallVector<Type>> &opBundleOperandTypes,
184	SmallVector<Attribute> &opBundleTags) {
185	SMLoc currentParserLoc = p.getCurrentLocation();
186	SmallVector<OpAsmParser::UnresolvedOperand> operands;
187	SmallVector<Type> types;
188	std::string tag;
189
190	if (p.parseString(string: &tag))
191	return p.emitError(loc: currentParserLoc, message: "expect operand bundle tag");
192
193	if (p.parseLParen())
194	return failure();
195
196	if (p.parseOptionalRParen()) {
197	if (p.parseOperandList(result&: operands) || p.parseColon() ||
198	p.parseTypeList(result&: types) || p.parseRParen())
199	return failure();
200	}
201
202	opBundleOperands.push_back(Elt: std::move(operands));
203	opBundleOperandTypes.push_back(Elt: std::move(types));
204	opBundleTags.push_back(StringAttr::get(p.getContext(), tag));
205
206	return success();
207	}
208
209	static std::optional<ParseResult> parseOpBundles(
210	OpAsmParser &p,
211	SmallVector<SmallVector<OpAsmParser::UnresolvedOperand>> &opBundleOperands,
212	SmallVector<SmallVector<Type>> &opBundleOperandTypes,
213	ArrayAttr &opBundleTags) {
214	if (p.parseOptionalLSquare())
215	return std::nullopt;
216
217	if (succeeded(Result: p.parseOptionalRSquare()))
218	return success();
219
220	SmallVector<Attribute> opBundleTagAttrs;
221	auto bundleParser = [&] {
222	return parseOneOpBundle(p, opBundleOperands, opBundleOperandTypes,
223	opBundleTags&: opBundleTagAttrs);
224	};
225	if (p.parseCommaSeparatedList(parseElementFn: bundleParser))
226	return failure();
227
228	if (p.parseRSquare())
229	return failure();
230
231	opBundleTags = ArrayAttr::get(p.getContext(), opBundleTagAttrs);
232
233	return success();
234	}
235
236	//===----------------------------------------------------------------------===//
237	// Printing, parsing, folding and builder for LLVM::CmpOp.
238	//===----------------------------------------------------------------------===//
239
240	void ICmpOp::print(OpAsmPrinter &p) {
241	p << " \"" << stringifyICmpPredicate(getPredicate()) << "\" " << getOperand(0)
242	<< ", " << getOperand(1);
243	p.printOptionalAttrDict((*this)->getAttrs(), {"predicate"});
244	p << " : " << getLhs().getType();
245	}
246
247	void FCmpOp::print(OpAsmPrinter &p) {
248	p << " \"" << stringifyFCmpPredicate(getPredicate()) << "\" " << getOperand(0)
249	<< ", " << getOperand(1);
250	p.printOptionalAttrDict(processFMFAttr((*this)->getAttrs()), {"predicate"});
251	p << " : " << getLhs().getType();
252	}
253
254	// <operation> ::= `llvm.icmp` string-literal ssa-use `,` ssa-use
255	// attribute-dict? `:` type
256	// <operation> ::= `llvm.fcmp` string-literal ssa-use `,` ssa-use
257	// attribute-dict? `:` type
258	template <typename CmpPredicateType>
259	static ParseResult parseCmpOp(OpAsmParser &parser, OperationState &result) {
260	StringAttr predicateAttr;
261	OpAsmParser::UnresolvedOperand lhs, rhs;
262	Type type;
263	SMLoc predicateLoc, trailingTypeLoc;
264	if (parser.getCurrentLocation(loc: &predicateLoc) ||
265	parser.parseAttribute(predicateAttr, "predicate", result.attributes) ||
266	parser.parseOperand(result&: lhs) || parser.parseComma() ||
267	parser.parseOperand(result&: rhs) ||
268	parser.parseOptionalAttrDict(result&: result.attributes) || parser.parseColon() ||
269	parser.getCurrentLocation(loc: &trailingTypeLoc) || parser.parseType(result&: type) ||
270	parser.resolveOperand(operand: lhs, type, result&: result.operands) ||
271	parser.resolveOperand(operand: rhs, type, result&: result.operands))
272	return failure();
273
274	// Replace the string attribute `predicate` with an integer attribute.
275	int64_t predicateValue = 0;
276	if (std::is_same<CmpPredicateType, ICmpPredicate>()) {
277	std::optional<ICmpPredicate> predicate =
278	symbolizeICmpPredicate(predicateAttr.getValue());
279	if (!predicate)
280	return parser.emitError(loc: predicateLoc)
281	<< "'" << predicateAttr.getValue()
282	<< "' is an incorrect value of the 'predicate' attribute";
283	predicateValue = static_cast<int64_t>(*predicate);
284	} else {
285	std::optional<FCmpPredicate> predicate =
286	symbolizeFCmpPredicate(predicateAttr.getValue());
287	if (!predicate)
288	return parser.emitError(loc: predicateLoc)
289	<< "'" << predicateAttr.getValue()
290	<< "' is an incorrect value of the 'predicate' attribute";
291	predicateValue = static_cast<int64_t>(*predicate);
292	}
293
294	result.attributes.set("predicate",
295	parser.getBuilder().getI64IntegerAttr(predicateValue));
296
297	// The result type is either i1 or a vector type <? x i1> if the inputs are
298	// vectors.
299	if (!isCompatibleType(type))
300	return parser.emitError(loc: trailingTypeLoc,
301	message: "expected LLVM dialect-compatible type");
302	result.addTypes(newTypes: getI1SameShape(type));
303	return success();
304	}
305
306	ParseResult ICmpOp::parse(OpAsmParser &parser, OperationState &result) {
307	return parseCmpOp<ICmpPredicate>(parser, result);
308	}
309
310	ParseResult FCmpOp::parse(OpAsmParser &parser, OperationState &result) {
311	return parseCmpOp<FCmpPredicate>(parser, result);
312	}
313
314	/// Returns a scalar or vector boolean attribute of the given type.
315	static Attribute getBoolAttribute(Type type, MLIRContext *ctx, bool value) {
316	auto boolAttr = BoolAttr::get(context: ctx, value);
317	ShapedType shapedType = dyn_cast<ShapedType>(type);
318	if (!shapedType)
319	return boolAttr;
320	return DenseElementsAttr::get(shapedType, boolAttr);
321	}
322
323	OpFoldResult ICmpOp::fold(FoldAdaptor adaptor) {
324	if (getPredicate() != ICmpPredicate::eq &&
325	getPredicate() != ICmpPredicate::ne)
326	return {};
327
328	// cmpi(eq/ne, x, x) -> true/false
329	if (getLhs() == getRhs())
330	return getBoolAttribute(getType(), getContext(),
331	getPredicate() == ICmpPredicate::eq);
332
333	// cmpi(eq/ne, alloca, null) -> false/true
334	if (getLhs().getDefiningOp<AllocaOp>() && getRhs().getDefiningOp<ZeroOp>())
335	return getBoolAttribute(getType(), getContext(),
336	getPredicate() == ICmpPredicate::ne);
337
338	// cmpi(eq/ne, null, alloca) -> cmpi(eq/ne, alloca, null)
339	if (getLhs().getDefiningOp<ZeroOp>() && getRhs().getDefiningOp<AllocaOp>()) {
340	Value lhs = getLhs();
341	Value rhs = getRhs();
342	getLhsMutable().assign(rhs);
343	getRhsMutable().assign(lhs);
344	return getResult();
345	}
346
347	return {};
348	}
349
350	//===----------------------------------------------------------------------===//
351	// Printing, parsing and verification for LLVM::AllocaOp.
352	//===----------------------------------------------------------------------===//
353
354	void AllocaOp::print(OpAsmPrinter &p) {
355	auto funcTy =
356	FunctionType::get(getContext(), {getArraySize().getType()}, {getType()});
357
358	if (getInalloca())
359	p << " inalloca";
360
361	p << ' ' << getArraySize() << " x " << getElemType();
362	if (getAlignment() && *getAlignment() != 0)
363	p.printOptionalAttrDict((*this)->getAttrs(),
364	{kElemTypeAttrName, getInallocaAttrName()});
365	else
366	p.printOptionalAttrDict(
367	(*this)->getAttrs(),
368	{getAlignmentAttrName(), kElemTypeAttrName, getInallocaAttrName()});
369	p << " : " << funcTy;
370	}
371
372	// <operation> ::= `llvm.alloca` `inalloca`? ssa-use `x` type
373	// attribute-dict? `:` type `,` type
374	ParseResult AllocaOp::parse(OpAsmParser &parser, OperationState &result) {
375	OpAsmParser::UnresolvedOperand arraySize;
376	Type type, elemType;
377	SMLoc trailingTypeLoc;
378
379	if (succeeded(parser.parseOptionalKeyword("inalloca")))
380	result.addAttribute(getInallocaAttrName(result.name),
381	UnitAttr::get(parser.getContext()));
382
383	if (parser.parseOperand(arraySize) || parser.parseKeyword("x") ||
384	parser.parseType(elemType) ||
385	parser.parseOptionalAttrDict(result.attributes) || parser.parseColon() ||
386	parser.getCurrentLocation(&trailingTypeLoc) || parser.parseType(type))
387	return failure();
388
389	std::optional<NamedAttribute> alignmentAttr =
390	result.attributes.getNamed("alignment");
391	if (alignmentAttr.has_value()) {
392	auto alignmentInt = llvm::dyn_cast<IntegerAttr>(alignmentAttr->getValue());
393	if (!alignmentInt)
394	return parser.emitError(parser.getNameLoc(),
395	"expected integer alignment");
396	if (alignmentInt.getValue().isZero())
397	result.attributes.erase("alignment");
398	}
399
400	// Extract the result type from the trailing function type.
401	auto funcType = llvm::dyn_cast<FunctionType>(type);
402	if (!funcType || funcType.getNumInputs() != 1 ||
403	funcType.getNumResults() != 1)
404	return parser.emitError(
405	trailingTypeLoc,
406	"expected trailing function type with one argument and one result");
407
408	if (parser.resolveOperand(arraySize, funcType.getInput(0), result.operands))
409	return failure();
410
411	Type resultType = funcType.getResult(0);
412	if (auto ptrResultType = llvm::dyn_cast<LLVMPointerType>(resultType))
413	result.addAttribute(kElemTypeAttrName, TypeAttr::get(elemType));
414
415	result.addTypes({funcType.getResult(0)});
416	return success();
417	}
418
419	LogicalResult AllocaOp::verify() {
420	// Only certain target extension types can be used in 'alloca'.
421	if (auto targetExtType = dyn_cast<LLVMTargetExtType>(getElemType());
422	targetExtType && !targetExtType.supportsMemOps())
423	return emitOpError()
424	<< "this target extension type cannot be used in alloca";
425
426	return success();
427	}
428
429	//===----------------------------------------------------------------------===//
430	// LLVM::BrOp
431	//===----------------------------------------------------------------------===//
432
433	SuccessorOperands BrOp::getSuccessorOperands(unsigned index) {
434	assert(index == 0 && "invalid successor index");
435	return SuccessorOperands(getDestOperandsMutable());
436	}
437
438	//===----------------------------------------------------------------------===//
439	// LLVM::CondBrOp
440	//===----------------------------------------------------------------------===//
441
442	SuccessorOperands CondBrOp::getSuccessorOperands(unsigned index) {
443	assert(index < getNumSuccessors() && "invalid successor index");
444	return SuccessorOperands(index == 0 ? getTrueDestOperandsMutable()
445	: getFalseDestOperandsMutable());
446	}
447
448	void CondBrOp::build(OpBuilder &builder, OperationState &result,
449	Value condition, Block *trueDest, ValueRange trueOperands,
450	Block *falseDest, ValueRange falseOperands,
451	std::optional<std::pair<uint32_t, uint32_t>> weights) {
452	DenseI32ArrayAttr weightsAttr;
453	if (weights)
454	weightsAttr =
455	builder.getDenseI32ArrayAttr({static_cast<int32_t>(weights->first),
456	static_cast<int32_t>(weights->second)});
457
458	build(builder, result, condition, trueOperands, falseOperands, weightsAttr,
459	/*loop_annotation=*/{}, trueDest, falseDest);
460	}
461
462	//===----------------------------------------------------------------------===//
463	// LLVM::SwitchOp
464	//===----------------------------------------------------------------------===//
465
466	void SwitchOp::build(OpBuilder &builder, OperationState &result, Value value,
467	Block *defaultDestination, ValueRange defaultOperands,
468	DenseIntElementsAttr caseValues,
469	BlockRange caseDestinations,
470	ArrayRef<ValueRange> caseOperands,
471	ArrayRef<int32_t> branchWeights) {
472	DenseI32ArrayAttr weightsAttr;
473	if (!branchWeights.empty())
474	weightsAttr = builder.getDenseI32ArrayAttr(branchWeights);
475
476	build(builder, result, value, defaultOperands, caseOperands, caseValues,
477	weightsAttr, defaultDestination, caseDestinations);
478	}
479
480	void SwitchOp::build(OpBuilder &builder, OperationState &result, Value value,
481	Block *defaultDestination, ValueRange defaultOperands,
482	ArrayRef<APInt> caseValues, BlockRange caseDestinations,
483	ArrayRef<ValueRange> caseOperands,
484	ArrayRef<int32_t> branchWeights) {
485	DenseIntElementsAttr caseValuesAttr;
486	if (!caseValues.empty()) {
487	ShapedType caseValueType = VectorType::get(
488	static_cast<int64_t>(caseValues.size()), value.getType());
489	caseValuesAttr = DenseIntElementsAttr::get(caseValueType, caseValues);
490	}
491
492	build(builder, result, value, defaultDestination, defaultOperands,
493	caseValuesAttr, caseDestinations, caseOperands, branchWeights);
494	}
495
496	void SwitchOp::build(OpBuilder &builder, OperationState &result, Value value,
497	Block *defaultDestination, ValueRange defaultOperands,
498	ArrayRef<int32_t> caseValues, BlockRange caseDestinations,
499	ArrayRef<ValueRange> caseOperands,
500	ArrayRef<int32_t> branchWeights) {
501	DenseIntElementsAttr caseValuesAttr;
502	if (!caseValues.empty()) {
503	ShapedType caseValueType = VectorType::get(
504	static_cast<int64_t>(caseValues.size()), value.getType());
505	caseValuesAttr = DenseIntElementsAttr::get(caseValueType, caseValues);
506	}
507
508	build(builder, result, value, defaultDestination, defaultOperands,
509	caseValuesAttr, caseDestinations, caseOperands, branchWeights);
510	}
511
512	/// <cases> ::= `[` (case (`,` case )* )? `]`
513	/// <case> ::= integer `:` bb-id (`(` ssa-use-and-type-list `)`)?
514	static ParseResult parseSwitchOpCases(
515	OpAsmParser &parser, Type flagType, DenseIntElementsAttr &caseValues,
516	SmallVectorImpl<Block *> &caseDestinations,
517	SmallVectorImpl<SmallVector<OpAsmParser::UnresolvedOperand>> &caseOperands,
518	SmallVectorImpl<SmallVector<Type>> &caseOperandTypes) {
519	if (failed(Result: parser.parseLSquare()))
520	return failure();
521	if (succeeded(Result: parser.parseOptionalRSquare()))
522	return success();
523	SmallVector<APInt> values;
524	unsigned bitWidth = flagType.getIntOrFloatBitWidth();
525	auto parseCase = [&]() {
526	int64_t value = 0;
527	if (failed(Result: parser.parseInteger(result&: value)))
528	return failure();
529	values.push_back(Elt: APInt(bitWidth, value, /*isSigned=*/true));
530
531	Block *destination;
532	SmallVector<OpAsmParser::UnresolvedOperand> operands;
533	SmallVector<Type> operandTypes;
534	if (parser.parseColon() || parser.parseSuccessor(dest&: destination))
535	return failure();
536	if (!parser.parseOptionalLParen()) {
537	if (parser.parseOperandList(result&: operands, delimiter: OpAsmParser::Delimiter::None,
538	/*allowResultNumber=*/false) ||
539	parser.parseColonTypeList(result&: operandTypes) || parser.parseRParen())
540	return failure();
541	}
542	caseDestinations.push_back(Elt: destination);
543	caseOperands.emplace_back(Args&: operands);
544	caseOperandTypes.emplace_back(Args&: operandTypes);
545	return success();
546	};
547	if (failed(Result: parser.parseCommaSeparatedList(parseElementFn: parseCase)))
548	return failure();
549
550	ShapedType caseValueType =
551	VectorType::get(static_cast<int64_t>(values.size()), flagType);
552	caseValues = DenseIntElementsAttr::get(caseValueType, values);
553	return parser.parseRSquare();
554	}
555
556	static void printSwitchOpCases(OpAsmPrinter &p, SwitchOp op, Type flagType,
557	DenseIntElementsAttr caseValues,
558	SuccessorRange caseDestinations,
559	OperandRangeRange caseOperands,
560	const TypeRangeRange &caseOperandTypes) {
561	p << '[';
562	p.printNewline();
563	if (!caseValues) {
564	p << ']';
565	return;
566	}
567
568	size_t index = 0;
569	llvm::interleave(
570	c: llvm::zip(t&: caseValues, u&: caseDestinations),
571	each_fn: [&](auto i) {
572	p << " ";
573	p << std::get<0>(i);
574	p << ": ";
575	p.printSuccessorAndUseList(successor: std::get<1>(i), succOperands: caseOperands[index++]);
576	},
577	between_fn: [&] {
578	p << ',';
579	p.printNewline();
580	});
581	p.printNewline();
582	p << ']';
583	}
584
585	LogicalResult SwitchOp::verify() {
586	if ((!getCaseValues() && !getCaseDestinations().empty()) ||
587	(getCaseValues() &&
588	getCaseValues()->size() !=
589	static_cast<int64_t>(getCaseDestinations().size())))
590	return emitOpError("expects number of case values to match number of "
591	"case destinations");
592	if (getBranchWeights() && getBranchWeights()->size() != getNumSuccessors())
593	return emitError("expects number of branch weights to match number of "
594	"successors: ")
595	<< getBranchWeights()->size() << " vs " << getNumSuccessors();
596	if (getCaseValues() &&
597	getValue().getType() != getCaseValues()->getElementType())
598	return emitError("expects case value type to match condition value type");
599	return success();
600	}
601
602	SuccessorOperands SwitchOp::getSuccessorOperands(unsigned index) {
603	assert(index < getNumSuccessors() && "invalid successor index");
604	return SuccessorOperands(index == 0 ? getDefaultOperandsMutable()
605	: getCaseOperandsMutable(index - 1));
606	}
607
608	//===----------------------------------------------------------------------===//
609	// Code for LLVM::GEPOp.
610	//===----------------------------------------------------------------------===//
611
612	constexpr int32_t GEPOp::kDynamicIndex;
613
614	GEPIndicesAdaptor<ValueRange> GEPOp::getIndices() {
615	return GEPIndicesAdaptor<ValueRange>(getRawConstantIndicesAttr(),
616	getDynamicIndices());
617	}
618
619	/// Returns the elemental type of any LLVM-compatible vector type or self.
620	static Type extractVectorElementType(Type type) {
621	if (auto vectorType = llvm::dyn_cast<VectorType>(type))
622	return vectorType.getElementType();
623	return type;
624	}
625
626	/// Destructures the 'indices' parameter into 'rawConstantIndices' and
627	/// 'dynamicIndices', encoding the former in the process. In the process,
628	/// dynamic indices which are used to index into a structure type are converted
629	/// to constant indices when possible. To do this, the GEPs element type should
630	/// be passed as first parameter.
631	static void destructureIndices(Type currType, ArrayRef<GEPArg> indices,
632	SmallVectorImpl<int32_t> &rawConstantIndices,
633	SmallVectorImpl<Value> &dynamicIndices) {
634	for (const GEPArg &iter : indices) {
635	// If the thing we are currently indexing into is a struct we must turn
636	// any integer constants into constant indices. If this is not possible
637	// we don't do anything here. The verifier will catch it and emit a proper
638	// error. All other canonicalization is done in the fold method.
639	bool requiresConst = !rawConstantIndices.empty() &&
640	isa_and_nonnull<LLVMStructType>(currType);
641	if (Value val = llvm::dyn_cast_if_present<Value>(Val: iter)) {
642	APInt intC;
643	if (requiresConst && matchPattern(val, m_ConstantInt(&intC)) &&
644	intC.isSignedIntN(kGEPConstantBitWidth)) {
645	rawConstantIndices.push_back(Elt: intC.getSExtValue());
646	} else {
647	rawConstantIndices.push_back(GEPOp::kDynamicIndex);
648	dynamicIndices.push_back(Elt: val);
649	}
650	} else {
651	rawConstantIndices.push_back(Elt: cast<GEPConstantIndex>(Val: iter));
652	}
653
654	// Skip for very first iteration of this loop. First index does not index
655	// within the aggregates, but is just a pointer offset.
656	if (rawConstantIndices.size() == 1 || !currType)
657	continue;
658
659	currType = TypeSwitch<Type, Type>(currType)
660	.Case<VectorType, LLVMArrayType>([](auto containerType) {
661	return containerType.getElementType();
662	})
663	.Case([&](LLVMStructType structType) -> Type {
664	int64_t memberIndex = rawConstantIndices.back();
665	if (memberIndex >= 0 && static_cast<size_t>(memberIndex) <
666	structType.getBody().size())
667	return structType.getBody()[memberIndex];
668	return nullptr;
669	})
670	.Default(Type(nullptr));
671	}
672	}
673
674	void GEPOp::build(OpBuilder &builder, OperationState &result, Type resultType,
675	Type elementType, Value basePtr, ArrayRef<GEPArg> indices,
676	GEPNoWrapFlags noWrapFlags,
677	ArrayRef<NamedAttribute> attributes) {
678	SmallVector<int32_t> rawConstantIndices;
679	SmallVector<Value> dynamicIndices;
680	destructureIndices(elementType, indices, rawConstantIndices, dynamicIndices);
681
682	result.addTypes(resultType);
683	result.addAttributes(attributes);
684	result.getOrAddProperties<Properties>().rawConstantIndices =
685	builder.getDenseI32ArrayAttr(rawConstantIndices);
686	result.getOrAddProperties<Properties>().noWrapFlags = noWrapFlags;
687	result.getOrAddProperties<Properties>().elem_type =
688	TypeAttr::get(elementType);
689	result.addOperands(basePtr);
690	result.addOperands(dynamicIndices);
691	}
692
693	void GEPOp::build(OpBuilder &builder, OperationState &result, Type resultType,
694	Type elementType, Value basePtr, ValueRange indices,
695	GEPNoWrapFlags noWrapFlags,
696	ArrayRef<NamedAttribute> attributes) {
697	build(builder, result, resultType, elementType, basePtr,
698	SmallVector<GEPArg>(indices), noWrapFlags, attributes);
699	}
700
701	static ParseResult
702	parseGEPIndices(OpAsmParser &parser,
703	SmallVectorImpl<OpAsmParser::UnresolvedOperand> &indices,
704	DenseI32ArrayAttr &rawConstantIndices) {
705	SmallVector<int32_t> constantIndices;
706
707	auto idxParser = [&]() -> ParseResult {
708	int32_t constantIndex;
709	OptionalParseResult parsedInteger =
710	parser.parseOptionalInteger(result&: constantIndex);
711	if (parsedInteger.has_value()) {
712	if (failed(Result: parsedInteger.value()))
713	return failure();
714	constantIndices.push_back(Elt: constantIndex);
715	return success();
716	}
717
718	constantIndices.push_back(LLVM::GEPOp::kDynamicIndex);
719	return parser.parseOperand(result&: indices.emplace_back());
720	};
721	if (parser.parseCommaSeparatedList(parseElementFn: idxParser))
722	return failure();
723
724	rawConstantIndices =
725	DenseI32ArrayAttr::get(parser.getContext(), constantIndices);
726	return success();
727	}
728
729	static void printGEPIndices(OpAsmPrinter &printer, LLVM::GEPOp gepOp,
730	OperandRange indices,
731	DenseI32ArrayAttr rawConstantIndices) {
732	llvm::interleaveComma(
733	c: GEPIndicesAdaptor<OperandRange>(rawConstantIndices, indices), os&: printer,
734	each_fn: [&](PointerUnion<IntegerAttr, Value> cst) {
735	if (Value val = llvm::dyn_cast_if_present<Value>(Val&: cst))
736	printer.printOperand(value: val);
737	else
738	printer << cast<IntegerAttr>(cst).getInt();
739	});
740	}
741
742	/// For the given `indices`, check if they comply with `baseGEPType`,
743	/// especially check against LLVMStructTypes nested within.
744	static LogicalResult
745	verifyStructIndices(Type baseGEPType, unsigned indexPos,
746	GEPIndicesAdaptor<ValueRange> indices,
747	function_ref<InFlightDiagnostic()> emitOpError) {
748	if (indexPos >= indices.size())
749	// Stop searching
750	return success();
751
752	return TypeSwitch<Type, LogicalResult>(baseGEPType)
753	.Case<LLVMStructType>([&](LLVMStructType structType) -> LogicalResult {
754	auto attr = dyn_cast<IntegerAttr>(indices[indexPos]);
755	if (!attr)
756	return emitOpError() << "expected index " << indexPos
757	<< " indexing a struct to be constant";
758
759	int32_t gepIndex = attr.getInt();
760	ArrayRef<Type> elementTypes = structType.getBody();
761	if (gepIndex < 0 ||
762	static_cast<size_t>(gepIndex) >= elementTypes.size())
763	return emitOpError() << "index " << indexPos
764	<< " indexing a struct is out of bounds";
765
766	// Instead of recursively going into every children types, we only
767	// dive into the one indexed by gepIndex.
768	return verifyStructIndices(elementTypes[gepIndex], indexPos + 1,
769	indices, emitOpError);
770	})
771	.Case<VectorType, LLVMArrayType>(
772	[&](auto containerType) -> LogicalResult {
773	return verifyStructIndices(containerType.getElementType(),
774	indexPos + 1, indices, emitOpError);
775	})
776	.Default([&](auto otherType) -> LogicalResult {
777	return emitOpError()
778	<< "type " << otherType << " cannot be indexed (index #"
779	<< indexPos << ")";
780	});
781	}
782
783	/// Driver function around `verifyStructIndices`.
784	static LogicalResult
785	verifyStructIndices(Type baseGEPType, GEPIndicesAdaptor<ValueRange> indices,
786	function_ref<InFlightDiagnostic()> emitOpError) {
787	return verifyStructIndices(baseGEPType, /*indexPos=*/1, indices, emitOpError);
788	}
789
790	LogicalResult LLVM::GEPOp::verify() {
791	if (static_cast<size_t>(
792	llvm::count(getRawConstantIndices(), kDynamicIndex)) !=
793	getDynamicIndices().size())
794	return emitOpError("expected as many dynamic indices as specified in '")
795	<< getRawConstantIndicesAttrName().getValue() << "'";
796
797	if (getNoWrapFlags() == GEPNoWrapFlags::inboundsFlag)
798	return emitOpError("'inbounds_flag' cannot be used directly.");
799
800	return verifyStructIndices(getElemType(), getIndices(),
801	[&] { return emitOpError(); });
802	}
803
804	//===----------------------------------------------------------------------===//
805	// LoadOp
806	//===----------------------------------------------------------------------===//
807
808	void LoadOp::getEffects(
809	SmallVectorImpl<SideEffects::EffectInstance<MemoryEffects::Effect>>
810	&effects) {
811	effects.emplace_back(MemoryEffects::Read::get(), &getAddrMutable());
812	// Volatile operations can have target-specific read-write effects on
813	// memory besides the one referred to by the pointer operand.
814	// Similarly, atomic operations that are monotonic or stricter cause
815	// synchronization that from a language point-of-view, are arbitrary
816	// read-writes into memory.
817	if (getVolatile_() || (getOrdering() != AtomicOrdering::not_atomic &&
818	getOrdering() != AtomicOrdering::unordered)) {
819	effects.emplace_back(MemoryEffects::Write::get());
820	effects.emplace_back(MemoryEffects::Read::get());
821	}
822	}
823
824	/// Returns true if the given type is supported by atomic operations. All
825	/// integer, float, and pointer types with a power-of-two bitsize and a minimal
826	/// size of 8 bits are supported.
827	static bool isTypeCompatibleWithAtomicOp(Type type,
828	const DataLayout &dataLayout) {
829	if (!isa<IntegerType, LLVMPointerType>(type))
830	if (!isCompatibleFloatingPointType(type))
831	return false;
832
833	llvm::TypeSize bitWidth = dataLayout.getTypeSizeInBits(t: type);
834	if (bitWidth.isScalable())
835	return false;
836	// Needs to be at least 8 bits and a power of two.
837	return bitWidth >= 8 && (bitWidth & (bitWidth - 1)) == 0;
838	}
839
840	/// Verifies the attributes and the type of atomic memory access operations.
841	template <typename OpTy>
842	LogicalResult verifyAtomicMemOp(OpTy memOp, Type valueType,
843	ArrayRef<AtomicOrdering> unsupportedOrderings) {
844	if (memOp.getOrdering() != AtomicOrdering::not_atomic) {
845	DataLayout dataLayout = DataLayout::closest(op: memOp);
846	if (!isTypeCompatibleWithAtomicOp(type: valueType, dataLayout))
847	return memOp.emitOpError("unsupported type ")
848	<< valueType << " for atomic access";
849	if (llvm::is_contained(unsupportedOrderings, memOp.getOrdering()))
850	return memOp.emitOpError("unsupported ordering '")
851	<< stringifyAtomicOrdering(memOp.getOrdering()) << "'";
852	if (!memOp.getAlignment())
853	return memOp.emitOpError("expected alignment for atomic access");
854	return success();
855	}
856	if (memOp.getSyncscope())
857	return memOp.emitOpError(
858	"expected syncscope to be null for non-atomic access");
859	return success();
860	}
861
862	LogicalResult LoadOp::verify() {
863	Type valueType = getResult().getType();
864	return verifyAtomicMemOp(*this, valueType,
865	{AtomicOrdering::release, AtomicOrdering::acq_rel});
866	}
867
868	void LoadOp::build(OpBuilder &builder, OperationState &state, Type type,
869	Value addr, unsigned alignment, bool isVolatile,
870	bool isNonTemporal, bool isInvariant, bool isInvariantGroup,
871	AtomicOrdering ordering, StringRef syncscope) {
872	build(builder, state, type, addr,
873	alignment ? builder.getI64IntegerAttr(alignment) : nullptr, isVolatile,
874	isNonTemporal, isInvariant, isInvariantGroup, ordering,
875	syncscope.empty() ? nullptr : builder.getStringAttr(syncscope),
876	/*dereferenceable=*/nullptr,
877	/*access_groups=*/nullptr,
878	/*alias_scopes=*/nullptr, /*noalias_scopes=*/nullptr,
879	/*tbaa=*/nullptr);
880	}
881
882	//===----------------------------------------------------------------------===//
883	// StoreOp
884	//===----------------------------------------------------------------------===//
885
886	void StoreOp::getEffects(
887	SmallVectorImpl<SideEffects::EffectInstance<MemoryEffects::Effect>>
888	&effects) {
889	effects.emplace_back(MemoryEffects::Write::get(), &getAddrMutable());
890	// Volatile operations can have target-specific read-write effects on
891	// memory besides the one referred to by the pointer operand.
892	// Similarly, atomic operations that are monotonic or stricter cause
893	// synchronization that from a language point-of-view, are arbitrary
894	// read-writes into memory.
895	if (getVolatile_() || (getOrdering() != AtomicOrdering::not_atomic &&
896	getOrdering() != AtomicOrdering::unordered)) {
897	effects.emplace_back(MemoryEffects::Write::get());
898	effects.emplace_back(MemoryEffects::Read::get());
899	}
900	}
901
902	LogicalResult StoreOp::verify() {
903	Type valueType = getValue().getType();
904	return verifyAtomicMemOp(*this, valueType,
905	{AtomicOrdering::acquire, AtomicOrdering::acq_rel});
906	}
907
908	void StoreOp::build(OpBuilder &builder, OperationState &state, Value value,
909	Value addr, unsigned alignment, bool isVolatile,
910	bool isNonTemporal, bool isInvariantGroup,
911	AtomicOrdering ordering, StringRef syncscope) {
912	build(builder, state, value, addr,
913	alignment ? builder.getI64IntegerAttr(alignment) : nullptr, isVolatile,
914	isNonTemporal, isInvariantGroup, ordering,
915	syncscope.empty() ? nullptr : builder.getStringAttr(syncscope),
916	/*access_groups=*/nullptr,
917	/*alias_scopes=*/nullptr, /*noalias_scopes=*/nullptr, /*tbaa=*/nullptr);
918	}
919
920	//===----------------------------------------------------------------------===//
921	// CallOp
922	//===----------------------------------------------------------------------===//
923
924	/// Gets the MLIR Op-like result types of a LLVMFunctionType.
925	static SmallVector<Type, 1> getCallOpResultTypes(LLVMFunctionType calleeType) {
926	SmallVector<Type, 1> results;
927	Type resultType = calleeType.getReturnType();
928	if (!isa<LLVM::LLVMVoidType>(Val: resultType))
929	results.push_back(Elt: resultType);
930	return results;
931	}
932
933	/// Gets the variadic callee type for a LLVMFunctionType.
934	static TypeAttr getCallOpVarCalleeType(LLVMFunctionType calleeType) {
935	return calleeType.isVarArg() ? TypeAttr::get(calleeType) : nullptr;
936	}
937
938	/// Constructs a LLVMFunctionType from MLIR `results` and `args`.
939	static LLVMFunctionType getLLVMFuncType(MLIRContext *context, TypeRange results,
940	ValueRange args) {
941	Type resultType;
942	if (results.empty())
943	resultType = LLVMVoidType::get(ctx: context);
944	else
945	resultType = results.front();
946	return LLVMFunctionType::get(resultType, llvm::to_vector(args.getTypes()),
947	/*isVarArg=*/false);
948	}
949
950	void CallOp::build(OpBuilder &builder, OperationState &state, TypeRange results,
951	StringRef callee, ValueRange args) {
952	build(builder, state, results, builder.getStringAttr(callee), args);
953	}
954
955	void CallOp::build(OpBuilder &builder, OperationState &state, TypeRange results,
956	StringAttr callee, ValueRange args) {
957	build(builder, state, results, SymbolRefAttr::get(callee), args);
958	}
959
960	void CallOp::build(OpBuilder &builder, OperationState &state, TypeRange results,
961	FlatSymbolRefAttr callee, ValueRange args) {
962	assert(callee && "expected non-null callee in direct call builder");
963	build(builder, state, results,
964	/*var_callee_type=*/nullptr, callee, args, /*fastmathFlags=*/nullptr,
965	/*branch_weights=*/nullptr,
966	/*CConv=*/nullptr, /*TailCallKind=*/nullptr,
967	/*memory_effects=*/nullptr,
968	/*convergent=*/nullptr, /*no_unwind=*/nullptr, /*will_return=*/nullptr,
969	/*op_bundle_operands=*/{}, /*op_bundle_tags=*/{},
970	/*arg_attrs=*/nullptr, /*res_attrs=*/nullptr,
971	/*access_groups=*/nullptr, /*alias_scopes=*/nullptr,
972	/*noalias_scopes=*/nullptr, /*tbaa=*/nullptr,
973	/*no_inline=*/nullptr, /*always_inline=*/nullptr,
974	/*inline_hint=*/nullptr);
975	}
976
977	void CallOp::build(OpBuilder &builder, OperationState &state,
978	LLVMFunctionType calleeType, StringRef callee,
979	ValueRange args) {
980	build(builder, state, calleeType, builder.getStringAttr(callee), args);
981	}
982
983	void CallOp::build(OpBuilder &builder, OperationState &state,
984	LLVMFunctionType calleeType, StringAttr callee,
985	ValueRange args) {
986	build(builder, state, calleeType, SymbolRefAttr::get(callee), args);
987	}
988
989	void CallOp::build(OpBuilder &builder, OperationState &state,
990	LLVMFunctionType calleeType, FlatSymbolRefAttr callee,
991	ValueRange args) {
992	build(builder, state, getCallOpResultTypes(calleeType),
993	getCallOpVarCalleeType(calleeType), callee, args,
994	/*fastmathFlags=*/nullptr,
995	/*branch_weights=*/nullptr, /*CConv=*/nullptr,
996	/*TailCallKind=*/nullptr, /*memory_effects=*/nullptr,
997	/*convergent=*/nullptr,
998	/*no_unwind=*/nullptr, /*will_return=*/nullptr,
999	/*op_bundle_operands=*/{}, /*op_bundle_tags=*/{},
1000	/*arg_attrs=*/nullptr, /*res_attrs=*/nullptr,
1001	/*access_groups=*/nullptr,
1002	/*alias_scopes=*/nullptr, /*noalias_scopes=*/nullptr, /*tbaa=*/nullptr,
1003	/*no_inline=*/nullptr, /*always_inline=*/nullptr,
1004	/*inline_hint=*/nullptr);
1005	}
1006
1007	void CallOp::build(OpBuilder &builder, OperationState &state,
1008	LLVMFunctionType calleeType, ValueRange args) {
1009	build(builder, state, getCallOpResultTypes(calleeType),
1010	getCallOpVarCalleeType(calleeType),
1011	/*callee=*/nullptr, args,
1012	/*fastmathFlags=*/nullptr, /*branch_weights=*/nullptr,
1013	/*CConv=*/nullptr, /*TailCallKind=*/nullptr, /*memory_effects=*/nullptr,
1014	/*convergent=*/nullptr, /*no_unwind=*/nullptr, /*will_return=*/nullptr,
1015	/*op_bundle_operands=*/{}, /*op_bundle_tags=*/{},
1016	/*arg_attrs=*/nullptr, /*res_attrs=*/nullptr,
1017	/*access_groups=*/nullptr, /*alias_scopes=*/nullptr,
1018	/*noalias_scopes=*/nullptr, /*tbaa=*/nullptr,
1019	/*no_inline=*/nullptr, /*always_inline=*/nullptr,
1020	/*inline_hint=*/nullptr);
1021	}
1022
1023	void CallOp::build(OpBuilder &builder, OperationState &state, LLVMFuncOp func,
1024	ValueRange args) {
1025	auto calleeType = func.getFunctionType();
1026	build(builder, state, getCallOpResultTypes(calleeType),
1027	getCallOpVarCalleeType(calleeType), SymbolRefAttr::get(func), args,
1028	/*fastmathFlags=*/nullptr, /*branch_weights=*/nullptr,
1029	/*CConv=*/nullptr, /*TailCallKind=*/nullptr, /*memory_effects=*/nullptr,
1030	/*convergent=*/nullptr, /*no_unwind=*/nullptr, /*will_return=*/nullptr,
1031	/*op_bundle_operands=*/{}, /*op_bundle_tags=*/{},
1032	/*access_groups=*/nullptr, /*alias_scopes=*/nullptr,
1033	/*arg_attrs=*/nullptr, /*res_attrs=*/nullptr,
1034	/*noalias_scopes=*/nullptr, /*tbaa=*/nullptr,
1035	/*no_inline=*/nullptr, /*always_inline=*/nullptr,
1036	/*inline_hint=*/nullptr);
1037	}
1038
1039	CallInterfaceCallable CallOp::getCallableForCallee() {
1040	// Direct call.
1041	if (FlatSymbolRefAttr calleeAttr = getCalleeAttr())
1042	return calleeAttr;
1043	// Indirect call, callee Value is the first operand.
1044	return getOperand(0);
1045	}
1046
1047	void CallOp::setCalleeFromCallable(CallInterfaceCallable callee) {
1048	// Direct call.
1049	if (FlatSymbolRefAttr calleeAttr = getCalleeAttr()) {
1050	auto symRef = cast<SymbolRefAttr>(callee);
1051	return setCalleeAttr(cast<FlatSymbolRefAttr>(symRef));
1052	}
1053	// Indirect call, callee Value is the first operand.
1054	return setOperand(0, cast<Value>(callee));
1055	}
1056
1057	Operation::operand_range CallOp::getArgOperands() {
1058	return getCalleeOperands().drop_front(getCallee().has_value() ? 0 : 1);
1059	}
1060
1061	MutableOperandRange CallOp::getArgOperandsMutable() {
1062	return MutableOperandRange(*this, getCallee().has_value() ? 0 : 1,
1063	getCalleeOperands().size());
1064	}
1065
1066	/// Verify that an inlinable callsite of a debug-info-bearing function in a
1067	/// debug-info-bearing function has a debug location attached to it. This
1068	/// mirrors an LLVM IR verifier.
1069	static LogicalResult verifyCallOpDebugInfo(CallOp callOp, LLVMFuncOp callee) {
1070	if (callee.isExternal())
1071	return success();
1072	auto parentFunc = callOp->getParentOfType<FunctionOpInterface>();
1073	if (!parentFunc)
1074	return success();
1075
1076	auto hasSubprogram = [](Operation *op) {
1077	return op->getLoc()
1078	->findInstanceOf<FusedLocWith<LLVM::DISubprogramAttr>>() !=
1079	nullptr;
1080	};
1081	if (!hasSubprogram(parentFunc) || !hasSubprogram(callee))
1082	return success();
1083	bool containsLoc = !isa<UnknownLoc>(callOp->getLoc());
1084	if (!containsLoc)
1085	return callOp.emitError()
1086	<< "inlinable function call in a function with a DISubprogram "
1087	"location must have a debug location";
1088	return success();
1089	}
1090
1091	/// Verify that the parameter and return types of the variadic callee type match
1092	/// the `callOp` argument and result types.
1093	template <typename OpTy>
1094	LogicalResult verifyCallOpVarCalleeType(OpTy callOp) {
1095	std::optional<LLVMFunctionType> varCalleeType = callOp.getVarCalleeType();
1096	if (!varCalleeType)
1097	return success();
1098
1099	// Verify the variadic callee type is a variadic function type.
1100	if (!varCalleeType->isVarArg())
1101	return callOp.emitOpError(
1102	"expected var_callee_type to be a variadic function type");
1103
1104	// Verify the variadic callee type has at most as many parameters as the call
1105	// has argument operands.
1106	if (varCalleeType->getNumParams() > callOp.getArgOperands().size())
1107	return callOp.emitOpError("expected var_callee_type to have at most ")
1108	<< callOp.getArgOperands().size() << " parameters";
1109
1110	// Verify the variadic callee type matches the call argument types.
1111	for (auto [paramType, operand] :
1112	llvm::zip(varCalleeType->getParams(), callOp.getArgOperands()))
1113	if (paramType != operand.getType())
1114	return callOp.emitOpError()
1115	<< "var_callee_type parameter type mismatch: " << paramType
1116	<< " != " << operand.getType();
1117
1118	// Verify the variadic callee type matches the call result type.
1119	if (!callOp.getNumResults()) {
1120	if (!isa<LLVMVoidType>(varCalleeType->getReturnType()))
1121	return callOp.emitOpError("expected var_callee_type to return void");
1122	} else {
1123	if (callOp.getResult().getType() != varCalleeType->getReturnType())
1124	return callOp.emitOpError("var_callee_type return type mismatch: ")
1125	<< varCalleeType->getReturnType()
1126	<< " != " << callOp.getResult().getType();
1127	}
1128	return success();
1129	}
1130
1131	template <typename OpType>
1132	static LogicalResult verifyOperandBundles(OpType &op) {
1133	OperandRangeRange opBundleOperands = op.getOpBundleOperands();
1134	std::optional<ArrayAttr> opBundleTags = op.getOpBundleTags();
1135
1136	auto isStringAttr = [](Attribute tagAttr) {
1137	return isa<StringAttr>(Val: tagAttr);
1138	};
1139	if (opBundleTags && !llvm::all_of(*opBundleTags, isStringAttr))
1140	return op.emitError("operand bundle tag must be a StringAttr");
1141
1142	size_t numOpBundles = opBundleOperands.size();
1143	size_t numOpBundleTags = opBundleTags ? opBundleTags->size() : 0;
1144	if (numOpBundles != numOpBundleTags)
1145	return op.emitError("expected ")
1146	<< numOpBundles << " operand bundle tags, but actually got "
1147	<< numOpBundleTags;
1148
1149	return success();
1150	}
1151
1152	LogicalResult CallOp::verify() { return verifyOperandBundles(*this); }
1153
1154	LogicalResult CallOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
1155	if (failed(verifyCallOpVarCalleeType(*this)))
1156	return failure();
1157
1158	// Type for the callee, we'll get it differently depending if it is a direct
1159	// or indirect call.
1160	Type fnType;
1161
1162	bool isIndirect = false;
1163
1164	// If this is an indirect call, the callee attribute is missing.
1165	FlatSymbolRefAttr calleeName = getCalleeAttr();
1166	if (!calleeName) {
1167	isIndirect = true;
1168	if (!getNumOperands())
1169	return emitOpError(
1170	"must have either a `callee` attribute or at least an operand");
1171	auto ptrType = llvm::dyn_cast<LLVMPointerType>(getOperand(0).getType());
1172	if (!ptrType)
1173	return emitOpError("indirect call expects a pointer as callee: ")
1174	<< getOperand(0).getType();
1175
1176	return success();
1177	} else {
1178	Operation *callee =
1179	symbolTable.lookupNearestSymbolFrom(*this, calleeName.getAttr());
1180	if (!callee)
1181	return emitOpError()
1182	<< "'" << calleeName.getValue()
1183	<< "' does not reference a symbol in the current scope";
1184	auto fn = dyn_cast<LLVMFuncOp>(callee);
1185	if (!fn)
1186	return emitOpError() << "'" << calleeName.getValue()
1187	<< "' does not reference a valid LLVM function";
1188
1189	if (failed(verifyCallOpDebugInfo(*this, fn)))
1190	return failure();
1191	fnType = fn.getFunctionType();
1192	}
1193
1194	LLVMFunctionType funcType = llvm::dyn_cast<LLVMFunctionType>(fnType);
1195	if (!funcType)
1196	return emitOpError("callee does not have a functional type: ") << fnType;
1197
1198	if (funcType.isVarArg() && !getVarCalleeType())
1199	return emitOpError() << "missing var_callee_type attribute for vararg call";
1200
1201	// Verify that the operand and result types match the callee.
1202
1203	if (!funcType.isVarArg() &&
1204	funcType.getNumParams() != (getCalleeOperands().size() - isIndirect))
1205	return emitOpError() << "incorrect number of operands ("
1206	<< (getCalleeOperands().size() - isIndirect)
1207	<< ") for callee (expecting: "
1208	<< funcType.getNumParams() << ")";
1209
1210	if (funcType.getNumParams() > (getCalleeOperands().size() - isIndirect))
1211	return emitOpError() << "incorrect number of operands ("
1212	<< (getCalleeOperands().size() - isIndirect)
1213	<< ") for varargs callee (expecting at least: "
1214	<< funcType.getNumParams() << ")";
1215
1216	for (unsigned i = 0, e = funcType.getNumParams(); i != e; ++i)
1217	if (getOperand(i + isIndirect).getType() != funcType.getParamType(i))
1218	return emitOpError() << "operand type mismatch for operand " << i << ": "
1219	<< getOperand(i + isIndirect).getType()
1220	<< " != " << funcType.getParamType(i);
1221
1222	if (getNumResults() == 0 &&
1223	!llvm::isa<LLVM::LLVMVoidType>(funcType.getReturnType()))
1224	return emitOpError() << "expected function call to produce a value";
1225
1226	if (getNumResults() != 0 &&
1227	llvm::isa<LLVM::LLVMVoidType>(funcType.getReturnType()))
1228	return emitOpError()
1229	<< "calling function with void result must not produce values";
1230
1231	if (getNumResults() > 1)
1232	return emitOpError()
1233	<< "expected LLVM function call to produce 0 or 1 result";
1234
1235	if (getNumResults() && getResult().getType() != funcType.getReturnType())
1236	return emitOpError() << "result type mismatch: " << getResult().getType()
1237	<< " != " << funcType.getReturnType();
1238
1239	return success();
1240	}
1241
1242	void CallOp::print(OpAsmPrinter &p) {
1243	auto callee = getCallee();
1244	bool isDirect = callee.has_value();
1245
1246	p << ' ';
1247
1248	// Print calling convention.
1249	if (getCConv() != LLVM::CConv::C)
1250	p << stringifyCConv(getCConv()) << ' ';
1251
1252	if (getTailCallKind() != LLVM::TailCallKind::None)
1253	p << tailcallkind::stringifyTailCallKind(getTailCallKind()) << ' ';
1254
1255	// Print the direct callee if present as a function attribute, or an indirect
1256	// callee (first operand) otherwise.
1257	if (isDirect)
1258	p.printSymbolName(callee.value());
1259	else
1260	p << getOperand(0);
1261
1262	auto args = getCalleeOperands().drop_front(isDirect ? 0 : 1);
1263	p << '(' << args << ')';
1264
1265	// Print the variadic callee type if the call is variadic.
1266	if (std::optional<LLVMFunctionType> varCalleeType = getVarCalleeType())
1267	p << " vararg(" << *varCalleeType << ")";
1268
1269	if (!getOpBundleOperands().empty()) {
1270	p << " ";
1271	printOpBundles(p, *this, getOpBundleOperands(),
1272	getOpBundleOperands().getTypes(), getOpBundleTags());
1273	}
1274
1275	p.printOptionalAttrDict(processFMFAttr((*this)->getAttrs()),
1276	{getCalleeAttrName(), getTailCallKindAttrName(),
1277	getVarCalleeTypeAttrName(), getCConvAttrName(),
1278	getOperandSegmentSizesAttrName(),
1279	getOpBundleSizesAttrName(),
1280	getOpBundleTagsAttrName(), getArgAttrsAttrName(),
1281	getResAttrsAttrName()});
1282
1283	p << " : ";
1284	if (!isDirect)
1285	p << getOperand(0).getType() << ", ";
1286
1287	// Reconstruct the MLIR function type from operand and result types.
1288	call_interface_impl::printFunctionSignature(
1289	p, args.getTypes(), getArgAttrsAttr(),
1290	/*isVariadic=*/false, getResultTypes(), getResAttrsAttr());
1291	}
1292
1293	/// Parses the type of a call operation and resolves the operands if the parsing
1294	/// succeeds. Returns failure otherwise.
1295	static ParseResult parseCallTypeAndResolveOperands(
1296	OpAsmParser &parser, OperationState &result, bool isDirect,
1297	ArrayRef<OpAsmParser::UnresolvedOperand> operands,
1298	SmallVectorImpl<DictionaryAttr> &argAttrs,
1299	SmallVectorImpl<DictionaryAttr> &resultAttrs) {
1300	SMLoc trailingTypesLoc = parser.getCurrentLocation();
1301	SmallVector<Type> types;
1302	if (parser.parseColon())
1303	return failure();
1304	if (!isDirect) {
1305	types.emplace_back();
1306	if (parser.parseType(result&: types.back()))
1307	return failure();
1308	if (parser.parseOptionalComma())
1309	return parser.emitError(
1310	loc: trailingTypesLoc, message: "expected indirect call to have 2 trailing types");
1311	}
1312	SmallVector<Type> argTypes;
1313	SmallVector<Type> resTypes;
1314	if (call_interface_impl::parseFunctionSignature(parser, argTypes, argAttrs,
1315	resultTypes&: resTypes, resultAttrs)) {
1316	if (isDirect)
1317	return parser.emitError(loc: trailingTypesLoc,
1318	message: "expected direct call to have 1 trailing types");
1319	return parser.emitError(loc: trailingTypesLoc,
1320	message: "expected trailing function type");
1321	}
1322
1323	if (resTypes.size() > 1)
1324	return parser.emitError(loc: trailingTypesLoc,
1325	message: "expected function with 0 or 1 result");
1326	if (resTypes.size() == 1 && llvm::isa<LLVM::LLVMVoidType>(Val: resTypes[0]))
1327	return parser.emitError(loc: trailingTypesLoc,
1328	message: "expected a non-void result type");
1329
1330	// The head element of the types list matches the callee type for
1331	// indirect calls, while the types list is emtpy for direct calls.
1332	// Append the function input types to resolve the call operation
1333	// operands.
1334	llvm::append_range(C&: types, R&: argTypes);
1335	if (parser.resolveOperands(operands, types, loc: parser.getNameLoc(),
1336	result&: result.operands))
1337	return failure();
1338	if (resTypes.size() != 0)
1339	result.addTypes(newTypes: resTypes);
1340
1341	return success();
1342	}
1343
1344	/// Parses an optional function pointer operand before the call argument list
1345	/// for indirect calls, or stops parsing at the function identifier otherwise.
1346	static ParseResult parseOptionalCallFuncPtr(
1347	OpAsmParser &parser,
1348	SmallVectorImpl<OpAsmParser::UnresolvedOperand> &operands) {
1349	OpAsmParser::UnresolvedOperand funcPtrOperand;
1350	OptionalParseResult parseResult = parser.parseOptionalOperand(result&: funcPtrOperand);
1351	if (parseResult.has_value()) {
1352	if (failed(Result: *parseResult))
1353	return *parseResult;
1354	operands.push_back(Elt: funcPtrOperand);
1355	}
1356	return success();
1357	}
1358
1359	static ParseResult resolveOpBundleOperands(
1360	OpAsmParser &parser, SMLoc loc, OperationState &state,
1361	ArrayRef<SmallVector<OpAsmParser::UnresolvedOperand>> opBundleOperands,
1362	ArrayRef<SmallVector<Type>> opBundleOperandTypes,
1363	StringAttr opBundleSizesAttrName) {
1364	unsigned opBundleIndex = 0;
1365	for (const auto &[operands, types] :
1366	llvm::zip_equal(t&: opBundleOperands, u&: opBundleOperandTypes)) {
1367	if (operands.size() != types.size())
1368	return parser.emitError(loc, message: "expected ")
1369	<< operands.size()
1370	<< " types for operand bundle operands for operand bundle #"
1371	<< opBundleIndex << ", but actually got " << types.size();
1372	if (parser.resolveOperands(operands, types, loc, result&: state.operands))
1373	return failure();
1374	}
1375
1376	SmallVector<int32_t> opBundleSizes;
1377	opBundleSizes.reserve(N: opBundleOperands.size());
1378	for (const auto &operands : opBundleOperands)
1379	opBundleSizes.push_back(Elt: operands.size());
1380
1381	state.addAttribute(
1382	opBundleSizesAttrName,
1383	DenseI32ArrayAttr::get(parser.getContext(), opBundleSizes));
1384
1385	return success();
1386	}
1387
1388	// <operation> ::= `llvm.call` (cconv)? (tailcallkind)? (function-id | ssa-use)
1389	// `(` ssa-use-list `)`
1390	// ( `vararg(` var-callee-type `)` )?
1391	// ( `[` op-bundles-list `]` )?
1392	// attribute-dict? `:` (type `,`)? function-type
1393	ParseResult CallOp::parse(OpAsmParser &parser, OperationState &result) {
1394	SymbolRefAttr funcAttr;
1395	TypeAttr varCalleeType;
1396	SmallVector<OpAsmParser::UnresolvedOperand> operands;
1397	SmallVector<SmallVector<OpAsmParser::UnresolvedOperand>> opBundleOperands;
1398	SmallVector<SmallVector<Type>> opBundleOperandTypes;
1399	ArrayAttr opBundleTags;
1400
1401	// Default to C Calling Convention if no keyword is provided.
1402	result.addAttribute(
1403	getCConvAttrName(result.name),
1404	CConvAttr::get(parser.getContext(), parseOptionalLLVMKeyword<CConv>(
1405	parser, result, LLVM::CConv::C)));
1406
1407	result.addAttribute(
1408	getTailCallKindAttrName(result.name),
1409	TailCallKindAttr::get(parser.getContext(),
1410	parseOptionalLLVMKeyword<TailCallKind>(
1411	parser, result, LLVM::TailCallKind::None)));
1412
1413	// Parse a function pointer for indirect calls.
1414	if (parseOptionalCallFuncPtr(parser, operands))
1415	return failure();
1416	bool isDirect = operands.empty();
1417
1418	// Parse a function identifier for direct calls.
1419	if (isDirect)
1420	if (parser.parseAttribute(funcAttr, "callee", result.attributes))
1421	return failure();
1422
1423	// Parse the function arguments.
1424	if (parser.parseOperandList(operands, OpAsmParser::Delimiter::Paren))
1425	return failure();
1426
1427	bool isVarArg = parser.parseOptionalKeyword("vararg").succeeded();
1428	if (isVarArg) {
1429	StringAttr varCalleeTypeAttrName =
1430	CallOp::getVarCalleeTypeAttrName(result.name);
1431	if (parser.parseLParen().failed() ||
1432	parser
1433	.parseAttribute(varCalleeType, varCalleeTypeAttrName,
1434	result.attributes)
1435	.failed() ||
1436	parser.parseRParen().failed())
1437	return failure();
1438	}
1439
1440	SMLoc opBundlesLoc = parser.getCurrentLocation();
1441	if (std::optional<ParseResult> result = parseOpBundles(
1442	parser, opBundleOperands, opBundleOperandTypes, opBundleTags);
1443	result && failed(*result))
1444	return failure();
1445	if (opBundleTags && !opBundleTags.empty())
1446	result.addAttribute(CallOp::getOpBundleTagsAttrName(result.name).getValue(),
1447	opBundleTags);
1448
1449	if (parser.parseOptionalAttrDict(result.attributes))
1450	return failure();
1451
1452	// Parse the trailing type list and resolve the operands.
1453	SmallVector<DictionaryAttr> argAttrs;
1454	SmallVector<DictionaryAttr> resultAttrs;
1455	if (parseCallTypeAndResolveOperands(parser, result, isDirect, operands,
1456	argAttrs, resultAttrs))
1457	return failure();
1458	call_interface_impl::addArgAndResultAttrs(
1459	parser.getBuilder(), result, argAttrs, resultAttrs,
1460	getArgAttrsAttrName(result.name), getResAttrsAttrName(result.name));
1461	if (resolveOpBundleOperands(parser, opBundlesLoc, result, opBundleOperands,
1462	opBundleOperandTypes,
1463	getOpBundleSizesAttrName(result.name)))
1464	return failure();
1465
1466	int32_t numOpBundleOperands = 0;
1467	for (const auto &operands : opBundleOperands)
1468	numOpBundleOperands += operands.size();
1469
1470	result.addAttribute(
1471	CallOp::getOperandSegmentSizeAttr(),
1472	parser.getBuilder().getDenseI32ArrayAttr(
1473	{static_cast<int32_t>(operands.size()), numOpBundleOperands}));
1474	return success();
1475	}
1476
1477	LLVMFunctionType CallOp::getCalleeFunctionType() {
1478	if (std::optional<LLVMFunctionType> varCalleeType = getVarCalleeType())
1479	return *varCalleeType;
1480	return getLLVMFuncType(getContext(), getResultTypes(), getArgOperands());
1481	}
1482
1483	///===---------------------------------------------------------------------===//
1484	/// LLVM::InvokeOp
1485	///===---------------------------------------------------------------------===//
1486
1487	void InvokeOp::build(OpBuilder &builder, OperationState &state, LLVMFuncOp func,
1488	ValueRange ops, Block *normal, ValueRange normalOps,
1489	Block *unwind, ValueRange unwindOps) {
1490	auto calleeType = func.getFunctionType();
1491	build(builder, state, getCallOpResultTypes(calleeType),
1492	getCallOpVarCalleeType(calleeType), SymbolRefAttr::get(func), ops,
1493	/*arg_attrs=*/nullptr, /*res_attrs=*/nullptr, normalOps, unwindOps,
1494	nullptr, nullptr, {}, {}, normal, unwind);
1495	}
1496
1497	void InvokeOp::build(OpBuilder &builder, OperationState &state, TypeRange tys,
1498	FlatSymbolRefAttr callee, ValueRange ops, Block *normal,
1499	ValueRange normalOps, Block *unwind,
1500	ValueRange unwindOps) {
1501	build(builder, state, tys,
1502	/*var_callee_type=*/nullptr, callee, ops, /*arg_attrs=*/nullptr,
1503	/*res_attrs=*/nullptr, normalOps, unwindOps, nullptr, nullptr, {}, {},
1504	normal, unwind);
1505	}
1506
1507	void InvokeOp::build(OpBuilder &builder, OperationState &state,
1508	LLVMFunctionType calleeType, FlatSymbolRefAttr callee,
1509	ValueRange ops, Block *normal, ValueRange normalOps,
1510	Block *unwind, ValueRange unwindOps) {
1511	build(builder, state, getCallOpResultTypes(calleeType),
1512	getCallOpVarCalleeType(calleeType), callee, ops,
1513	/*arg_attrs=*/nullptr, /*res_attrs=*/nullptr, normalOps, unwindOps,
1514	nullptr, nullptr, {}, {}, normal, unwind);
1515	}
1516
1517	SuccessorOperands InvokeOp::getSuccessorOperands(unsigned index) {
1518	assert(index < getNumSuccessors() && "invalid successor index");
1519	return SuccessorOperands(index == 0 ? getNormalDestOperandsMutable()
1520	: getUnwindDestOperandsMutable());
1521	}
1522
1523	CallInterfaceCallable InvokeOp::getCallableForCallee() {
1524	// Direct call.
1525	if (FlatSymbolRefAttr calleeAttr = getCalleeAttr())
1526	return calleeAttr;
1527	// Indirect call, callee Value is the first operand.
1528	return getOperand(0);
1529	}
1530
1531	void InvokeOp::setCalleeFromCallable(CallInterfaceCallable callee) {
1532	// Direct call.
1533	if (FlatSymbolRefAttr calleeAttr = getCalleeAttr()) {
1534	auto symRef = cast<SymbolRefAttr>(callee);
1535	return setCalleeAttr(cast<FlatSymbolRefAttr>(symRef));
1536	}
1537	// Indirect call, callee Value is the first operand.
1538	return setOperand(0, cast<Value>(callee));
1539	}
1540
1541	Operation::operand_range InvokeOp::getArgOperands() {
1542	return getCalleeOperands().drop_front(getCallee().has_value() ? 0 : 1);
1543	}
1544
1545	MutableOperandRange InvokeOp::getArgOperandsMutable() {
1546	return MutableOperandRange(*this, getCallee().has_value() ? 0 : 1,
1547	getCalleeOperands().size());
1548	}
1549
1550	LogicalResult InvokeOp::verify() {
1551	if (failed(verifyCallOpVarCalleeType(*this)))
1552	return failure();
1553
1554	Block *unwindDest = getUnwindDest();
1555	if (unwindDest->empty())
1556	return emitError("must have at least one operation in unwind destination");
1557
1558	// In unwind destination, first operation must be LandingpadOp
1559	if (!isa<LandingpadOp>(unwindDest->front()))
1560	return emitError("first operation in unwind destination should be a "
1561	"llvm.landingpad operation");
1562
1563	if (failed(verifyOperandBundles(*this)))
1564	return failure();
1565
1566	return success();
1567	}
1568
1569	void InvokeOp::print(OpAsmPrinter &p) {
1570	auto callee = getCallee();
1571	bool isDirect = callee.has_value();
1572
1573	p << ' ';
1574
1575	// Print calling convention.
1576	if (getCConv() != LLVM::CConv::C)
1577	p << stringifyCConv(getCConv()) << ' ';
1578
1579	// Either function name or pointer
1580	if (isDirect)
1581	p.printSymbolName(callee.value());
1582	else
1583	p << getOperand(0);
1584
1585	p << '(' << getCalleeOperands().drop_front(isDirect ? 0 : 1) << ')';
1586	p << " to ";
1587	p.printSuccessorAndUseList(getNormalDest(), getNormalDestOperands());
1588	p << " unwind ";
1589	p.printSuccessorAndUseList(getUnwindDest(), getUnwindDestOperands());
1590
1591	// Print the variadic callee type if the invoke is variadic.
1592	if (std::optional<LLVMFunctionType> varCalleeType = getVarCalleeType())
1593	p << " vararg(" << *varCalleeType << ")";
1594
1595	if (!getOpBundleOperands().empty()) {
1596	p << " ";
1597	printOpBundles(p, *this, getOpBundleOperands(),
1598	getOpBundleOperands().getTypes(), getOpBundleTags());
1599	}
1600
1601	p.printOptionalAttrDict((*this)->getAttrs(),
1602	{getCalleeAttrName(), getOperandSegmentSizeAttr(),
1603	getCConvAttrName(), getVarCalleeTypeAttrName(),
1604	getOpBundleSizesAttrName(),
1605	getOpBundleTagsAttrName(), getArgAttrsAttrName(),
1606	getResAttrsAttrName()});
1607
1608	p << " : ";
1609	if (!isDirect)
1610	p << getOperand(0).getType() << ", ";
1611	call_interface_impl::printFunctionSignature(
1612	p, getCalleeOperands().drop_front(isDirect ? 0 : 1).getTypes(),
1613	getArgAttrsAttr(),
1614	/*isVariadic=*/false, getResultTypes(), getResAttrsAttr());
1615	}
1616
1617	// <operation> ::= `llvm.invoke` (cconv)? (function-id | ssa-use)
1618	// `(` ssa-use-list `)`
1619	// `to` bb-id (`[` ssa-use-and-type-list `]`)?
1620	// `unwind` bb-id (`[` ssa-use-and-type-list `]`)?
1621	// ( `vararg(` var-callee-type `)` )?
1622	// ( `[` op-bundles-list `]` )?
1623	// attribute-dict? `:` (type `,`)?
1624	// function-type-with-argument-attributes
1625	ParseResult InvokeOp::parse(OpAsmParser &parser, OperationState &result) {
1626	SmallVector<OpAsmParser::UnresolvedOperand, 8> operands;
1627	SymbolRefAttr funcAttr;
1628	TypeAttr varCalleeType;
1629	SmallVector<SmallVector<OpAsmParser::UnresolvedOperand>> opBundleOperands;
1630	SmallVector<SmallVector<Type>> opBundleOperandTypes;
1631	ArrayAttr opBundleTags;
1632	Block *normalDest, *unwindDest;
1633	SmallVector<Value, 4> normalOperands, unwindOperands;
1634	Builder &builder = parser.getBuilder();
1635
1636	// Default to C Calling Convention if no keyword is provided.
1637	result.addAttribute(
1638	getCConvAttrName(result.name),
1639	CConvAttr::get(parser.getContext(), parseOptionalLLVMKeyword<CConv>(
1640	parser, result, LLVM::CConv::C)));
1641
1642	// Parse a function pointer for indirect calls.
1643	if (parseOptionalCallFuncPtr(parser, operands))
1644	return failure();
1645	bool isDirect = operands.empty();
1646
1647	// Parse a function identifier for direct calls.
1648	if (isDirect && parser.parseAttribute(funcAttr, "callee", result.attributes))
1649	return failure();
1650
1651	// Parse the function arguments.
1652	if (parser.parseOperandList(operands, OpAsmParser::Delimiter::Paren) ||
1653	parser.parseKeyword("to") ||
1654	parser.parseSuccessorAndUseList(normalDest, normalOperands) ||
1655	parser.parseKeyword("unwind") ||
1656	parser.parseSuccessorAndUseList(unwindDest, unwindOperands))
1657	return failure();
1658
1659	bool isVarArg = parser.parseOptionalKeyword("vararg").succeeded();
1660	if (isVarArg) {
1661	StringAttr varCalleeTypeAttrName =
1662	InvokeOp::getVarCalleeTypeAttrName(result.name);
1663	if (parser.parseLParen().failed() ||
1664	parser
1665	.parseAttribute(varCalleeType, varCalleeTypeAttrName,
1666	result.attributes)
1667	.failed() ||
1668	parser.parseRParen().failed())
1669	return failure();
1670	}
1671
1672	SMLoc opBundlesLoc = parser.getCurrentLocation();
1673	if (std::optional<ParseResult> result = parseOpBundles(
1674	parser, opBundleOperands, opBundleOperandTypes, opBundleTags);
1675	result && failed(*result))
1676	return failure();
1677	if (opBundleTags && !opBundleTags.empty())
1678	result.addAttribute(
1679	InvokeOp::getOpBundleTagsAttrName(result.name).getValue(),
1680	opBundleTags);
1681
1682	if (parser.parseOptionalAttrDict(result.attributes))
1683	return failure();
1684
1685	// Parse the trailing type list and resolve the function operands.
1686	SmallVector<DictionaryAttr> argAttrs;
1687	SmallVector<DictionaryAttr> resultAttrs;
1688	if (parseCallTypeAndResolveOperands(parser, result, isDirect, operands,
1689	argAttrs, resultAttrs))
1690	return failure();
1691	call_interface_impl::addArgAndResultAttrs(
1692	parser.getBuilder(), result, argAttrs, resultAttrs,
1693	getArgAttrsAttrName(result.name), getResAttrsAttrName(result.name));
1694
1695	if (resolveOpBundleOperands(parser, opBundlesLoc, result, opBundleOperands,
1696	opBundleOperandTypes,
1697	getOpBundleSizesAttrName(result.name)))
1698	return failure();
1699
1700	result.addSuccessors({normalDest, unwindDest});
1701	result.addOperands(normalOperands);
1702	result.addOperands(unwindOperands);
1703
1704	int32_t numOpBundleOperands = 0;
1705	for (const auto &operands : opBundleOperands)
1706	numOpBundleOperands += operands.size();
1707
1708	result.addAttribute(
1709	InvokeOp::getOperandSegmentSizeAttr(),
1710	builder.getDenseI32ArrayAttr({static_cast<int32_t>(operands.size()),
1711	static_cast<int32_t>(normalOperands.size()),
1712	static_cast<int32_t>(unwindOperands.size()),
1713	numOpBundleOperands}));
1714	return success();
1715	}
1716
1717	LLVMFunctionType InvokeOp::getCalleeFunctionType() {
1718	if (std::optional<LLVMFunctionType> varCalleeType = getVarCalleeType())
1719	return *varCalleeType;
1720	return getLLVMFuncType(getContext(), getResultTypes(), getArgOperands());
1721	}
1722
1723	///===----------------------------------------------------------------------===//
1724	/// Verifying/Printing/Parsing for LLVM::LandingpadOp.
1725	///===----------------------------------------------------------------------===//
1726
1727	LogicalResult LandingpadOp::verify() {
1728	Value value;
1729	if (LLVMFuncOp func = (*this)->getParentOfType<LLVMFuncOp>()) {
1730	if (!func.getPersonality())
1731	return emitError(
1732	"llvm.landingpad needs to be in a function with a personality");
1733	}
1734
1735	// Consistency of llvm.landingpad result types is checked in
1736	// LLVMFuncOp::verify().
1737
1738	if (!getCleanup() && getOperands().empty())
1739	return emitError("landingpad instruction expects at least one clause or "
1740	"cleanup attribute");
1741
1742	for (unsigned idx = 0, ie = getNumOperands(); idx < ie; idx++) {
1743	value = getOperand(idx);
1744	bool isFilter = llvm::isa<LLVMArrayType>(value.getType());
1745	if (isFilter) {
1746	// FIXME: Verify filter clauses when arrays are appropriately handled
1747	} else {
1748	// catch - global addresses only.
1749	// Bitcast ops should have global addresses as their args.
1750	if (auto bcOp = value.getDefiningOp<BitcastOp>()) {
1751	if (auto addrOp = bcOp.getArg().getDefiningOp<AddressOfOp>())
1752	continue;
1753	return emitError("constant clauses expected").attachNote(bcOp.getLoc())
1754	<< "global addresses expected as operand to "
1755	"bitcast used in clauses for landingpad";
1756	}
1757	// ZeroOp and AddressOfOp allowed
1758	if (value.getDefiningOp<ZeroOp>())
1759	continue;
1760	if (value.getDefiningOp<AddressOfOp>())
1761	continue;
1762	return emitError("clause #")
1763	<< idx << " is not a known constant - null, addressof, bitcast";
1764	}
1765	}
1766	return success();
1767	}
1768
1769	void LandingpadOp::print(OpAsmPrinter &p) {
1770	p << (getCleanup() ? " cleanup " : " ");
1771
1772	// Clauses
1773	for (auto value : getOperands()) {
1774	// Similar to llvm - if clause is an array type then it is filter
1775	// clause else catch clause
1776	bool isArrayTy = llvm::isa<LLVMArrayType>(value.getType());
1777	p << '(' << (isArrayTy ? "filter " : "catch ") << value << " : "
1778	<< value.getType() << ") ";
1779	}
1780
1781	p.printOptionalAttrDict((*this)->getAttrs(), {"cleanup"});
1782
1783	p << ": " << getType();
1784	}
1785
1786	// <operation> ::= `llvm.landingpad` `cleanup`?
1787	// ((`catch` | `filter`) operand-type ssa-use)* attribute-dict?
1788	ParseResult LandingpadOp::parse(OpAsmParser &parser, OperationState &result) {
1789	// Check for cleanup
1790	if (succeeded(parser.parseOptionalKeyword("cleanup")))
1791	result.addAttribute("cleanup", parser.getBuilder().getUnitAttr());
1792
1793	// Parse clauses with types
1794	while (succeeded(parser.parseOptionalLParen()) &&
1795	(succeeded(parser.parseOptionalKeyword("filter")) ||
1796	succeeded(parser.parseOptionalKeyword("catch")))) {
1797	OpAsmParser::UnresolvedOperand operand;
1798	Type ty;
1799	if (parser.parseOperand(operand) || parser.parseColon() ||
1800	parser.parseType(ty) ||
1801	parser.resolveOperand(operand, ty, result.operands) ||
1802	parser.parseRParen())
1803	return failure();
1804	}
1805
1806	Type type;
1807	if (parser.parseColon() || parser.parseType(type))
1808	return failure();
1809
1810	result.addTypes(type);
1811	return success();
1812	}
1813
1814	//===----------------------------------------------------------------------===//
1815	// ExtractValueOp
1816	//===----------------------------------------------------------------------===//
1817
1818	/// Extract the type at `position` in the LLVM IR aggregate type
1819	/// `containerType`. Each element of `position` is an index into a nested
1820	/// aggregate type. Return the resulting type or emit an error.
1821	static Type getInsertExtractValueElementType(
1822	function_ref<InFlightDiagnostic(StringRef)> emitError, Type containerType,
1823	ArrayRef<int64_t> position) {
1824	Type llvmType = containerType;
1825	if (!isCompatibleType(type: containerType)) {
1826	emitError("expected LLVM IR Dialect type, got ") << containerType;
1827	return {};
1828	}
1829
1830	// Infer the element type from the structure type: iteratively step inside the
1831	// type by taking the element type, indexed by the position attribute for
1832	// structures. Check the position index before accessing, it is supposed to
1833	// be in bounds.
1834	for (int64_t idx : position) {
1835	if (auto arrayType = llvm::dyn_cast<LLVMArrayType>(llvmType)) {
1836	if (idx < 0 || static_cast<unsigned>(idx) >= arrayType.getNumElements()) {
1837	emitError("position out of bounds: ") << idx;
1838	return {};
1839	}
1840	llvmType = arrayType.getElementType();
1841	} else if (auto structType = llvm::dyn_cast<LLVMStructType>(llvmType)) {
1842	if (idx < 0 ||
1843	static_cast<unsigned>(idx) >= structType.getBody().size()) {
1844	emitError("position out of bounds: ") << idx;
1845	return {};
1846	}
1847	llvmType = structType.getBody()[idx];
1848	} else {
1849	emitError("expected LLVM IR structure/array type, got: ") << llvmType;
1850	return {};
1851	}
1852	}
1853	return llvmType;
1854	}
1855
1856	/// Extract the type at `position` in the wrapped LLVM IR aggregate type
1857	/// `containerType`.
1858	static Type getInsertExtractValueElementType(Type llvmType,
1859	ArrayRef<int64_t> position) {
1860	for (int64_t idx : position) {
1861	if (auto structType = llvm::dyn_cast<LLVMStructType>(llvmType))
1862	llvmType = structType.getBody()[idx];
1863	else
1864	llvmType = llvm::cast<LLVMArrayType>(llvmType).getElementType();
1865	}
1866	return llvmType;
1867	}
1868
1869	OpFoldResult LLVM::ExtractValueOp::fold(FoldAdaptor adaptor) {
1870	if (auto extractValueOp = getContainer().getDefiningOp<ExtractValueOp>()) {
1871	SmallVector<int64_t, 4> newPos(extractValueOp.getPosition());
1872	newPos.append(getPosition().begin(), getPosition().end());
1873	setPosition(newPos);
1874	getContainerMutable().set(extractValueOp.getContainer());
1875	return getResult();
1876	}
1877
1878	{
1879	DenseElementsAttr constval;
1880	matchPattern(getContainer(), m_Constant(&constval));
1881	if (constval && constval.getElementType() == getType()) {
1882	if (isa<SplatElementsAttr>(constval))
1883	return constval.getSplatValue<Attribute>();
1884	if (getPosition().size() == 1)
1885	return constval.getValues<Attribute>()[getPosition()[0]];
1886	}
1887	}
1888
1889	auto insertValueOp = getContainer().getDefiningOp<InsertValueOp>();
1890	OpFoldResult result = {};
1891	ArrayRef<int64_t> extractPos = getPosition();
1892	bool switchedToInsertedValue = false;
1893	while (insertValueOp) {
1894	ArrayRef<int64_t> insertPos = insertValueOp.getPosition();
1895	auto extractPosSize = extractPos.size();
1896	auto insertPosSize = insertPos.size();
1897
1898	// Case 1: Exact match of positions.
1899	if (extractPos == insertPos)
1900	return insertValueOp.getValue();
1901
1902	// Case 2: Insert position is a prefix of extract position. Continue
1903	// traversal with the inserted value. Example:
1904	// ```
1905	// %0 = llvm.insertvalue %arg1, %undef[0] : !llvm.struct<(i32, i32, i32)>
1906	// %1 = llvm.insertvalue %arg2, %0[1] : !llvm.struct<(i32, i32, i32)>
1907	// %2 = llvm.insertvalue %arg3, %1[2] : !llvm.struct<(i32, i32, i32)>
1908	// %3 = llvm.insertvalue %2, %foo[0]
1909	// : !llvm.struct<(struct<(i32, i32, i32)>, i64)>
1910	// %4 = llvm.extractvalue %3[0, 0]
1911	// : !llvm.struct<(struct<(i32, i32, i32)>, i64)>
1912	// ```
1913	// In the above example, %4 is folded to %arg1.
1914	if (extractPosSize > insertPosSize &&
1915	extractPos.take_front(insertPosSize) == insertPos) {
1916	insertValueOp = insertValueOp.getValue().getDefiningOp<InsertValueOp>();
1917	extractPos = extractPos.drop_front(insertPosSize);
1918	switchedToInsertedValue = true;
1919	continue;
1920	}
1921
1922	// Case 3: Try to continue the traversal with the container value.
1923	unsigned min = std::min(extractPosSize, insertPosSize);
1924
1925	// If one is fully prefix of the other, stop propagating back as it will
1926	// miss dependencies. For instance, %3 should not fold to %f0 in the
1927	// following example:
1928	// ```
1929	// %1 = llvm.insertvalue %f0, %0[0, 0] :
1930	// !llvm.array<4 x !llvm.array<4 x f32>>
1931	// %2 = llvm.insertvalue %arr, %1[0] :
1932	// !llvm.array<4 x !llvm.array<4 x f32>>
1933	// %3 = llvm.extractvalue %2[0, 0] : !llvm.array<4 x !llvm.array<4 x f32>>
1934	// ```
1935	if (extractPos.take_front(min) == insertPos.take_front(min))
1936	return result;
1937	// If neither a prefix, nor the exact position, we can extract out of the
1938	// value being inserted into. Moreover, we can try again if that operand
1939	// is itself an insertvalue expression.
1940	if (!switchedToInsertedValue) {
1941	// Do not swap out the container operand if we decided earlier to
1942	// continue the traversal with the inserted value (Case 2).
1943	getContainerMutable().assign(insertValueOp.getContainer());
1944	result = getResult();
1945	}
1946	insertValueOp = insertValueOp.getContainer().getDefiningOp<InsertValueOp>();
1947	}
1948	return result;
1949	}
1950
1951	LogicalResult ExtractValueOp::verify() {
1952	auto emitError = [this](StringRef msg) { return emitOpError(msg); };
1953	Type valueType = getInsertExtractValueElementType(
1954	emitError, getContainer().getType(), getPosition());
1955	if (!valueType)
1956	return failure();
1957
1958	if (getRes().getType() != valueType)
1959	return emitOpError() << "Type mismatch: extracting from "
1960	<< getContainer().getType() << " should produce "
1961	<< valueType << " but this op returns "
1962	<< getRes().getType();
1963	return success();
1964	}
1965
1966	void ExtractValueOp::build(OpBuilder &builder, OperationState &state,
1967	Value container, ArrayRef<int64_t> position) {
1968	build(builder, state,
1969	getInsertExtractValueElementType(container.getType(), position),
1970	container, builder.getAttr<DenseI64ArrayAttr>(position));
1971	}
1972
1973	//===----------------------------------------------------------------------===//
1974	// InsertValueOp
1975	//===----------------------------------------------------------------------===//
1976
1977	/// Infer the value type from the container type and position.
1978	static ParseResult
1979	parseInsertExtractValueElementType(AsmParser &parser, Type &valueType,
1980	Type containerType,
1981	DenseI64ArrayAttr position) {
1982	valueType = getInsertExtractValueElementType(
1983	[&](StringRef msg) {
1984	return parser.emitError(loc: parser.getCurrentLocation(), message: msg);
1985	},
1986	containerType, position.asArrayRef());
1987	return success(IsSuccess: !!valueType);
1988	}
1989
1990	/// Nothing to print for an inferred type.
1991	static void printInsertExtractValueElementType(AsmPrinter &printer,
1992	Operation *op, Type valueType,
1993	Type containerType,
1994	DenseI64ArrayAttr position) {}
1995
1996	LogicalResult InsertValueOp::verify() {
1997	auto emitError = [this](StringRef msg) { return emitOpError(msg); };
1998	Type valueType = getInsertExtractValueElementType(
1999	emitError, getContainer().getType(), getPosition());
2000	if (!valueType)
2001	return failure();
2002
2003	if (getValue().getType() != valueType)
2004	return emitOpError() << "Type mismatch: cannot insert "
2005	<< getValue().getType() << " into "
2006	<< getContainer().getType();
2007
2008	return success();
2009	}
2010
2011	//===----------------------------------------------------------------------===//
2012	// ReturnOp
2013	//===----------------------------------------------------------------------===//
2014
2015	LogicalResult ReturnOp::verify() {
2016	auto parent = (*this)->getParentOfType<LLVMFuncOp>();
2017	if (!parent)
2018	return success();
2019
2020	Type expectedType = parent.getFunctionType().getReturnType();
2021	if (llvm::isa<LLVMVoidType>(expectedType)) {
2022	if (!getArg())
2023	return success();
2024	InFlightDiagnostic diag = emitOpError("expected no operands");
2025	diag.attachNote(parent->getLoc()) << "when returning from function";
2026	return diag;
2027	}
2028	if (!getArg()) {
2029	if (llvm::isa<LLVMVoidType>(expectedType))
2030	return success();
2031	InFlightDiagnostic diag = emitOpError("expected 1 operand");
2032	diag.attachNote(parent->getLoc()) << "when returning from function";
2033	return diag;
2034	}
2035	if (expectedType != getArg().getType()) {
2036	InFlightDiagnostic diag = emitOpError("mismatching result types");
2037	diag.attachNote(parent->getLoc()) << "when returning from function";
2038	return diag;
2039	}
2040	return success();
2041	}
2042
2043	//===----------------------------------------------------------------------===//
2044	// LLVM::AddressOfOp.
2045	//===----------------------------------------------------------------------===//
2046
2047	static Operation *parentLLVMModule(Operation *op) {
2048	Operation *module = op->getParentOp();
2049	while (module && !satisfiesLLVMModule(op: module))
2050	module = module->getParentOp();
2051	assert(module && "unexpected operation outside of a module");
2052	return module;
2053	}
2054
2055	GlobalOp AddressOfOp::getGlobal(SymbolTableCollection &symbolTable) {
2056	return dyn_cast_or_null<GlobalOp>(
2057	symbolTable.lookupSymbolIn(parentLLVMModule(*this), getGlobalNameAttr()));
2058	}
2059
2060	LLVMFuncOp AddressOfOp::getFunction(SymbolTableCollection &symbolTable) {
2061	return dyn_cast_or_null<LLVMFuncOp>(
2062	symbolTable.lookupSymbolIn(parentLLVMModule(*this), getGlobalNameAttr()));
2063	}
2064
2065	AliasOp AddressOfOp::getAlias(SymbolTableCollection &symbolTable) {
2066	return dyn_cast_or_null<AliasOp>(
2067	symbolTable.lookupSymbolIn(parentLLVMModule(*this), getGlobalNameAttr()));
2068	}
2069
2070	LogicalResult
2071	AddressOfOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
2072	Operation *symbol =
2073	symbolTable.lookupSymbolIn(parentLLVMModule(*this), getGlobalNameAttr());
2074
2075	auto global = dyn_cast_or_null<GlobalOp>(symbol);
2076	auto function = dyn_cast_or_null<LLVMFuncOp>(symbol);
2077	auto alias = dyn_cast_or_null<AliasOp>(symbol);
2078
2079	if (!global && !function && !alias)
2080	return emitOpError("must reference a global defined by 'llvm.mlir.global', "
2081	"'llvm.mlir.alias' or 'llvm.func'");
2082
2083	LLVMPointerType type = getType();
2084	if ((global && global.getAddrSpace() != type.getAddressSpace()) ||
2085	(alias && alias.getAddrSpace() != type.getAddressSpace()))
2086	return emitOpError("pointer address space must match address space of the "
2087	"referenced global or alias");
2088
2089	return success();
2090	}
2091
2092	// AddressOfOp constant-folds to the global symbol name.
2093	OpFoldResult LLVM::AddressOfOp::fold(FoldAdaptor) {
2094	return getGlobalNameAttr();
2095	}
2096
2097	//===----------------------------------------------------------------------===//
2098	// LLVM::DSOLocalEquivalentOp
2099	//===----------------------------------------------------------------------===//
2100
2101	LLVMFuncOp
2102	DSOLocalEquivalentOp::getFunction(SymbolTableCollection &symbolTable) {
2103	return dyn_cast_or_null<LLVMFuncOp>(symbolTable.lookupSymbolIn(
2104	parentLLVMModule(*this), getFunctionNameAttr()));
2105	}
2106
2107	AliasOp DSOLocalEquivalentOp::getAlias(SymbolTableCollection &symbolTable) {
2108	return dyn_cast_or_null<AliasOp>(symbolTable.lookupSymbolIn(
2109	parentLLVMModule(*this), getFunctionNameAttr()));
2110	}
2111
2112	LogicalResult
2113	DSOLocalEquivalentOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
2114	Operation *symbol = symbolTable.lookupSymbolIn(parentLLVMModule(*this),
2115	getFunctionNameAttr());
2116	auto function = dyn_cast_or_null<LLVMFuncOp>(symbol);
2117	auto alias = dyn_cast_or_null<AliasOp>(symbol);
2118
2119	if (!function && !alias)
2120	return emitOpError(
2121	"must reference a global defined by 'llvm.func' or 'llvm.mlir.alias'");
2122
2123	if (alias) {
2124	if (alias.getInitializer()
2125	.walk([&](AddressOfOp addrOp) {
2126	if (addrOp.getGlobal(symbolTable))
2127	return WalkResult::interrupt();
2128	return WalkResult::advance();
2129	})
2130	.wasInterrupted())
2131	return emitOpError("must reference an alias to a function");
2132	}
2133
2134	if ((function && function.getLinkage() == LLVM::Linkage::ExternWeak) ||
2135	(alias && alias.getLinkage() == LLVM::Linkage::ExternWeak))
2136	return emitOpError(
2137	"target function with 'extern_weak' linkage not allowed");
2138
2139	return success();
2140	}
2141
2142	/// Fold a dso_local_equivalent operation to a dedicated dso_local_equivalent
2143	/// attribute.
2144	OpFoldResult DSOLocalEquivalentOp::fold(FoldAdaptor) {
2145	return DSOLocalEquivalentAttr::get(getContext(), getFunctionNameAttr());
2146	}
2147
2148	//===----------------------------------------------------------------------===//
2149	// Verifier for LLVM::ComdatOp.
2150	//===----------------------------------------------------------------------===//
2151
2152	void ComdatOp::build(OpBuilder &builder, OperationState &result,
2153	StringRef symName) {
2154	result.addAttribute(getSymNameAttrName(result.name),
2155	builder.getStringAttr(symName));
2156	Region *body = result.addRegion();
2157	body->emplaceBlock();
2158	}
2159
2160	LogicalResult ComdatOp::verifyRegions() {
2161	Region &body = getBody();
2162	for (Operation &op : body.getOps())
2163	if (!isa<ComdatSelectorOp>(op))
2164	return op.emitError(
2165	"only comdat selector symbols can appear in a comdat region");
2166
2167	return success();
2168	}
2169
2170	//===----------------------------------------------------------------------===//
2171	// Builder, printer and verifier for LLVM::GlobalOp.
2172	//===----------------------------------------------------------------------===//
2173
2174	void GlobalOp::build(OpBuilder &builder, OperationState &result, Type type,
2175	bool isConstant, Linkage linkage, StringRef name,
2176	Attribute value, uint64_t alignment, unsigned addrSpace,
2177	bool dsoLocal, bool threadLocal, SymbolRefAttr comdat,
2178	ArrayRef<NamedAttribute> attrs,
2179	ArrayRef<Attribute> dbgExprs) {
2180	result.addAttribute(getSymNameAttrName(result.name),
2181	builder.getStringAttr(name));
2182	result.addAttribute(getGlobalTypeAttrName(result.name), TypeAttr::get(type));
2183	if (isConstant)
2184	result.addAttribute(getConstantAttrName(result.name),
2185	builder.getUnitAttr());
2186	if (value)
2187	result.addAttribute(getValueAttrName(result.name), value);
2188	if (dsoLocal)
2189	result.addAttribute(getDsoLocalAttrName(result.name),
2190	builder.getUnitAttr());
2191	if (threadLocal)
2192	result.addAttribute(getThreadLocal_AttrName(result.name),
2193	builder.getUnitAttr());
2194	if (comdat)
2195	result.addAttribute(getComdatAttrName(result.name), comdat);
2196
2197	// Only add an alignment attribute if the "alignment" input
2198	// is different from 0. The value must also be a power of two, but
2199	// this is tested in GlobalOp::verify, not here.
2200	if (alignment != 0)
2201	result.addAttribute(getAlignmentAttrName(result.name),
2202	builder.getI64IntegerAttr(alignment));
2203
2204	result.addAttribute(getLinkageAttrName(result.name),
2205	LinkageAttr::get(builder.getContext(), linkage));
2206	if (addrSpace != 0)
2207	result.addAttribute(getAddrSpaceAttrName(result.name),
2208	builder.getI32IntegerAttr(addrSpace));
2209	result.attributes.append(attrs.begin(), attrs.end());
2210
2211	if (!dbgExprs.empty())
2212	result.addAttribute(getDbgExprsAttrName(result.name),
2213	ArrayAttr::get(builder.getContext(), dbgExprs));
2214
2215	result.addRegion();
2216	}
2217
2218	template <typename OpType>
2219	static void printCommonGlobalAndAlias(OpAsmPrinter &p, OpType op) {
2220	p << ' ' << stringifyLinkage(op.getLinkage()) << ' ';
2221	StringRef visibility = stringifyVisibility(op.getVisibility_());
2222	if (!visibility.empty())
2223	p << visibility << ' ';
2224	if (op.getThreadLocal_())
2225	p << "thread_local ";
2226	if (auto unnamedAddr = op.getUnnamedAddr()) {
2227	StringRef str = stringifyUnnamedAddr(*unnamedAddr);
2228	if (!str.empty())
2229	p << str << ' ';
2230	}
2231	}
2232
2233	void GlobalOp::print(OpAsmPrinter &p) {
2234	printCommonGlobalAndAlias<GlobalOp>(p, *this);
2235	if (getConstant())
2236	p << "constant ";
2237	p.printSymbolName(getSymName());
2238	p << '(';
2239	if (auto value = getValueOrNull())
2240	p.printAttribute(value);
2241	p << ')';
2242	if (auto comdat = getComdat())
2243	p << " comdat(" << *comdat << ')';
2244
2245	// Note that the alignment attribute is printed using the
2246	// default syntax here, even though it is an inherent attribute
2247	// (as defined in https://mlir.llvm.org/docs/LangRef/#attributes)
2248	p.printOptionalAttrDict((*this)->getAttrs(),
2249	{SymbolTable::getSymbolAttrName(),
2250	getGlobalTypeAttrName(), getConstantAttrName(),
2251	getValueAttrName(), getLinkageAttrName(),
2252	getUnnamedAddrAttrName(), getThreadLocal_AttrName(),
2253	getVisibility_AttrName(), getComdatAttrName(),
2254	getUnnamedAddrAttrName()});
2255
2256	// Print the trailing type unless it's a string global.
2257	if (llvm::dyn_cast_or_null<StringAttr>(getValueOrNull()))
2258	return;
2259	p << " : " << getType();
2260
2261	Region &initializer = getInitializerRegion();
2262	if (!initializer.empty()) {
2263	p << ' ';
2264	p.printRegion(initializer, /*printEntryBlockArgs=*/false);
2265	}
2266	}
2267
2268	static LogicalResult verifyComdat(Operation *op,
2269	std::optional<SymbolRefAttr> attr) {
2270	if (!attr)
2271	return success();
2272
2273	auto *comdatSelector = SymbolTable::lookupNearestSymbolFrom(op, *attr);
2274	if (!isa_and_nonnull<ComdatSelectorOp>(comdatSelector))
2275	return op->emitError() << "expected comdat symbol";
2276
2277	return success();
2278	}
2279
2280	static LogicalResult verifyBlockTags(LLVMFuncOp funcOp) {
2281	llvm::DenseSet<BlockTagAttr> blockTags;
2282	// Note that presence of `BlockTagOp`s currently can't prevent an unrecheable
2283	// block to be removed by canonicalizer's region simplify pass, which needs to
2284	// be dialect aware to allow extra constraints to be described.
2285	WalkResult res = funcOp.walk([&](BlockTagOp blockTagOp) {
2286	if (blockTags.contains(blockTagOp.getTag())) {
2287	blockTagOp.emitError()
2288	<< "duplicate block tag '" << blockTagOp.getTag().getId()
2289	<< "' in the same function: ";
2290	return WalkResult::interrupt();
2291	}
2292	blockTags.insert(blockTagOp.getTag());
2293	return WalkResult::advance();
2294	});
2295
2296	return failure(IsFailure: res.wasInterrupted());
2297	}
2298
2299	/// Parse common attributes that might show up in the same order in both
2300	/// GlobalOp and AliasOp.
2301	template <typename OpType>
2302	static ParseResult parseCommonGlobalAndAlias(OpAsmParser &parser,
2303	OperationState &result) {
2304	MLIRContext *ctx = parser.getContext();
2305	// Parse optional linkage, default to External.
2306	result.addAttribute(OpType::getLinkageAttrName(result.name),
2307	LLVM::LinkageAttr::get(
2308	ctx, parseOptionalLLVMKeyword<Linkage>(
2309	parser, result, LLVM::Linkage::External)));
2310
2311	// Parse optional visibility, default to Default.
2312	result.addAttribute(OpType::getVisibility_AttrName(result.name),
2313	parser.getBuilder().getI64IntegerAttr(
2314	parseOptionalLLVMKeyword<LLVM::Visibility, int64_t>(
2315	parser, result, LLVM::Visibility::Default)));
2316
2317	if (succeeded(Result: parser.parseOptionalKeyword(keyword: "thread_local")))
2318	result.addAttribute(OpType::getThreadLocal_AttrName(result.name),
2319	parser.getBuilder().getUnitAttr());
2320
2321	// Parse optional UnnamedAddr, default to None.
2322	result.addAttribute(OpType::getUnnamedAddrAttrName(result.name),
2323	parser.getBuilder().getI64IntegerAttr(
2324	parseOptionalLLVMKeyword<UnnamedAddr, int64_t>(
2325	parser, result, LLVM::UnnamedAddr::None)));
2326
2327	return success();
2328	}
2329
2330	// operation ::= `llvm.mlir.global` linkage? visibility?
2331	// (`unnamed_addr` | `local_unnamed_addr`)?
2332	// `thread_local`? `constant`? `@` identifier
2333	// `(` attribute? `)` (`comdat(` symbol-ref-id `)`)?
2334	// attribute-list? (`:` type)? region?
2335	//
2336	// The type can be omitted for string attributes, in which case it will be
2337	// inferred from the value of the string as [strlen(value) x i8].
2338	ParseResult GlobalOp::parse(OpAsmParser &parser, OperationState &result) {
2339	// Call into common parsing between GlobalOp and AliasOp.
2340	if (parseCommonGlobalAndAlias<GlobalOp>(parser, result).failed())
2341	return failure();
2342
2343	if (succeeded(parser.parseOptionalKeyword("constant")))
2344	result.addAttribute(getConstantAttrName(result.name),
2345	parser.getBuilder().getUnitAttr());
2346
2347	StringAttr name;
2348	if (parser.parseSymbolName(name, getSymNameAttrName(result.name),
2349	result.attributes) ||
2350	parser.parseLParen())
2351	return failure();
2352
2353	Attribute value;
2354	if (parser.parseOptionalRParen()) {
2355	if (parser.parseAttribute(value, getValueAttrName(result.name),
2356	result.attributes) ||
2357	parser.parseRParen())
2358	return failure();
2359	}
2360
2361	if (succeeded(parser.parseOptionalKeyword("comdat"))) {
2362	SymbolRefAttr comdat;
2363	if (parser.parseLParen() || parser.parseAttribute(comdat) ||
2364	parser.parseRParen())
2365	return failure();
2366
2367	result.addAttribute(getComdatAttrName(result.name), comdat);
2368	}
2369
2370	SmallVector<Type, 1> types;
2371	if (parser.parseOptionalAttrDict(result.attributes) ||
2372	parser.parseOptionalColonTypeList(types))
2373	return failure();
2374
2375	if (types.size() > 1)
2376	return parser.emitError(parser.getNameLoc(), "expected zero or one type");
2377
2378	Region &initRegion = *result.addRegion();
2379	if (types.empty()) {
2380	if (auto strAttr = llvm::dyn_cast_or_null<StringAttr>(value)) {
2381	MLIRContext *context = parser.getContext();
2382	auto arrayType = LLVM::LLVMArrayType::get(IntegerType::get(context, 8),
2383	strAttr.getValue().size());
2384	types.push_back(arrayType);
2385	} else {
2386	return parser.emitError(parser.getNameLoc(),
2387	"type can only be omitted for string globals");
2388	}
2389	} else {
2390	OptionalParseResult parseResult =
2391	parser.parseOptionalRegion(initRegion, /*arguments=*/{},
2392	/*argTypes=*/{});
2393	if (parseResult.has_value() && failed(*parseResult))
2394	return failure();
2395	}
2396
2397	result.addAttribute(getGlobalTypeAttrName(result.name),
2398	TypeAttr::get(types[0]));
2399	return success();
2400	}
2401
2402	static bool isZeroAttribute(Attribute value) {
2403	if (auto intValue = llvm::dyn_cast<IntegerAttr>(value))
2404	return intValue.getValue().isZero();
2405	if (auto fpValue = llvm::dyn_cast<FloatAttr>(value))
2406	return fpValue.getValue().isZero();
2407	if (auto splatValue = llvm::dyn_cast<SplatElementsAttr>(Val&: value))
2408	return isZeroAttribute(value: splatValue.getSplatValue<Attribute>());
2409	if (auto elementsValue = llvm::dyn_cast<ElementsAttr>(value))
2410	return llvm::all_of(elementsValue.getValues<Attribute>(), isZeroAttribute);
2411	if (auto arrayValue = llvm::dyn_cast<ArrayAttr>(value))
2412	return llvm::all_of(arrayValue.getValue(), isZeroAttribute);
2413	return false;
2414	}
2415
2416	LogicalResult GlobalOp::verify() {
2417	bool validType = isCompatibleOuterType(getType())
2418	? !llvm::isa<LLVMVoidType, LLVMTokenType,
2419	LLVMMetadataType, LLVMLabelType>(getType())
2420	: llvm::isa<PointerElementTypeInterface>(getType());
2421	if (!validType)
2422	return emitOpError(
2423	"expects type to be a valid element type for an LLVM global");
2424	if ((*this)->getParentOp() && !satisfiesLLVMModule((*this)->getParentOp()))
2425	return emitOpError("must appear at the module level");
2426
2427	if (auto strAttr = llvm::dyn_cast_or_null<StringAttr>(getValueOrNull())) {
2428	auto type = llvm::dyn_cast<LLVMArrayType>(getType());
2429	IntegerType elementType =
2430	type ? llvm::dyn_cast<IntegerType>(type.getElementType()) : nullptr;
2431	if (!elementType || elementType.getWidth() != 8 ||
2432	type.getNumElements() != strAttr.getValue().size())
2433	return emitOpError(
2434	"requires an i8 array type of the length equal to that of the string "
2435	"attribute");
2436	}
2437
2438	if (auto targetExtType = dyn_cast<LLVMTargetExtType>(getType())) {
2439	if (!targetExtType.hasProperty(LLVMTargetExtType::CanBeGlobal))
2440	return emitOpError()
2441	<< "this target extension type cannot be used in a global";
2442
2443	if (Attribute value = getValueOrNull())
2444	return emitOpError() << "global with target extension type can only be "
2445	"initialized with zero-initializer";
2446	}
2447
2448	if (getLinkage() == Linkage::Common) {
2449	if (Attribute value = getValueOrNull()) {
2450	if (!isZeroAttribute(value)) {
2451	return emitOpError()
2452	<< "expected zero value for '"
2453	<< stringifyLinkage(Linkage::Common) << "' linkage";
2454	}
2455	}
2456	}
2457
2458	if (getLinkage() == Linkage::Appending) {
2459	if (!llvm::isa<LLVMArrayType>(getType())) {
2460	return emitOpError() << "expected array type for '"
2461	<< stringifyLinkage(Linkage::Appending)
2462	<< "' linkage";
2463	}
2464	}
2465
2466	if (failed(verifyComdat(*this, getComdat())))
2467	return failure();
2468
2469	std::optional<uint64_t> alignAttr = getAlignment();
2470	if (alignAttr.has_value()) {
2471	uint64_t value = alignAttr.value();
2472	if (!llvm::isPowerOf2_64(value))
2473	return emitError() << "alignment attribute is not a power of 2";
2474	}
2475
2476	return success();
2477	}
2478
2479	LogicalResult GlobalOp::verifyRegions() {
2480	if (Block *b = getInitializerBlock()) {
2481	ReturnOp ret = cast<ReturnOp>(b->getTerminator());
2482	if (ret.operand_type_begin() == ret.operand_type_end())
2483	return emitOpError("initializer region cannot return void");
2484	if (*ret.operand_type_begin() != getType())
2485	return emitOpError("initializer region type ")
2486	<< *ret.operand_type_begin() << " does not match global type "
2487	<< getType();
2488
2489	for (Operation &op : *b) {
2490	auto iface = dyn_cast<MemoryEffectOpInterface>(op);
2491	if (!iface || !iface.hasNoEffect())
2492	return op.emitError()
2493	<< "ops with side effects not allowed in global initializers";
2494	}
2495
2496	if (getValueOrNull())
2497	return emitOpError("cannot have both initializer value and region");
2498	}
2499
2500	return success();
2501	}
2502
2503	//===----------------------------------------------------------------------===//
2504	// LLVM::GlobalCtorsOp
2505	//===----------------------------------------------------------------------===//
2506
2507	LogicalResult checkGlobalXtorData(Operation *op, ArrayAttr data) {
2508	if (data.empty())
2509	return success();
2510
2511	if (llvm::all_of(data.getAsRange<Attribute>(), [](Attribute v) {
2512	return isa<FlatSymbolRefAttr, ZeroAttr>(v);
2513	}))
2514	return success();
2515	return op->emitError(message: "data element must be symbol or #llvm.zero");
2516	}
2517
2518	LogicalResult
2519	GlobalCtorsOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
2520	for (Attribute ctor : getCtors()) {
2521	if (failed(verifySymbolAttrUse(llvm::cast<FlatSymbolRefAttr>(ctor), *this,
2522	symbolTable)))
2523	return failure();
2524	}
2525	return success();
2526	}
2527
2528	LogicalResult GlobalCtorsOp::verify() {
2529	if (checkGlobalXtorData(*this, getData()).failed())
2530	return failure();
2531
2532	if (getCtors().size() == getPriorities().size() &&
2533	getCtors().size() == getData().size())
2534	return success();
2535	return emitError(
2536	"ctors, priorities, and data must have the same number of elements");
2537	}
2538
2539	//===----------------------------------------------------------------------===//
2540	// LLVM::GlobalDtorsOp
2541	//===----------------------------------------------------------------------===//
2542
2543	LogicalResult
2544	GlobalDtorsOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
2545	for (Attribute dtor : getDtors()) {
2546	if (failed(verifySymbolAttrUse(llvm::cast<FlatSymbolRefAttr>(dtor), *this,
2547	symbolTable)))
2548	return failure();
2549	}
2550	return success();
2551	}
2552
2553	LogicalResult GlobalDtorsOp::verify() {
2554	if (checkGlobalXtorData(*this, getData()).failed())
2555	return failure();
2556
2557	if (getDtors().size() == getPriorities().size() &&
2558	getDtors().size() == getData().size())
2559	return success();
2560	return emitError(
2561	"dtors, priorities, and data must have the same number of elements");
2562	}
2563
2564	//===----------------------------------------------------------------------===//
2565	// Builder, printer and verifier for LLVM::AliasOp.
2566	//===----------------------------------------------------------------------===//
2567
2568	void AliasOp::build(OpBuilder &builder, OperationState &result, Type type,
2569	Linkage linkage, StringRef name, bool dsoLocal,
2570	bool threadLocal, ArrayRef<NamedAttribute> attrs) {
2571	result.addAttribute(getSymNameAttrName(result.name),
2572	builder.getStringAttr(name));
2573	result.addAttribute(getAliasTypeAttrName(result.name), TypeAttr::get(type));
2574	if (dsoLocal)
2575	result.addAttribute(getDsoLocalAttrName(result.name),
2576	builder.getUnitAttr());
2577	if (threadLocal)
2578	result.addAttribute(getThreadLocal_AttrName(result.name),
2579	builder.getUnitAttr());
2580
2581	result.addAttribute(getLinkageAttrName(result.name),
2582	LinkageAttr::get(builder.getContext(), linkage));
2583	result.attributes.append(attrs.begin(), attrs.end());
2584
2585	result.addRegion();
2586	}
2587
2588	void AliasOp::print(OpAsmPrinter &p) {
2589	printCommonGlobalAndAlias<AliasOp>(p, *this);
2590
2591	p.printSymbolName(getSymName());
2592	p.printOptionalAttrDict((*this)->getAttrs(),
2593	{SymbolTable::getSymbolAttrName(),
2594	getAliasTypeAttrName(), getLinkageAttrName(),
2595	getUnnamedAddrAttrName(), getThreadLocal_AttrName(),
2596	getVisibility_AttrName(), getUnnamedAddrAttrName()});
2597
2598	// Print the trailing type.
2599	p << " : " << getType() << ' ';
2600	// Print the initializer region.
2601	p.printRegion(getInitializerRegion(), /*printEntryBlockArgs=*/false);
2602	}
2603
2604	// operation ::= `llvm.mlir.alias` linkage? visibility?
2605	// (`unnamed_addr` | `local_unnamed_addr`)?
2606	// `thread_local`? `@` identifier
2607	// `(` attribute? `)`
2608	// attribute-list? `:` type region
2609	//
2610	ParseResult AliasOp::parse(OpAsmParser &parser, OperationState &result) {
2611	// Call into common parsing between GlobalOp and AliasOp.
2612	if (parseCommonGlobalAndAlias<AliasOp>(parser, result).failed())
2613	return failure();
2614
2615	StringAttr name;
2616	if (parser.parseSymbolName(name, getSymNameAttrName(result.name),
2617	result.attributes))
2618	return failure();
2619
2620	SmallVector<Type, 1> types;
2621	if (parser.parseOptionalAttrDict(result.attributes) ||
2622	parser.parseOptionalColonTypeList(types))
2623	return failure();
2624
2625	if (types.size() > 1)
2626	return parser.emitError(parser.getNameLoc(), "expected zero or one type");
2627
2628	Region &initRegion = *result.addRegion();
2629	if (parser.parseRegion(initRegion).failed())
2630	return failure();
2631
2632	result.addAttribute(getAliasTypeAttrName(result.name),
2633	TypeAttr::get(types[0]));
2634	return success();
2635	}
2636
2637	LogicalResult AliasOp::verify() {
2638	bool validType = isCompatibleOuterType(getType())
2639	? !llvm::isa<LLVMVoidType, LLVMTokenType,
2640	LLVMMetadataType, LLVMLabelType>(getType())
2641	: llvm::isa<PointerElementTypeInterface>(getType());
2642	if (!validType)
2643	return emitOpError(
2644	"expects type to be a valid element type for an LLVM global alias");
2645
2646	// This matches LLVM IR verification logic, see llvm/lib/IR/Verifier.cpp
2647	switch (getLinkage()) {
2648	case Linkage::External:
2649	case Linkage::Internal:
2650	case Linkage::Private:
2651	case Linkage::Weak:
2652	case Linkage::WeakODR:
2653	case Linkage::Linkonce:
2654	case Linkage::LinkonceODR:
2655	case Linkage::AvailableExternally:
2656	break;
2657	default:
2658	return emitOpError()
2659	<< "'" << stringifyLinkage(getLinkage())
2660	<< "' linkage not supported in aliases, available options: private, "
2661	"internal, linkonce, weak, linkonce_odr, weak_odr, external or "
2662	"available_externally";
2663	}
2664
2665	return success();
2666	}
2667
2668	LogicalResult AliasOp::verifyRegions() {
2669	Block &b = getInitializerBlock();
2670	auto ret = cast<ReturnOp>(b.getTerminator());
2671	if (ret.getNumOperands() == 0 ||
2672	!isa<LLVM::LLVMPointerType>(ret.getOperand(0).getType()))
2673	return emitOpError("initializer region must always return a pointer");
2674
2675	for (Operation &op : b) {
2676	auto iface = dyn_cast<MemoryEffectOpInterface>(op);
2677	if (!iface || !iface.hasNoEffect())
2678	return op.emitError()
2679	<< "ops with side effects are not allowed in alias initializers";
2680	}
2681
2682	return success();
2683	}
2684
2685	unsigned AliasOp::getAddrSpace() {
2686	Block &initializer = getInitializerBlock();
2687	auto ret = cast<ReturnOp>(initializer.getTerminator());
2688	auto ptrTy = cast<LLVMPointerType>(ret.getOperand(0).getType());
2689	return ptrTy.getAddressSpace();
2690	}
2691
2692	//===----------------------------------------------------------------------===//
2693	// ShuffleVectorOp
2694	//===----------------------------------------------------------------------===//
2695
2696	void ShuffleVectorOp::build(OpBuilder &builder, OperationState &state, Value v1,
2697	Value v2, DenseI32ArrayAttr mask,
2698	ArrayRef<NamedAttribute> attrs) {
2699	auto containerType = v1.getType();
2700	auto vType = LLVM::getVectorType(
2701	cast<VectorType>(containerType).getElementType(), mask.size(),
2702	LLVM::isScalableVectorType(containerType));
2703	build(builder, state, vType, v1, v2, mask);
2704	state.addAttributes(attrs);
2705	}
2706
2707	void ShuffleVectorOp::build(OpBuilder &builder, OperationState &state, Value v1,
2708	Value v2, ArrayRef<int32_t> mask) {
2709	build(builder, state, v1, v2, builder.getDenseI32ArrayAttr(mask));
2710	}
2711
2712	/// Build the result type of a shuffle vector operation.
2713	static ParseResult parseShuffleType(AsmParser &parser, Type v1Type,
2714	Type &resType, DenseI32ArrayAttr mask) {
2715	if (!LLVM::isCompatibleVectorType(type: v1Type))
2716	return parser.emitError(loc: parser.getCurrentLocation(),
2717	message: "expected an LLVM compatible vector type");
2718	resType =
2719	LLVM::getVectorType(cast<VectorType>(v1Type).getElementType(),
2720	mask.size(), LLVM::isScalableVectorType(vectorType: v1Type));
2721	return success();
2722	}
2723
2724	/// Nothing to do when the result type is inferred.
2725	static void printShuffleType(AsmPrinter &printer, Operation *op, Type v1Type,
2726	Type resType, DenseI32ArrayAttr mask) {}
2727
2728	LogicalResult ShuffleVectorOp::verify() {
2729	if (LLVM::isScalableVectorType(getV1().getType()) &&
2730	llvm::any_of(getMask(), [](int32_t v) { return v != 0; }))
2731	return emitOpError("expected a splat operation for scalable vectors");
2732	return success();
2733	}
2734
2735	//===----------------------------------------------------------------------===//
2736	// Implementations for LLVM::LLVMFuncOp.
2737	//===----------------------------------------------------------------------===//
2738
2739	// Add the entry block to the function.
2740	Block *LLVMFuncOp::addEntryBlock(OpBuilder &builder) {
2741	assert(empty() && "function already has an entry block");
2742	OpBuilder::InsertionGuard g(builder);
2743	Block *entry = builder.createBlock(&getBody());
2744
2745	// FIXME: Allow passing in proper locations for the entry arguments.
2746	LLVMFunctionType type = getFunctionType();
2747	for (unsigned i = 0, e = type.getNumParams(); i < e; ++i)
2748	entry->addArgument(type.getParamType(i), getLoc());
2749	return entry;
2750	}
2751
2752	void LLVMFuncOp::build(OpBuilder &builder, OperationState &result,
2753	StringRef name, Type type, LLVM::Linkage linkage,
2754	bool dsoLocal, CConv cconv, SymbolRefAttr comdat,
2755	ArrayRef<NamedAttribute> attrs,
2756	ArrayRef<DictionaryAttr> argAttrs,
2757	std::optional<uint64_t> functionEntryCount) {
2758	result.addRegion();
2759	result.addAttribute(SymbolTable::getSymbolAttrName(),
2760	builder.getStringAttr(name));
2761	result.addAttribute(getFunctionTypeAttrName(result.name),
2762	TypeAttr::get(type));
2763	result.addAttribute(getLinkageAttrName(result.name),
2764	LinkageAttr::get(builder.getContext(), linkage));
2765	result.addAttribute(getCConvAttrName(result.name),
2766	CConvAttr::get(builder.getContext(), cconv));
2767	result.attributes.append(attrs.begin(), attrs.end());
2768	if (dsoLocal)
2769	result.addAttribute(getDsoLocalAttrName(result.name),
2770	builder.getUnitAttr());
2771	if (comdat)
2772	result.addAttribute(getComdatAttrName(result.name), comdat);
2773	if (functionEntryCount)
2774	result.addAttribute(getFunctionEntryCountAttrName(result.name),
2775	builder.getI64IntegerAttr(functionEntryCount.value()));
2776	if (argAttrs.empty())
2777	return;
2778
2779	assert(llvm::cast<LLVMFunctionType>(type).getNumParams() == argAttrs.size() &&
2780	"expected as many argument attribute lists as arguments");
2781	call_interface_impl::addArgAndResultAttrs(
2782	builder, result, argAttrs, /*resultAttrs=*/std::nullopt,
2783	getArgAttrsAttrName(result.name), getResAttrsAttrName(result.name));
2784	}
2785
2786	// Builds an LLVM function type from the given lists of input and output types.
2787	// Returns a null type if any of the types provided are non-LLVM types, or if
2788	// there is more than one output type.
2789	static Type
2790	buildLLVMFunctionType(OpAsmParser &parser, SMLoc loc, ArrayRef<Type> inputs,
2791	ArrayRef<Type> outputs,
2792	function_interface_impl::VariadicFlag variadicFlag) {
2793	Builder &b = parser.getBuilder();
2794	if (outputs.size() > 1) {
2795	parser.emitError(loc, message: "failed to construct function type: expected zero or "
2796	"one function result");
2797	return {};
2798	}
2799
2800	// Convert inputs to LLVM types, exit early on error.
2801	SmallVector<Type, 4> llvmInputs;
2802	for (auto t : inputs) {
2803	if (!isCompatibleType(type: t)) {
2804	parser.emitError(loc, message: "failed to construct function type: expected LLVM "
2805	"type for function arguments");
2806	return {};
2807	}
2808	llvmInputs.push_back(Elt: t);
2809	}
2810
2811	// No output is denoted as "void" in LLVM type system.
2812	Type llvmOutput =
2813	outputs.empty() ? LLVMVoidType::get(ctx: b.getContext()) : outputs.front();
2814	if (!isCompatibleType(type: llvmOutput)) {
2815	parser.emitError(loc, message: "failed to construct function type: expected LLVM "
2816	"type for function results")
2817	<< llvmOutput;
2818	return {};
2819	}
2820	return LLVMFunctionType::get(llvmOutput, llvmInputs,
2821	variadicFlag.isVariadic());
2822	}
2823
2824	// Parses an LLVM function.
2825	//
2826	// operation ::= `llvm.func` linkage? cconv? function-signature
2827	// (`comdat(` symbol-ref-id `)`)?
2828	// function-attributes?
2829	// function-body
2830	//
2831	ParseResult LLVMFuncOp::parse(OpAsmParser &parser, OperationState &result) {
2832	// Default to external linkage if no keyword is provided.
2833	result.addAttribute(
2834	getLinkageAttrName(result.name),
2835	LinkageAttr::get(parser.getContext(),
2836	parseOptionalLLVMKeyword<Linkage>(
2837	parser, result, LLVM::Linkage::External)));
2838
2839	// Parse optional visibility, default to Default.
2840	result.addAttribute(getVisibility_AttrName(result.name),
2841	parser.getBuilder().getI64IntegerAttr(
2842	parseOptionalLLVMKeyword<LLVM::Visibility, int64_t>(
2843	parser, result, LLVM::Visibility::Default)));
2844
2845	// Parse optional UnnamedAddr, default to None.
2846	result.addAttribute(getUnnamedAddrAttrName(result.name),
2847	parser.getBuilder().getI64IntegerAttr(
2848	parseOptionalLLVMKeyword<UnnamedAddr, int64_t>(
2849	parser, result, LLVM::UnnamedAddr::None)));
2850
2851	// Default to C Calling Convention if no keyword is provided.
2852	result.addAttribute(
2853	getCConvAttrName(result.name),
2854	CConvAttr::get(parser.getContext(), parseOptionalLLVMKeyword<CConv>(
2855	parser, result, LLVM::CConv::C)));
2856
2857	StringAttr nameAttr;
2858	SmallVector<OpAsmParser::Argument> entryArgs;
2859	SmallVector<DictionaryAttr> resultAttrs;
2860	SmallVector<Type> resultTypes;
2861	bool isVariadic;
2862
2863	auto signatureLocation = parser.getCurrentLocation();
2864	if (parser.parseSymbolName(nameAttr, SymbolTable::getSymbolAttrName(),
2865	result.attributes) ||
2866	function_interface_impl::parseFunctionSignatureWithArguments(
2867	parser, /*allowVariadic=*/true, entryArgs, isVariadic, resultTypes,
2868	resultAttrs))
2869	return failure();
2870
2871	SmallVector<Type> argTypes;
2872	for (auto &arg : entryArgs)
2873	argTypes.push_back(arg.type);
2874	auto type =
2875	buildLLVMFunctionType(parser, signatureLocation, argTypes, resultTypes,
2876	function_interface_impl::VariadicFlag(isVariadic));
2877	if (!type)
2878	return failure();
2879	result.addAttribute(getFunctionTypeAttrName(result.name),
2880	TypeAttr::get(type));
2881
2882	if (succeeded(parser.parseOptionalKeyword("vscale_range"))) {
2883	int64_t minRange, maxRange;
2884	if (parser.parseLParen() || parser.parseInteger(minRange) ||
2885	parser.parseComma() || parser.parseInteger(maxRange) ||
2886	parser.parseRParen())
2887	return failure();
2888	auto intTy = IntegerType::get(parser.getContext(), 32);
2889	result.addAttribute(
2890	getVscaleRangeAttrName(result.name),
2891	LLVM::VScaleRangeAttr::get(parser.getContext(),
2892	IntegerAttr::get(intTy, minRange),
2893	IntegerAttr::get(intTy, maxRange)));
2894	}
2895	// Parse the optional comdat selector.
2896	if (succeeded(parser.parseOptionalKeyword("comdat"))) {
2897	SymbolRefAttr comdat;
2898	if (parser.parseLParen() || parser.parseAttribute(comdat) ||
2899	parser.parseRParen())
2900	return failure();
2901
2902	result.addAttribute(getComdatAttrName(result.name), comdat);
2903	}
2904
2905	if (failed(parser.parseOptionalAttrDictWithKeyword(result.attributes)))
2906	return failure();
2907	call_interface_impl::addArgAndResultAttrs(
2908	parser.getBuilder(), result, entryArgs, resultAttrs,
2909	getArgAttrsAttrName(result.name), getResAttrsAttrName(result.name));
2910
2911	auto *body = result.addRegion();
2912	OptionalParseResult parseResult =
2913	parser.parseOptionalRegion(*body, entryArgs);
2914	return failure(parseResult.has_value() && failed(*parseResult));
2915	}
2916
2917	// Print the LLVMFuncOp. Collects argument and result types and passes them to
2918	// helper functions. Drops "void" result since it cannot be parsed back. Skips
2919	// the external linkage since it is the default value.
2920	void LLVMFuncOp::print(OpAsmPrinter &p) {
2921	p << ' ';
2922	if (getLinkage() != LLVM::Linkage::External)
2923	p << stringifyLinkage(getLinkage()) << ' ';
2924	StringRef visibility = stringifyVisibility(getVisibility_());
2925	if (!visibility.empty())
2926	p << visibility << ' ';
2927	if (auto unnamedAddr = getUnnamedAddr()) {
2928	StringRef str = stringifyUnnamedAddr(*unnamedAddr);
2929	if (!str.empty())
2930	p << str << ' ';
2931	}
2932	if (getCConv() != LLVM::CConv::C)
2933	p << stringifyCConv(getCConv()) << ' ';
2934
2935	p.printSymbolName(getName());
2936
2937	LLVMFunctionType fnType = getFunctionType();
2938	SmallVector<Type, 8> argTypes;
2939	SmallVector<Type, 1> resTypes;
2940	argTypes.reserve(fnType.getNumParams());
2941	for (unsigned i = 0, e = fnType.getNumParams(); i < e; ++i)
2942	argTypes.push_back(fnType.getParamType(i));
2943
2944	Type returnType = fnType.getReturnType();
2945	if (!llvm::isa<LLVMVoidType>(returnType))
2946	resTypes.push_back(returnType);
2947
2948	function_interface_impl::printFunctionSignature(p, *this, argTypes,
2949	isVarArg(), resTypes);
2950
2951	// Print vscale range if present
2952	if (std::optional<VScaleRangeAttr> vscale = getVscaleRange())
2953	p << " vscale_range(" << vscale->getMinRange().getInt() << ", "
2954	<< vscale->getMaxRange().getInt() << ')';
2955
2956	// Print the optional comdat selector.
2957	if (auto comdat = getComdat())
2958	p << " comdat(" << *comdat << ')';
2959
2960	function_interface_impl::printFunctionAttributes(
2961	p, *this,
2962	{getFunctionTypeAttrName(), getArgAttrsAttrName(), getResAttrsAttrName(),
2963	getLinkageAttrName(), getCConvAttrName(), getVisibility_AttrName(),
2964	getComdatAttrName(), getUnnamedAddrAttrName(),
2965	getVscaleRangeAttrName()});
2966
2967	// Print the body if this is not an external function.
2968	Region &body = getBody();
2969	if (!body.empty()) {
2970	p << ' ';
2971	p.printRegion(body, /*printEntryBlockArgs=*/false,
2972	/*printBlockTerminators=*/true);
2973	}
2974	}
2975
2976	// Verifies LLVM- and implementation-specific properties of the LLVM func Op:
2977	// - functions don't have 'common' linkage
2978	// - external functions have 'external' or 'extern_weak' linkage;
2979	// - vararg is (currently) only supported for external functions;
2980	LogicalResult LLVMFuncOp::verify() {
2981	if (getLinkage() == LLVM::Linkage::Common)
2982	return emitOpError() << "functions cannot have '"
2983	<< stringifyLinkage(LLVM::Linkage::Common)
2984	<< "' linkage";
2985
2986	if (failed(verifyComdat(*this, getComdat())))
2987	return failure();
2988
2989	if (isExternal()) {
2990	if (getLinkage() != LLVM::Linkage::External &&
2991	getLinkage() != LLVM::Linkage::ExternWeak)
2992	return emitOpError() << "external functions must have '"
2993	<< stringifyLinkage(LLVM::Linkage::External)
2994	<< "' or '"
2995	<< stringifyLinkage(LLVM::Linkage::ExternWeak)
2996	<< "' linkage";
2997	return success();
2998	}
2999
3000	// In LLVM IR, these attributes are composed by convention, not by design.
3001	if (isNoInline() && isAlwaysInline())
3002	return emitError("no_inline and always_inline attributes are incompatible");
3003
3004	if (isOptimizeNone() && !isNoInline())
3005	return emitOpError("with optimize_none must also be no_inline");
3006
3007	Type landingpadResultTy;
3008	StringRef diagnosticMessage;
3009	bool isLandingpadTypeConsistent =
3010	!walk([&](Operation *op) {
3011	const auto checkType = [&](Type type, StringRef errorMessage) {
3012	if (!landingpadResultTy) {
3013	landingpadResultTy = type;
3014	return WalkResult::advance();
3015	}
3016	if (landingpadResultTy != type) {
3017	diagnosticMessage = errorMessage;
3018	return WalkResult::interrupt();
3019	}
3020	return WalkResult::advance();
3021	};
3022	return TypeSwitch<Operation *, WalkResult>(op)
3023	.Case<LandingpadOp>([&](auto landingpad) {
3024	constexpr StringLiteral errorMessage =
3025	"'llvm.landingpad' should have a consistent result type "
3026	"inside a function";
3027	return checkType(landingpad.getType(), errorMessage);
3028	})
3029	.Case<ResumeOp>([&](auto resume) {
3030	constexpr StringLiteral errorMessage =
3031	"'llvm.resume' should have a consistent input type inside a "
3032	"function";
3033	return checkType(resume.getValue().getType(), errorMessage);
3034	})
3035	.Default([](auto) { return WalkResult::skip(); });
3036	}).wasInterrupted();
3037	if (!isLandingpadTypeConsistent) {
3038	assert(!diagnosticMessage.empty() &&
3039	"Expecting a non-empty diagnostic message");
3040	return emitError(diagnosticMessage);
3041	}
3042
3043	if (failed(verifyBlockTags(*this)))
3044	return failure();
3045
3046	return success();
3047	}
3048
3049	/// Verifies LLVM- and implementation-specific properties of the LLVM func Op:
3050	/// - entry block arguments are of LLVM types.
3051	LogicalResult LLVMFuncOp::verifyRegions() {
3052	if (isExternal())
3053	return success();
3054
3055	unsigned numArguments = getFunctionType().getNumParams();
3056	Block &entryBlock = front();
3057	for (unsigned i = 0; i < numArguments; ++i) {
3058	Type argType = entryBlock.getArgument(i).getType();
3059	if (!isCompatibleType(argType))
3060	return emitOpError("entry block argument #")
3061	<< i << " is not of LLVM type";
3062	}
3063
3064	return success();
3065	}
3066
3067	Region *LLVMFuncOp::getCallableRegion() {
3068	if (isExternal())
3069	return nullptr;
3070	return &getBody();
3071	}
3072
3073	//===----------------------------------------------------------------------===//
3074	// UndefOp.
3075	//===----------------------------------------------------------------------===//
3076
3077	/// Fold an undef operation to a dedicated undef attribute.
3078	OpFoldResult LLVM::UndefOp::fold(FoldAdaptor) {
3079	return LLVM::UndefAttr::get(getContext());
3080	}
3081
3082	//===----------------------------------------------------------------------===//
3083	// PoisonOp.
3084	//===----------------------------------------------------------------------===//
3085
3086	/// Fold a poison operation to a dedicated poison attribute.
3087	OpFoldResult LLVM::PoisonOp::fold(FoldAdaptor) {
3088	return LLVM::PoisonAttr::get(getContext());
3089	}
3090
3091	//===----------------------------------------------------------------------===//
3092	// ZeroOp.
3093	//===----------------------------------------------------------------------===//
3094
3095	LogicalResult LLVM::ZeroOp::verify() {
3096	if (auto targetExtType = dyn_cast<LLVMTargetExtType>(getType()))
3097	if (!targetExtType.hasProperty(LLVM::LLVMTargetExtType::HasZeroInit))
3098	return emitOpError()
3099	<< "target extension type does not support zero-initializer";
3100
3101	return success();
3102	}
3103
3104	/// Fold a zero operation to a builtin zero attribute when possible and fall
3105	/// back to a dedicated zero attribute.
3106	OpFoldResult LLVM::ZeroOp::fold(FoldAdaptor) {
3107	OpFoldResult result = Builder(getContext()).getZeroAttr(getType());
3108	if (result)
3109	return result;
3110	return LLVM::ZeroAttr::get(getContext());
3111	}
3112
3113	//===----------------------------------------------------------------------===//
3114	// ConstantOp.
3115	//===----------------------------------------------------------------------===//
3116
3117	/// Compute the total number of elements in the given type, also taking into
3118	/// account nested types. Supported types are `VectorType` and `LLVMArrayType`.
3119	/// Everything else is treated as a scalar.
3120	static int64_t getNumElements(Type t) {
3121	if (auto vecType = dyn_cast<VectorType>(t)) {
3122	assert(!vecType.isScalable() &&
3123	"number of elements of a scalable vector type is unknown");
3124	return vecType.getNumElements() * getNumElements(vecType.getElementType());
3125	}
3126	if (auto arrayType = dyn_cast<LLVM::LLVMArrayType>(t))
3127	return arrayType.getNumElements() *
3128	getNumElements(arrayType.getElementType());
3129	return 1;
3130	}
3131
3132	/// Check if the given type is a scalable vector type or a vector/array type
3133	/// that contains a nested scalable vector type.
3134	static bool hasScalableVectorType(Type t) {
3135	if (auto vecType = dyn_cast<VectorType>(t)) {
3136	if (vecType.isScalable())
3137	return true;
3138	return hasScalableVectorType(vecType.getElementType());
3139	}
3140	if (auto arrayType = dyn_cast<LLVM::LLVMArrayType>(t))
3141	return hasScalableVectorType(arrayType.getElementType());
3142	return false;
3143	}
3144
3145	/// Verifies the constant array represented by `arrayAttr` matches the provided
3146	/// `arrayType`.
3147	static LogicalResult verifyStructArrayConstant(LLVM::ConstantOp op,
3148	LLVM::LLVMArrayType arrayType,
3149	ArrayAttr arrayAttr, int dim) {
3150	if (arrayType.getNumElements() != arrayAttr.size())
3151	return op.emitOpError()
3152	<< "array attribute size does not match array type size in "
3153	"dimension "
3154	<< dim << ": " << arrayAttr.size() << " vs. "
3155	<< arrayType.getNumElements();
3156
3157	llvm::DenseSet<Attribute> elementsVerified;
3158
3159	// Recursively verify sub-dimensions for multidimensional arrays.
3160	if (auto subArrayType =
3161	dyn_cast<LLVM::LLVMArrayType>(arrayType.getElementType())) {
3162	for (auto [idx, elementAttr] : llvm::enumerate(arrayAttr))
3163	if (elementsVerified.insert(elementAttr).second) {
3164	if (isa<LLVM::ZeroAttr, LLVM::UndefAttr>(elementAttr))
3165	continue;
3166	auto subArrayAttr = dyn_cast<ArrayAttr>(elementAttr);
3167	if (!subArrayAttr)
3168	return op.emitOpError()
3169	<< "nested attribute for sub-array in dimension " << dim
3170	<< " at index " << idx
3171	<< " must be a zero, or undef, or array attribute";
3172	if (failed(verifyStructArrayConstant(op, subArrayType, subArrayAttr,
3173	dim + 1)))
3174	return failure();
3175	}
3176	return success();
3177	}
3178
3179	// Forbid usages of ArrayAttr for simple array types that should use
3180	// DenseElementsAttr instead. Note that there would be a use case for such
3181	// array types when one element value is obtained via a ptr-to-int conversion
3182	// from a symbol and cannot be represented in a DenseElementsAttr, but no MLIR
3183	// user needs this so far, and it seems better to avoid people misusing the
3184	// ArrayAttr for simple types.
3185	auto structType = dyn_cast<LLVM::LLVMStructType>(arrayType.getElementType());
3186	if (!structType)
3187	return op.emitOpError() << "for array with an array attribute must have a "
3188	"struct element type";
3189
3190	// Shallow verification that leaf attributes are appropriate as struct initial
3191	// value.
3192	size_t numStructElements = structType.getBody().size();
3193	for (auto [idx, elementAttr] : llvm::enumerate(arrayAttr)) {
3194	if (elementsVerified.insert(elementAttr).second) {
3195	if (isa<LLVM::ZeroAttr, LLVM::UndefAttr>(elementAttr))
3196	continue;
3197	auto subArrayAttr = dyn_cast<ArrayAttr>(elementAttr);
3198	if (!subArrayAttr)
3199	return op.emitOpError()
3200	<< "nested attribute for struct element at index " << idx
3201	<< " must be a zero, or undef, or array attribute";
3202	if (subArrayAttr.size() != numStructElements)
3203	return op.emitOpError()
3204	<< "nested array attribute size for struct element at index "
3205	<< idx << " must match struct size: " << subArrayAttr.size()
3206	<< " vs. " << numStructElements;
3207	}
3208	}
3209
3210	return success();
3211	}
3212
3213	LogicalResult LLVM::ConstantOp::verify() {
3214	if (StringAttr sAttr = llvm::dyn_cast<StringAttr>(getValue())) {
3215	auto arrayType = llvm::dyn_cast<LLVMArrayType>(getType());
3216	if (!arrayType || arrayType.getNumElements() != sAttr.getValue().size() ||
3217	!arrayType.getElementType().isInteger(8)) {
3218	return emitOpError() << "expected array type of "
3219	<< sAttr.getValue().size()
3220	<< " i8 elements for the string constant";
3221	}
3222	return success();
3223	}
3224	if (auto structType = dyn_cast<LLVMStructType>(getType())) {
3225	auto arrayAttr = dyn_cast<ArrayAttr>(getValue());
3226	if (!arrayAttr) {
3227	return emitOpError() << "expected array attribute for a struct constant";
3228	}
3229
3230	ArrayRef<Type> elementTypes = structType.getBody();
3231	if (arrayAttr.size() != elementTypes.size()) {
3232	return emitOpError() << "expected array attribute of size "
3233	<< elementTypes.size();
3234	}
3235	for (auto elementTy : elementTypes) {
3236	if (!isa<IntegerType, FloatType, LLVMPPCFP128Type>(elementTy)) {
3237	return emitOpError() << "expected struct element types to be floating "
3238	"point type or integer type";
3239	}
3240	}
3241
3242	for (size_t i = 0; i < elementTypes.size(); ++i) {
3243	Attribute element = arrayAttr[i];
3244	if (!isa<IntegerAttr, FloatAttr>(element)) {
3245	return emitOpError()
3246	<< "expected struct element attribute types to be floating "
3247	"point type or integer type";
3248	}
3249	auto elementType = cast<TypedAttr>(element).getType();
3250	if (elementType != elementTypes[i]) {
3251	return emitOpError()
3252	<< "struct element at index " << i << " is of wrong type";
3253	}
3254	}
3255
3256	return success();
3257	}
3258	if (auto targetExtType = dyn_cast<LLVMTargetExtType>(getType())) {
3259	return emitOpError() << "does not support target extension type.";
3260	}
3261
3262	// Verification of IntegerAttr, FloatAttr, ElementsAttr, ArrayAttr.
3263	if (auto intAttr = dyn_cast<IntegerAttr>(getValue())) {
3264	if (!llvm::isa<IntegerType>(getType()))
3265	return emitOpError() << "expected integer type";
3266	} else if (auto floatAttr = dyn_cast<FloatAttr>(getValue())) {
3267	const llvm::fltSemantics &sem = floatAttr.getValue().getSemantics();
3268	unsigned floatWidth = APFloat::getSizeInBits(sem);
3269	if (auto floatTy = dyn_cast<FloatType>(getType())) {
3270	if (floatTy.getWidth() != floatWidth) {
3271	return emitOpError() << "expected float type of width " << floatWidth;
3272	}
3273	}
3274	// See the comment for getLLVMConstant for more details about why 8-bit
3275	// floats can be represented by integers.
3276	if (isa<IntegerType>(getType()) && !getType().isInteger(floatWidth)) {
3277	return emitOpError() << "expected integer type of width " << floatWidth;
3278	}
3279	} else if (isa<ElementsAttr>(getValue())) {
3280	if (hasScalableVectorType(getType())) {
3281	// The exact number of elements of a scalable vector is unknown, so we
3282	// allow only splat attributes.
3283	auto splatElementsAttr = dyn_cast<SplatElementsAttr>(getValue());
3284	if (!splatElementsAttr)
3285	return emitOpError()
3286	<< "scalable vector type requires a splat attribute";
3287	return success();
3288	}
3289	if (!isa<VectorType, LLVM::LLVMArrayType>(getType()))
3290	return emitOpError() << "expected vector or array type";
3291	// The number of elements of the attribute and the type must match.
3292	if (auto elementsAttr = dyn_cast<ElementsAttr>(getValue())) {
3293	int64_t attrNumElements = elementsAttr.getNumElements();
3294	if (getNumElements(getType()) != attrNumElements)
3295	return emitOpError()
3296	<< "type and attribute have a different number of elements: "
3297	<< getNumElements(getType()) << " vs. " << attrNumElements;
3298	}
3299	} else if (auto arrayAttr = dyn_cast<ArrayAttr>(getValue())) {
3300	auto arrayType = dyn_cast<LLVM::LLVMArrayType>(getType());
3301	if (!arrayType)
3302	return emitOpError() << "expected array type";
3303	// When the attribute is an ArrayAttr, check that its nesting matches the
3304	// corresponding ArrayType or VectorType nesting.
3305	return verifyStructArrayConstant(*this, arrayType, arrayAttr, /*dim=*/0);
3306	} else {
3307	return emitOpError()
3308	<< "only supports integer, float, string or elements attributes";
3309	}
3310
3311	return success();
3312	}
3313
3314	bool LLVM::ConstantOp::isBuildableWith(Attribute value, Type type) {
3315	// The value's type must be the same as the provided type.
3316	auto typedAttr = dyn_cast<TypedAttr>(value);
3317	if (!typedAttr || typedAttr.getType() != type || !isCompatibleType(type))
3318	return false;
3319	// The value's type must be an LLVM compatible type.
3320	if (!isCompatibleType(type))
3321	return false;
3322	// TODO: Add support for additional attributes kinds once needed.
3323	return isa<IntegerAttr, FloatAttr, ElementsAttr>(value);
3324	}
3325
3326	ConstantOp LLVM::ConstantOp::materialize(OpBuilder &builder, Attribute value,
3327	Type type, Location loc) {
3328	if (isBuildableWith(value, type))
3329	return builder.create<LLVM::ConstantOp>(loc, cast<TypedAttr>(value));
3330	return nullptr;
3331	}
3332
3333	// Constant op constant-folds to its value.
3334	OpFoldResult LLVM::ConstantOp::fold(FoldAdaptor) { return getValue(); }
3335
3336	//===----------------------------------------------------------------------===//
3337	// AtomicRMWOp
3338	//===----------------------------------------------------------------------===//
3339
3340	void AtomicRMWOp::build(OpBuilder &builder, OperationState &state,
3341	AtomicBinOp binOp, Value ptr, Value val,
3342	AtomicOrdering ordering, StringRef syncscope,
3343	unsigned alignment, bool isVolatile) {
3344	build(builder, state, val.getType(), binOp, ptr, val, ordering,
3345	!syncscope.empty() ? builder.getStringAttr(syncscope) : nullptr,
3346	alignment ? builder.getI64IntegerAttr(alignment) : nullptr, isVolatile,
3347	/*access_groups=*/nullptr,
3348	/*alias_scopes=*/nullptr, /*noalias_scopes=*/nullptr, /*tbaa=*/nullptr);
3349	}
3350
3351	LogicalResult AtomicRMWOp::verify() {
3352	auto valType = getVal().getType();
3353	if (getBinOp() == AtomicBinOp::fadd || getBinOp() == AtomicBinOp::fsub ||
3354	getBinOp() == AtomicBinOp::fmin || getBinOp() == AtomicBinOp::fmax ||
3355	getBinOp() == AtomicBinOp::fminimum ||
3356	getBinOp() == AtomicBinOp::fmaximum) {
3357	if (isCompatibleVectorType(valType)) {
3358	if (isScalableVectorType(valType))
3359	return emitOpError("expected LLVM IR fixed vector type");
3360	Type elemType = llvm::cast<VectorType>(valType).getElementType();
3361	if (!isCompatibleFloatingPointType(elemType))
3362	return emitOpError(
3363	"expected LLVM IR floating point type for vector element");
3364	} else if (!isCompatibleFloatingPointType(valType)) {
3365	return emitOpError("expected LLVM IR floating point type");
3366	}
3367	} else if (getBinOp() == AtomicBinOp::xchg) {
3368	DataLayout dataLayout = DataLayout::closest(*this);
3369	if (!isTypeCompatibleWithAtomicOp(valType, dataLayout))
3370	return emitOpError("unexpected LLVM IR type for 'xchg' bin_op");
3371	} else {
3372	auto intType = llvm::dyn_cast<IntegerType>(valType);
3373	unsigned intBitWidth = intType ? intType.getWidth() : 0;
3374	if (intBitWidth != 8 && intBitWidth != 16 && intBitWidth != 32 &&
3375	intBitWidth != 64)
3376	return emitOpError("expected LLVM IR integer type");
3377	}
3378
3379	if (static_cast<unsigned>(getOrdering()) <
3380	static_cast<unsigned>(AtomicOrdering::monotonic))
3381	return emitOpError() << "expected at least '"
3382	<< stringifyAtomicOrdering(AtomicOrdering::monotonic)
3383	<< "' ordering";
3384
3385	return success();
3386	}
3387
3388	//===----------------------------------------------------------------------===//
3389	// AtomicCmpXchgOp
3390	//===----------------------------------------------------------------------===//
3391
3392	/// Returns an LLVM struct type that contains a value type and a boolean type.
3393	static LLVMStructType getValAndBoolStructType(Type valType) {
3394	auto boolType = IntegerType::get(valType.getContext(), 1);
3395	return LLVMStructType::getLiteral(valType.getContext(), {valType, boolType});
3396	}
3397
3398	void AtomicCmpXchgOp::build(OpBuilder &builder, OperationState &state,
3399	Value ptr, Value cmp, Value val,
3400	AtomicOrdering successOrdering,
3401	AtomicOrdering failureOrdering, StringRef syncscope,
3402	unsigned alignment, bool isWeak, bool isVolatile) {
3403	build(builder, state, getValAndBoolStructType(val.getType()), ptr, cmp, val,
3404	successOrdering, failureOrdering,
3405	!syncscope.empty() ? builder.getStringAttr(syncscope) : nullptr,
3406	alignment ? builder.getI64IntegerAttr(alignment) : nullptr, isWeak,
3407	isVolatile, /*access_groups=*/nullptr,
3408	/*alias_scopes=*/nullptr, /*noalias_scopes=*/nullptr, /*tbaa=*/nullptr);
3409	}
3410
3411	LogicalResult AtomicCmpXchgOp::verify() {
3412	auto ptrType = llvm::cast<LLVM::LLVMPointerType>(getPtr().getType());
3413	if (!ptrType)
3414	return emitOpError("expected LLVM IR pointer type for operand #0");
3415	auto valType = getVal().getType();
3416	DataLayout dataLayout = DataLayout::closest(*this);
3417	if (!isTypeCompatibleWithAtomicOp(valType, dataLayout))
3418	return emitOpError("unexpected LLVM IR type");
3419	if (getSuccessOrdering() < AtomicOrdering::monotonic ||
3420	getFailureOrdering() < AtomicOrdering::monotonic)
3421	return emitOpError("ordering must be at least 'monotonic'");
3422	if (getFailureOrdering() == AtomicOrdering::release ||
3423	getFailureOrdering() == AtomicOrdering::acq_rel)
3424	return emitOpError("failure ordering cannot be 'release' or 'acq_rel'");
3425	return success();
3426	}
3427
3428	//===----------------------------------------------------------------------===//
3429	// FenceOp
3430	//===----------------------------------------------------------------------===//
3431
3432	void FenceOp::build(OpBuilder &builder, OperationState &state,
3433	AtomicOrdering ordering, StringRef syncscope) {
3434	build(builder, state, ordering,
3435	syncscope.empty() ? nullptr : builder.getStringAttr(syncscope));
3436	}
3437
3438	LogicalResult FenceOp::verify() {
3439	if (getOrdering() == AtomicOrdering::not_atomic ||
3440	getOrdering() == AtomicOrdering::unordered ||
3441	getOrdering() == AtomicOrdering::monotonic)
3442	return emitOpError("can be given only acquire, release, acq_rel, "
3443	"and seq_cst orderings");
3444	return success();
3445	}
3446
3447	//===----------------------------------------------------------------------===//
3448	// Verifier for extension ops
3449	//===----------------------------------------------------------------------===//
3450
3451	/// Verifies that the given extension operation operates on consistent scalars
3452	/// or vectors, and that the target width is larger than the input width.
3453	template <class ExtOp>
3454	static LogicalResult verifyExtOp(ExtOp op) {
3455	IntegerType inputType, outputType;
3456	if (isCompatibleVectorType(op.getArg().getType())) {
3457	if (!isCompatibleVectorType(op.getResult().getType()))
3458	return op.emitError(
3459	"input type is a vector but output type is an integer");
3460	if (getVectorNumElements(op.getArg().getType()) !=
3461	getVectorNumElements(op.getResult().getType()))
3462	return op.emitError("input and output vectors are of incompatible shape");
3463	// Because this is a CastOp, the element of vectors is guaranteed to be an
3464	// integer.
3465	inputType = cast<IntegerType>(
3466	cast<VectorType>(op.getArg().getType()).getElementType());
3467	outputType = cast<IntegerType>(
3468	cast<VectorType>(op.getResult().getType()).getElementType());
3469	} else {
3470	// Because this is a CastOp and arg is not a vector, arg is guaranteed to be
3471	// an integer.
3472	inputType = cast<IntegerType>(op.getArg().getType());
3473	outputType = dyn_cast<IntegerType>(op.getResult().getType());
3474	if (!outputType)
3475	return op.emitError(
3476	"input type is an integer but output type is a vector");
3477	}
3478
3479	if (outputType.getWidth() <= inputType.getWidth())
3480	return op.emitError("integer width of the output type is smaller or "
3481	"equal to the integer width of the input type");
3482	return success();
3483	}
3484
3485	//===----------------------------------------------------------------------===//
3486	// ZExtOp
3487	//===----------------------------------------------------------------------===//
3488
3489	LogicalResult ZExtOp::verify() { return verifyExtOp<ZExtOp>(*this); }
3490
3491	OpFoldResult LLVM::ZExtOp::fold(FoldAdaptor adaptor) {
3492	auto arg = dyn_cast_or_null<IntegerAttr>(adaptor.getArg());
3493	if (!arg)
3494	return {};
3495
3496	size_t targetSize = cast<IntegerType>(getType()).getWidth();
3497	return IntegerAttr::get(getType(), arg.getValue().zext(targetSize));
3498	}
3499
3500	//===----------------------------------------------------------------------===//
3501	// SExtOp
3502	//===----------------------------------------------------------------------===//
3503
3504	LogicalResult SExtOp::verify() { return verifyExtOp<SExtOp>(*this); }
3505
3506	//===----------------------------------------------------------------------===//
3507	// Folder and verifier for LLVM::BitcastOp
3508	//===----------------------------------------------------------------------===//
3509
3510	/// Folds a cast op that can be chained.
3511	template <typename T>
3512	static OpFoldResult foldChainableCast(T castOp,
3513	typename T::FoldAdaptor adaptor) {
3514	// cast(x : T0, T0) -> x
3515	if (castOp.getArg().getType() == castOp.getType())
3516	return castOp.getArg();
3517	if (auto prev = castOp.getArg().template getDefiningOp<T>()) {
3518	// cast(cast(x : T0, T1), T0) -> x
3519	if (prev.getArg().getType() == castOp.getType())
3520	return prev.getArg();
3521	// cast(cast(x : T0, T1), T2) -> cast(x: T0, T2)
3522	castOp.getArgMutable().set(prev.getArg());
3523	return Value{castOp};
3524	}
3525	return {};
3526	}
3527
3528	OpFoldResult LLVM::BitcastOp::fold(FoldAdaptor adaptor) {
3529	return foldChainableCast(*this, adaptor);
3530	}
3531
3532	LogicalResult LLVM::BitcastOp::verify() {
3533	auto resultType = llvm::dyn_cast<LLVMPointerType>(
3534	extractVectorElementType(getResult().getType()));
3535	auto sourceType = llvm::dyn_cast<LLVMPointerType>(
3536	extractVectorElementType(getArg().getType()));
3537
3538	// If one of the types is a pointer (or vector of pointers), then
3539	// both source and result type have to be pointers.
3540	if (static_cast<bool>(resultType) != static_cast<bool>(sourceType))
3541	return emitOpError("can only cast pointers from and to pointers");
3542
3543	if (!resultType)
3544	return success();
3545
3546	auto isVector = llvm::IsaPred<VectorType>;
3547
3548	// Due to bitcast requiring both operands to be of the same size, it is not
3549	// possible for only one of the two to be a pointer of vectors.
3550	if (isVector(getResult().getType()) && !isVector(getArg().getType()))
3551	return emitOpError("cannot cast pointer to vector of pointers");
3552
3553	if (!isVector(getResult().getType()) && isVector(getArg().getType()))
3554	return emitOpError("cannot cast vector of pointers to pointer");
3555
3556	// Bitcast cannot cast between pointers of different address spaces.
3557	// 'llvm.addrspacecast' must be used for this purpose instead.
3558	if (resultType.getAddressSpace() != sourceType.getAddressSpace())
3559	return emitOpError("cannot cast pointers of different address spaces, "
3560	"use 'llvm.addrspacecast' instead");
3561
3562	return success();
3563	}
3564
3565	//===----------------------------------------------------------------------===//
3566	// Folder for LLVM::AddrSpaceCastOp
3567	//===----------------------------------------------------------------------===//
3568
3569	OpFoldResult LLVM::AddrSpaceCastOp::fold(FoldAdaptor adaptor) {
3570	return foldChainableCast(*this, adaptor);
3571	}
3572
3573	Value LLVM::AddrSpaceCastOp::getViewSource() { return getArg(); }
3574
3575	//===----------------------------------------------------------------------===//
3576	// Folder for LLVM::GEPOp
3577	//===----------------------------------------------------------------------===//
3578
3579	OpFoldResult LLVM::GEPOp::fold(FoldAdaptor adaptor) {
3580	GEPIndicesAdaptor<ArrayRef<Attribute>> indices(getRawConstantIndicesAttr(),
3581	adaptor.getDynamicIndices());
3582
3583	// gep %x:T, 0 -> %x
3584	if (getBase().getType() == getType() && indices.size() == 1)
3585	if (auto integer = llvm::dyn_cast_or_null<IntegerAttr>(indices[0]))
3586	if (integer.getValue().isZero())
3587	return getBase();
3588
3589	// Canonicalize any dynamic indices of constant value to constant indices.
3590	bool changed = false;
3591	SmallVector<GEPArg> gepArgs;
3592	for (auto iter : llvm::enumerate(indices)) {
3593	auto integer = llvm::dyn_cast_or_null<IntegerAttr>(iter.value());
3594	// Constant indices can only be int32_t, so if integer does not fit we
3595	// are forced to keep it dynamic, despite being a constant.
3596	if (!indices.isDynamicIndex(iter.index()) || !integer ||
3597	!integer.getValue().isSignedIntN(kGEPConstantBitWidth)) {
3598
3599	PointerUnion<IntegerAttr, Value> existing = getIndices()[iter.index()];
3600	if (Value val = llvm::dyn_cast_if_present<Value>(existing))
3601	gepArgs.emplace_back(val);
3602	else
3603	gepArgs.emplace_back(cast<IntegerAttr>(existing).getInt());
3604
3605	continue;
3606	}
3607
3608	changed = true;
3609	gepArgs.emplace_back(integer.getInt());
3610	}
3611	if (changed) {
3612	SmallVector<int32_t> rawConstantIndices;
3613	SmallVector<Value> dynamicIndices;
3614	destructureIndices(getElemType(), gepArgs, rawConstantIndices,
3615	dynamicIndices);
3616
3617	getDynamicIndicesMutable().assign(dynamicIndices);
3618	setRawConstantIndices(rawConstantIndices);
3619	return Value{*this};
3620	}
3621
3622	return {};
3623	}
3624
3625	Value LLVM::GEPOp::getViewSource() { return getBase(); }
3626
3627	//===----------------------------------------------------------------------===//
3628	// ShlOp
3629	//===----------------------------------------------------------------------===//
3630
3631	OpFoldResult LLVM::ShlOp::fold(FoldAdaptor adaptor) {
3632	auto rhs = dyn_cast_or_null<IntegerAttr>(adaptor.getRhs());
3633	if (!rhs)
3634	return {};
3635
3636	if (rhs.getValue().getZExtValue() >=
3637	getLhs().getType().getIntOrFloatBitWidth())
3638	return {}; // TODO: Fold into poison.
3639
3640	auto lhs = dyn_cast_or_null<IntegerAttr>(adaptor.getLhs());
3641	if (!lhs)
3642	return {};
3643
3644	return IntegerAttr::get(getType(), lhs.getValue().shl(rhs.getValue()));
3645	}
3646
3647	//===----------------------------------------------------------------------===//
3648	// OrOp
3649	//===----------------------------------------------------------------------===//
3650
3651	OpFoldResult LLVM::OrOp::fold(FoldAdaptor adaptor) {
3652	auto lhs = dyn_cast_or_null<IntegerAttr>(adaptor.getLhs());
3653	if (!lhs)
3654	return {};
3655
3656	auto rhs = dyn_cast_or_null<IntegerAttr>(adaptor.getRhs());
3657	if (!rhs)
3658	return {};
3659
3660	return IntegerAttr::get(getType(), lhs.getValue() | rhs.getValue());
3661	}
3662
3663	//===----------------------------------------------------------------------===//
3664	// CallIntrinsicOp
3665	//===----------------------------------------------------------------------===//
3666
3667	LogicalResult CallIntrinsicOp::verify() {
3668	if (!getIntrin().starts_with("llvm."))
3669	return emitOpError() << "intrinsic name must start with 'llvm.'";
3670	if (failed(verifyOperandBundles(*this)))
3671	return failure();
3672	return success();
3673	}
3674
3675	void CallIntrinsicOp::build(OpBuilder &builder, OperationState &state,
3676	mlir::StringAttr intrin, mlir::ValueRange args) {
3677	build(builder, state, /*resultTypes=*/TypeRange{}, intrin, args,
3678	FastmathFlagsAttr{},
3679	/*op_bundle_operands=*/{}, /*op_bundle_tags=*/{}, /*arg_attrs=*/{},
3680	/*res_attrs=*/{});
3681	}
3682
3683	void CallIntrinsicOp::build(OpBuilder &builder, OperationState &state,
3684	mlir::StringAttr intrin, mlir::ValueRange args,
3685	mlir::LLVM::FastmathFlagsAttr fastMathFlags) {
3686	build(builder, state, /*resultTypes=*/TypeRange{}, intrin, args,
3687	fastMathFlags,
3688	/*op_bundle_operands=*/{}, /*op_bundle_tags=*/{}, /*arg_attrs=*/{},
3689	/*res_attrs=*/{});
3690	}
3691
3692	void CallIntrinsicOp::build(OpBuilder &builder, OperationState &state,
3693	mlir::Type resultType, mlir::StringAttr intrin,
3694	mlir::ValueRange args) {
3695	build(builder, state, {resultType}, intrin, args, FastmathFlagsAttr{},
3696	/*op_bundle_operands=*/{}, /*op_bundle_tags=*/{}, /*arg_attrs=*/{},
3697	/*res_attrs=*/{});
3698	}
3699
3700	void CallIntrinsicOp::build(OpBuilder &builder, OperationState &state,
3701	mlir::TypeRange resultTypes,
3702	mlir::StringAttr intrin, mlir::ValueRange args,
3703	mlir::LLVM::FastmathFlagsAttr fastMathFlags) {
3704	build(builder, state, resultTypes, intrin, args, fastMathFlags,
3705	/*op_bundle_operands=*/{}, /*op_bundle_tags=*/{}, /*arg_attrs=*/{},
3706	/*res_attrs=*/{});
3707	}
3708
3709	ParseResult CallIntrinsicOp::parse(OpAsmParser &parser,
3710	OperationState &result) {
3711	StringAttr intrinAttr;
3712	SmallVector<OpAsmParser::UnresolvedOperand, 4> operands;
3713	SmallVector<SmallVector<OpAsmParser::UnresolvedOperand>> opBundleOperands;
3714	SmallVector<SmallVector<Type>> opBundleOperandTypes;
3715	ArrayAttr opBundleTags;
3716
3717	// Parse intrinsic name.
3718	if (parser.parseCustomAttributeWithFallback(
3719	intrinAttr, parser.getBuilder().getType<NoneType>()))
3720	return failure();
3721	result.addAttribute(CallIntrinsicOp::getIntrinAttrName(result.name),
3722	intrinAttr);
3723
3724	if (parser.parseLParen())
3725	return failure();
3726
3727	// Parse the function arguments.
3728	if (parser.parseOperandList(operands))
3729	return mlir::failure();
3730
3731	if (parser.parseRParen())
3732	return mlir::failure();
3733
3734	// Handle bundles.
3735	SMLoc opBundlesLoc = parser.getCurrentLocation();
3736	if (std::optional<ParseResult> result = parseOpBundles(
3737	parser, opBundleOperands, opBundleOperandTypes, opBundleTags);
3738	result && failed(*result))
3739	return failure();
3740	if (opBundleTags && !opBundleTags.empty())
3741	result.addAttribute(
3742	CallIntrinsicOp::getOpBundleTagsAttrName(result.name).getValue(),
3743	opBundleTags);
3744
3745	if (parser.parseOptionalAttrDict(result.attributes))
3746	return mlir::failure();
3747
3748	SmallVector<DictionaryAttr> argAttrs;
3749	SmallVector<DictionaryAttr> resultAttrs;
3750	if (parseCallTypeAndResolveOperands(parser, result, /*isDirect=*/true,
3751	operands, argAttrs, resultAttrs))
3752	return failure();
3753	call_interface_impl::addArgAndResultAttrs(
3754	parser.getBuilder(), result, argAttrs, resultAttrs,
3755	getArgAttrsAttrName(result.name), getResAttrsAttrName(result.name));
3756
3757	if (resolveOpBundleOperands(parser, opBundlesLoc, result, opBundleOperands,
3758	opBundleOperandTypes,
3759	getOpBundleSizesAttrName(result.name)))
3760	return failure();
3761
3762	int32_t numOpBundleOperands = 0;
3763	for (const auto &operands : opBundleOperands)
3764	numOpBundleOperands += operands.size();
3765
3766	result.addAttribute(
3767	CallIntrinsicOp::getOperandSegmentSizeAttr(),
3768	parser.getBuilder().getDenseI32ArrayAttr(
3769	{static_cast<int32_t>(operands.size()), numOpBundleOperands}));
3770
3771	return mlir::success();
3772	}
3773
3774	void CallIntrinsicOp::print(OpAsmPrinter &p) {
3775	p << ' ';
3776	p.printAttributeWithoutType(getIntrinAttr());
3777
3778	OperandRange args = getArgs();
3779	p << "(" << args << ")";
3780
3781	// Operand bundles.
3782	if (!getOpBundleOperands().empty()) {
3783	p << ' ';
3784	printOpBundles(p, *this, getOpBundleOperands(),
3785	getOpBundleOperands().getTypes(), getOpBundleTagsAttr());
3786	}
3787
3788	p.printOptionalAttrDict(processFMFAttr((*this)->getAttrs()),
3789	{getOperandSegmentSizesAttrName(),
3790	getOpBundleSizesAttrName(), getIntrinAttrName(),
3791	getOpBundleTagsAttrName(), getArgAttrsAttrName(),
3792	getResAttrsAttrName()});
3793
3794	p << " : ";
3795
3796	// Reconstruct the MLIR function type from operand and result types.
3797	call_interface_impl::printFunctionSignature(
3798	p, args.getTypes(), getArgAttrsAttr(),
3799	/*isVariadic=*/false, getResultTypes(), getResAttrsAttr());
3800	}
3801
3802	//===----------------------------------------------------------------------===//
3803	// LinkerOptionsOp
3804	//===----------------------------------------------------------------------===//
3805
3806	LogicalResult LinkerOptionsOp::verify() {
3807	if (mlir::Operation *parentOp = (*this)->getParentOp();
3808	parentOp && !satisfiesLLVMModule(parentOp))
3809	return emitOpError("must appear at the module level");
3810	return success();
3811	}
3812
3813	//===----------------------------------------------------------------------===//
3814	// ModuleFlagsOp
3815	//===----------------------------------------------------------------------===//
3816
3817	LogicalResult ModuleFlagsOp::verify() {
3818	if (Operation *parentOp = (*this)->getParentOp();
3819	parentOp && !satisfiesLLVMModule(parentOp))
3820	return emitOpError("must appear at the module level");
3821	for (Attribute flag : getFlags())
3822	if (!isa<ModuleFlagAttr>(flag))
3823	return emitOpError("expected a module flag attribute");
3824	return success();
3825	}
3826
3827	//===----------------------------------------------------------------------===//
3828	// InlineAsmOp
3829	//===----------------------------------------------------------------------===//
3830
3831	void InlineAsmOp::getEffects(
3832	SmallVectorImpl<SideEffects::EffectInstance<MemoryEffects::Effect>>
3833	&effects) {
3834	if (getHasSideEffects()) {
3835	effects.emplace_back(MemoryEffects::Write::get());
3836	effects.emplace_back(MemoryEffects::Read::get());
3837	}
3838	}
3839
3840	//===----------------------------------------------------------------------===//
3841	// BlockAddressOp
3842	//===----------------------------------------------------------------------===//
3843
3844	LogicalResult
3845	BlockAddressOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
3846	Operation *symbol = symbolTable.lookupSymbolIn(parentLLVMModule(*this),
3847	getBlockAddr().getFunction());
3848	auto function = dyn_cast_or_null<LLVMFuncOp>(symbol);
3849
3850	if (!function)
3851	return emitOpError("must reference a function defined by 'llvm.func'");
3852
3853	return success();
3854	}
3855
3856	LLVMFuncOp BlockAddressOp::getFunction(SymbolTableCollection &symbolTable) {
3857	return dyn_cast_or_null<LLVMFuncOp>(symbolTable.lookupSymbolIn(
3858	parentLLVMModule(*this), getBlockAddr().getFunction()));
3859	}
3860
3861	BlockTagOp BlockAddressOp::getBlockTagOp() {
3862	auto funcOp = dyn_cast<LLVMFuncOp>(mlir::SymbolTable::lookupNearestSymbolFrom(
3863	parentLLVMModule(*this), getBlockAddr().getFunction()));
3864	if (!funcOp)
3865	return nullptr;
3866
3867	BlockTagOp blockTagOp = nullptr;
3868	funcOp.walk([&](LLVM::BlockTagOp labelOp) {
3869	if (labelOp.getTag() == getBlockAddr().getTag()) {
3870	blockTagOp = labelOp;
3871	return WalkResult::interrupt();
3872	}
3873	return WalkResult::advance();
3874	});
3875	return blockTagOp;
3876	}
3877
3878	LogicalResult BlockAddressOp::verify() {
3879	if (!getBlockTagOp())
3880	return emitOpError(
3881	"expects an existing block label target in the referenced function");
3882
3883	return success();
3884	}
3885
3886	/// Fold a blockaddress operation to a dedicated blockaddress
3887	/// attribute.
3888	OpFoldResult BlockAddressOp::fold(FoldAdaptor) { return getBlockAddr(); }
3889
3890	//===----------------------------------------------------------------------===//
3891	// LLVM::IndirectBrOp
3892	//===----------------------------------------------------------------------===//
3893
3894	SuccessorOperands IndirectBrOp::getSuccessorOperands(unsigned index) {
3895	assert(index < getNumSuccessors() && "invalid successor index");
3896	return SuccessorOperands(getSuccOperandsMutable()[index]);
3897	}
3898
3899	void IndirectBrOp::build(OpBuilder &odsBuilder, OperationState &odsState,
3900	Value addr, ArrayRef<ValueRange> succOperands,
3901	BlockRange successors) {
3902	odsState.addOperands(addr);
3903	for (ValueRange range : succOperands)
3904	odsState.addOperands(range);
3905	SmallVector<int32_t> rangeSegments;
3906	for (ValueRange range : succOperands)
3907	rangeSegments.push_back(range.size());
3908	odsState.getOrAddProperties<Properties>().indbr_operand_segments =
3909	odsBuilder.getDenseI32ArrayAttr(rangeSegments);
3910	odsState.addSuccessors(successors);
3911	}
3912
3913	static ParseResult parseIndirectBrOpSucessors(
3914	OpAsmParser &parser, Type &flagType,
3915	SmallVectorImpl<Block *> &succOperandBlocks,
3916	SmallVectorImpl<SmallVector<OpAsmParser::UnresolvedOperand>> &succOperands,
3917	SmallVectorImpl<SmallVector<Type>> &succOperandsTypes) {
3918	if (failed(Result: parser.parseCommaSeparatedList(
3919	delimiter: OpAsmParser::Delimiter::Square,
3920	parseElementFn: [&]() {
3921	Block *destination = nullptr;
3922	SmallVector<OpAsmParser::UnresolvedOperand> operands;
3923	SmallVector<Type> operandTypes;
3924
3925	if (parser.parseSuccessor(dest&: destination).failed())
3926	return failure();
3927
3928	if (succeeded(Result: parser.parseOptionalLParen())) {
3929	if (failed(Result: parser.parseOperandList(
3930	result&: operands, delimiter: OpAsmParser::Delimiter::None)) ||
3931	failed(Result: parser.parseColonTypeList(result&: operandTypes)) ||
3932	failed(Result: parser.parseRParen()))
3933	return failure();
3934	}
3935	succOperandBlocks.push_back(Elt: destination);
3936	succOperands.emplace_back(Args&: operands);
3937	succOperandsTypes.emplace_back(Args&: operandTypes);
3938	return success();
3939	},
3940	contextMessage: "successor blocks")))
3941	return failure();
3942	return success();
3943	}
3944
3945	static void
3946	printIndirectBrOpSucessors(OpAsmPrinter &p, IndirectBrOp op, Type flagType,
3947	SuccessorRange succs, OperandRangeRange succOperands,
3948	const TypeRangeRange &succOperandsTypes) {
3949	p << "[";
3950	llvm::interleave(
3951	c: llvm::zip(t&: succs, u&: succOperands),
3952	each_fn: [&](auto i) {
3953	p.printNewline();
3954	p.printSuccessorAndUseList(successor: std::get<0>(i), succOperands: std::get<1>(i));
3955	},
3956	between_fn: [&] { p << ','; });
3957	if (!succOperands.empty())
3958	p.printNewline();
3959	p << "]";
3960	}
3961
3962	//===----------------------------------------------------------------------===//
3963	// AssumeOp (intrinsic)
3964	//===----------------------------------------------------------------------===//
3965
3966	void LLVM::AssumeOp::build(OpBuilder &builder, OperationState &state,
3967	mlir::Value cond) {
3968	return build(builder, state, cond, /*op_bundle_operands=*/{},
3969	/*op_bundle_tags=*/ArrayAttr{});
3970	}
3971
3972	void LLVM::AssumeOp::build(OpBuilder &builder, OperationState &state,
3973	Value cond,
3974	ArrayRef<llvm::OperandBundleDefT<Value>> opBundles) {
3975	SmallVector<ValueRange> opBundleOperands;
3976	SmallVector<Attribute> opBundleTags;
3977	opBundleOperands.reserve(opBundles.size());
3978	opBundleTags.reserve(opBundles.size());
3979
3980	for (const llvm::OperandBundleDefT<Value> &bundle : opBundles) {
3981	opBundleOperands.emplace_back(bundle.inputs());
3982	opBundleTags.push_back(
3983	StringAttr::get(builder.getContext(), bundle.getTag()));
3984	}
3985
3986	auto opBundleTagsAttr = ArrayAttr::get(builder.getContext(), opBundleTags);
3987	return build(builder, state, cond, opBundleOperands, opBundleTagsAttr);
3988	}
3989
3990	void LLVM::AssumeOp::build(OpBuilder &builder, OperationState &state,
3991	Value cond, llvm::StringRef tag, ValueRange args) {
3992	llvm::OperandBundleDefT<Value> opBundle(
3993	tag.str(), SmallVector<Value>(args.begin(), args.end()));
3994	return build(builder, state, cond, opBundle);
3995	}
3996
3997	void LLVM::AssumeOp::build(OpBuilder &builder, OperationState &state,
3998	Value cond, AssumeAlignTag, Value ptr, Value align) {
3999	return build(builder, state, cond, "align", ValueRange{ptr, align});
4000	}
4001
4002	void LLVM::AssumeOp::build(OpBuilder &builder, OperationState &state,
4003	Value cond, AssumeSeparateStorageTag, Value ptr1,
4004	Value ptr2) {
4005	return build(builder, state, cond, "separate_storage",
4006	ValueRange{ptr1, ptr2});
4007	}
4008
4009	LogicalResult LLVM::AssumeOp::verify() { return verifyOperandBundles(*this); }
4010
4011	//===----------------------------------------------------------------------===//
4012	// masked_gather (intrinsic)
4013	//===----------------------------------------------------------------------===//
4014
4015	LogicalResult LLVM::masked_gather::verify() {
4016	auto ptrsVectorType = getPtrs().getType();
4017	Type expectedPtrsVectorType =
4018	LLVM::getVectorType(extractVectorElementType(ptrsVectorType),
4019	LLVM::getVectorNumElements(getRes().getType()));
4020	// Vector of pointers type should match result vector type, other than the
4021	// element type.
4022	if (ptrsVectorType != expectedPtrsVectorType)
4023	return emitOpError("expected operand #1 type to be ")
4024	<< expectedPtrsVectorType;
4025	return success();
4026	}
4027
4028	//===----------------------------------------------------------------------===//
4029	// masked_scatter (intrinsic)
4030	//===----------------------------------------------------------------------===//
4031
4032	LogicalResult LLVM::masked_scatter::verify() {
4033	auto ptrsVectorType = getPtrs().getType();
4034	Type expectedPtrsVectorType =
4035	LLVM::getVectorType(extractVectorElementType(ptrsVectorType),
4036	LLVM::getVectorNumElements(getValue().getType()));
4037	// Vector of pointers type should match value vector type, other than the
4038	// element type.
4039	if (ptrsVectorType != expectedPtrsVectorType)
4040	return emitOpError("expected operand #2 type to be ")
4041	<< expectedPtrsVectorType;
4042	return success();
4043	}
4044
4045	//===----------------------------------------------------------------------===//
4046	// InlineAsmOp
4047	//===----------------------------------------------------------------------===//
4048
4049	LogicalResult InlineAsmOp::verify() {
4050	if (!getTailCallKindAttr())
4051	return success();
4052
4053	if (getTailCallKindAttr().getTailCallKind() == TailCallKind::MustTail)
4054	return emitOpError(
4055	"tail call kind 'musttail' is not supported by this operation");
4056
4057	return success();
4058	}
4059
4060	//===----------------------------------------------------------------------===//
4061	// LLVMDialect initialization, type parsing, and registration.
4062	//===----------------------------------------------------------------------===//
4063
4064	void LLVMDialect::initialize() {
4065	registerAttributes();
4066
4067	// clang-format off
4068	addTypes<LLVMVoidType,
4069	LLVMTokenType,
4070	LLVMLabelType,
4071	LLVMMetadataType>();
4072	// clang-format on
4073	registerTypes();
4074
4075	addOperations<
4076	#define GET_OP_LIST
4077	#include "mlir/Dialect/LLVMIR/LLVMOps.cpp.inc"
4078	,
4079	#define GET_OP_LIST
4080	#include "mlir/Dialect/LLVMIR/LLVMIntrinsicOps.cpp.inc"
4081	>();
4082
4083	// Support unknown operations because not all LLVM operations are registered.
4084	allowUnknownOperations();
4085	declarePromisedInterface<DialectInlinerInterface, LLVMDialect>();
4086	}
4087
4088	#define GET_OP_CLASSES
4089	#include "mlir/Dialect/LLVMIR/LLVMOps.cpp.inc"
4090
4091	#define GET_OP_CLASSES
4092	#include "mlir/Dialect/LLVMIR/LLVMIntrinsicOps.cpp.inc"
4093
4094	LogicalResult LLVMDialect::verifyDataLayoutString(
4095	StringRef descr, llvm::function_ref<void(const Twine &)> reportError) {
4096	llvm::Expected<llvm::DataLayout> maybeDataLayout =
4097	llvm::DataLayout::parse(descr);
4098	if (maybeDataLayout)
4099	return success();
4100
4101	std::string message;
4102	llvm::raw_string_ostream messageStream(message);
4103	llvm::logAllUnhandledErrors(maybeDataLayout.takeError(), messageStream);
4104	reportError("invalid data layout descriptor: " + message);
4105	return failure();
4106	}
4107
4108	/// Verify LLVM dialect attributes.
4109	LogicalResult LLVMDialect::verifyOperationAttribute(Operation *op,
4110	NamedAttribute attr) {
4111	// If the data layout attribute is present, it must use the LLVM data layout
4112	// syntax. Try parsing it and report errors in case of failure. Users of this
4113	// attribute may assume it is well-formed and can pass it to the (asserting)
4114	// llvm::DataLayout constructor.
4115	if (attr.getName() != LLVM::LLVMDialect::getDataLayoutAttrName())
4116	return success();
4117	if (auto stringAttr = llvm::dyn_cast<StringAttr>(attr.getValue()))
4118	return verifyDataLayoutString(
4119	stringAttr.getValue(),
4120	[op](const Twine &message) { op->emitOpError() << message.str(); });
4121
4122	return op->emitOpError() << "expected '"
4123	<< LLVM::LLVMDialect::getDataLayoutAttrName()
4124	<< "' to be a string attributes";
4125	}
4126
4127	LogicalResult LLVMDialect::verifyParameterAttribute(Operation *op,
4128	Type paramType,
4129	NamedAttribute paramAttr) {
4130	// LLVM attribute may be attached to a result of operation that has not been
4131	// converted to LLVM dialect yet, so the result may have a type with unknown
4132	// representation in LLVM dialect type space. In this case we cannot verify
4133	// whether the attribute may be
4134	bool verifyValueType = isCompatibleType(paramType);
4135	StringAttr name = paramAttr.getName();
4136
4137	auto checkUnitAttrType = [&]() -> LogicalResult {
4138	if (!llvm::isa<UnitAttr>(paramAttr.getValue()))
4139	return op->emitError() << name << " should be a unit attribute";
4140	return success();
4141	};
4142	auto checkTypeAttrType = [&]() -> LogicalResult {
4143	if (!llvm::isa<TypeAttr>(paramAttr.getValue()))
4144	return op->emitError() << name << " should be a type attribute";
4145	return success();
4146	};
4147	auto checkIntegerAttrType = [&]() -> LogicalResult {
4148	if (!llvm::isa<IntegerAttr>(paramAttr.getValue()))
4149	return op->emitError() << name << " should be an integer attribute";
4150	return success();
4151	};
4152	auto checkPointerType = [&]() -> LogicalResult {
4153	if (!llvm::isa<LLVMPointerType>(paramType))
4154	return op->emitError()
4155	<< name << " attribute attached to non-pointer LLVM type";
4156	return success();
4157	};
4158	auto checkIntegerType = [&]() -> LogicalResult {
4159	if (!llvm::isa<IntegerType>(paramType))
4160	return op->emitError()
4161	<< name << " attribute attached to non-integer LLVM type";
4162	return success();
4163	};
4164	auto checkPointerTypeMatches = [&]() -> LogicalResult {
4165	if (failed(checkPointerType()))
4166	return failure();
4167
4168	return success();
4169	};
4170
4171	// Check a unit attribute that is attached to a pointer value.
4172	if (name == LLVMDialect::getNoAliasAttrName() ||
4173	name == LLVMDialect::getReadonlyAttrName() ||
4174	name == LLVMDialect::getReadnoneAttrName() ||
4175	name == LLVMDialect::getWriteOnlyAttrName() ||
4176	name == LLVMDialect::getNestAttrName() ||
4177	name == LLVMDialect::getNoCaptureAttrName() ||
4178	name == LLVMDialect::getNoFreeAttrName() ||
4179	name == LLVMDialect::getNonNullAttrName()) {
4180	if (failed(checkUnitAttrType()))
4181	return failure();
4182	if (verifyValueType && failed(checkPointerType()))
4183	return failure();
4184	return success();
4185	}
4186
4187	// Check a type attribute that is attached to a pointer value.
4188	if (name == LLVMDialect::getStructRetAttrName() ||
4189	name == LLVMDialect::getByValAttrName() ||
4190	name == LLVMDialect::getByRefAttrName() ||
4191	name == LLVMDialect::getElementTypeAttrName() ||
4192	name == LLVMDialect::getInAllocaAttrName() ||
4193	name == LLVMDialect::getPreallocatedAttrName()) {
4194	if (failed(checkTypeAttrType()))
4195	return failure();
4196	if (verifyValueType && failed(checkPointerTypeMatches()))
4197	return failure();
4198	return success();
4199	}
4200
4201	// Check a unit attribute that is attached to an integer value.
4202	if (name == LLVMDialect::getSExtAttrName() ||
4203	name == LLVMDialect::getZExtAttrName()) {
4204	if (failed(checkUnitAttrType()))
4205	return failure();
4206	if (verifyValueType && failed(checkIntegerType()))
4207	return failure();
4208	return success();
4209	}
4210
4211	// Check an integer attribute that is attached to a pointer value.
4212	if (name == LLVMDialect::getAlignAttrName() ||
4213	name == LLVMDialect::getDereferenceableAttrName() ||
4214	name == LLVMDialect::getDereferenceableOrNullAttrName()) {
4215	if (failed(checkIntegerAttrType()))
4216	return failure();
4217	if (verifyValueType && failed(checkPointerType()))
4218	return failure();
4219	return success();
4220	}
4221
4222	// Check an integer attribute that is attached to a pointer value.
4223	if (name == LLVMDialect::getStackAlignmentAttrName()) {
4224	if (failed(checkIntegerAttrType()))
4225	return failure();
4226	return success();
4227	}
4228
4229	// Check a unit attribute that can be attached to arbitrary types.
4230	if (name == LLVMDialect::getNoUndefAttrName() ||
4231	name == LLVMDialect::getInRegAttrName() ||
4232	name == LLVMDialect::getReturnedAttrName())
4233	return checkUnitAttrType();
4234
4235	return success();
4236	}
4237
4238	/// Verify LLVMIR function argument attributes.
4239	LogicalResult LLVMDialect::verifyRegionArgAttribute(Operation *op,
4240	unsigned regionIdx,
4241	unsigned argIdx,
4242	NamedAttribute argAttr) {
4243	auto funcOp = dyn_cast<FunctionOpInterface>(op);
4244	if (!funcOp)
4245	return success();
4246	Type argType = funcOp.getArgumentTypes()[argIdx];
4247
4248	return verifyParameterAttribute(op, argType, argAttr);
4249	}
4250
4251	LogicalResult LLVMDialect::verifyRegionResultAttribute(Operation *op,
4252	unsigned regionIdx,
4253	unsigned resIdx,
4254	NamedAttribute resAttr) {
4255	auto funcOp = dyn_cast<FunctionOpInterface>(op);
4256	if (!funcOp)
4257	return success();
4258	Type resType = funcOp.getResultTypes()[resIdx];
4259
4260	// Check to see if this function has a void return with a result attribute
4261	// to it. It isn't clear what semantics we would assign to that.
4262	if (llvm::isa<LLVMVoidType>(resType))
4263	return op->emitError() << "cannot attach result attributes to functions "
4264	"with a void return";
4265
4266	// Check to see if this attribute is allowed as a result attribute. Only
4267	// explicitly forbidden LLVM attributes will cause an error.
4268	auto name = resAttr.getName();
4269	if (name == LLVMDialect::getAllocAlignAttrName() ||
4270	name == LLVMDialect::getAllocatedPointerAttrName() ||
4271	name == LLVMDialect::getByValAttrName() ||
4272	name == LLVMDialect::getByRefAttrName() ||
4273	name == LLVMDialect::getInAllocaAttrName() ||
4274	name == LLVMDialect::getNestAttrName() ||
4275	name == LLVMDialect::getNoCaptureAttrName() ||
4276	name == LLVMDialect::getNoFreeAttrName() ||
4277	name == LLVMDialect::getPreallocatedAttrName() ||
4278	name == LLVMDialect::getReadnoneAttrName() ||
4279	name == LLVMDialect::getReadonlyAttrName() ||
4280	name == LLVMDialect::getReturnedAttrName() ||
4281	name == LLVMDialect::getStackAlignmentAttrName() ||
4282	name == LLVMDialect::getStructRetAttrName() ||
4283	name == LLVMDialect::getWriteOnlyAttrName())
4284	return op->emitError() << name << " is not a valid result attribute";
4285	return verifyParameterAttribute(op, resType, resAttr);
4286	}
4287
4288	Operation *LLVMDialect::materializeConstant(OpBuilder &builder, Attribute value,
4289	Type type, Location loc) {
4290	// If this was folded from an operation other than llvm.mlir.constant, it
4291	// should be materialized as such. Note that an llvm.mlir.zero may fold into
4292	// a builtin zero attribute and thus will materialize as a llvm.mlir.constant.
4293	if (auto symbol = dyn_cast<FlatSymbolRefAttr>(value))
4294	if (isa<LLVM::LLVMPointerType>(type))
4295	return builder.create<LLVM::AddressOfOp>(loc, type, symbol);
4296	if (isa<LLVM::UndefAttr>(value))
4297	return builder.create<LLVM::UndefOp>(loc, type);
4298	if (isa<LLVM::PoisonAttr>(value))
4299	return builder.create<LLVM::PoisonOp>(loc, type);
4300	if (isa<LLVM::ZeroAttr>(value))
4301	return builder.create<LLVM::ZeroOp>(loc, type);
4302	// Otherwise try materializing it as a regular llvm.mlir.constant op.
4303	return LLVM::ConstantOp::materialize(builder, value, type, loc);
4304	}
4305
4306	//===----------------------------------------------------------------------===//
4307	// Utility functions.
4308	//===----------------------------------------------------------------------===//
4309
4310	Value mlir::LLVM::createGlobalString(Location loc, OpBuilder &builder,
4311	StringRef name, StringRef value,
4312	LLVM::Linkage linkage) {
4313	assert(builder.getInsertionBlock() &&
4314	builder.getInsertionBlock()->getParentOp() &&
4315	"expected builder to point to a block constrained in an op");
4316	auto module =
4317	builder.getInsertionBlock()->getParentOp()->getParentOfType<ModuleOp>();
4318	assert(module && "builder points to an op outside of a module");
4319
4320	// Create the global at the entry of the module.
4321	OpBuilder moduleBuilder(module.getBodyRegion(), builder.getListener());
4322	MLIRContext *ctx = builder.getContext();
4323	auto type = LLVM::LLVMArrayType::get(IntegerType::get(ctx, 8), value.size());
4324	auto global = moduleBuilder.create<LLVM::GlobalOp>(
4325	loc, type, /*isConstant=*/true, linkage, name,
4326	builder.getStringAttr(value), /*alignment=*/0);
4327
4328	LLVMPointerType ptrType = LLVMPointerType::get(ctx);
4329	// Get the pointer to the first character in the global string.
4330	Value globalPtr =
4331	builder.create<LLVM::AddressOfOp>(loc, ptrType, global.getSymNameAttr());
4332	return builder.create<LLVM::GEPOp>(loc, ptrType, type, globalPtr,
4333	ArrayRef<GEPArg>{0, 0});
4334	}
4335
4336	bool mlir::LLVM::satisfiesLLVMModule(Operation *op) {
4337	return op->hasTrait<OpTrait::SymbolTable>() &&
4338	op->hasTrait<OpTrait::IsIsolatedFromAbove>();
4339	}
4340