1	//===- Operation.cpp - Operation support code -----------------------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8
9	#include "mlir/IR/Operation.h"
10	#include "mlir/IR/Attributes.h"
11	#include "mlir/IR/BuiltinAttributes.h"
12	#include "mlir/IR/BuiltinTypes.h"
13	#include "mlir/IR/Dialect.h"
14	#include "mlir/IR/IRMapping.h"
15	#include "mlir/IR/Matchers.h"
16	#include "mlir/IR/OpImplementation.h"
17	#include "mlir/IR/OperationSupport.h"
18	#include "mlir/IR/PatternMatch.h"
19	#include "mlir/IR/TypeUtilities.h"
20	#include "mlir/Interfaces/FoldInterfaces.h"
21	#include "llvm/ADT/SmallVector.h"
22	#include "llvm/ADT/StringExtras.h"
23	#include "llvm/Support/ErrorHandling.h"
24	#include <numeric>
25	#include <optional>
26
27	using namespace mlir;
28
29	//===----------------------------------------------------------------------===//
30	// Operation
31	//===----------------------------------------------------------------------===//
32
33	/// Create a new Operation from operation state.
34	Operation *Operation::create(const OperationState &state) {
35	Operation *op =
36	create(state.location, state.name, state.types, state.operands,
37	state.attributes.getDictionary(state.getContext()),
38	state.properties, state.successors, state.regions);
39	if (LLVM_UNLIKELY(state.propertiesAttr)) {
40	assert(!state.properties);
41	LogicalResult result =
42	op->setPropertiesFromAttribute(attr: state.propertiesAttr,
43	/*diagnostic=*/emitError: nullptr);
44	assert(result.succeeded() && "invalid properties in op creation");
45	(void)result;
46	}
47	return op;
48	}
49
50	/// Create a new Operation with the specific fields.
51	Operation *Operation::create(Location location, OperationName name,
52	TypeRange resultTypes, ValueRange operands,
53	NamedAttrList &&attributes,
54	OpaqueProperties properties, BlockRange successors,
55	RegionRange regions) {
56	unsigned numRegions = regions.size();
57	Operation *op =
58	create(location, name, resultTypes, operands, attributes: std::move(attributes),
59	properties, successors, numRegions);
60	for (unsigned i = 0; i < numRegions; ++i)
61	if (regions[i])
62	op->getRegion(index: i).takeBody(other&: *regions[i]);
63	return op;
64	}
65
66	/// Create a new Operation with the specific fields.
67	Operation *Operation::create(Location location, OperationName name,
68	TypeRange resultTypes, ValueRange operands,
69	NamedAttrList &&attributes,
70	OpaqueProperties properties, BlockRange successors,
71	unsigned numRegions) {
72	// Populate default attributes.
73	name.populateDefaultAttrs(attrs&: attributes);
74
75	return create(location, name, resultTypes, operands,
76	attributes.getDictionary(location.getContext()), properties,
77	successors, numRegions);
78	}
79
80	/// Overload of create that takes an existing DictionaryAttr to avoid
81	/// unnecessarily uniquing a list of attributes.
82	Operation *Operation::create(Location location, OperationName name,
83	TypeRange resultTypes, ValueRange operands,
84	DictionaryAttr attributes,
85	OpaqueProperties properties, BlockRange successors,
86	unsigned numRegions) {
87	assert(llvm::all_of(resultTypes, [](Type t) { return t; }) &&
88	"unexpected null result type");
89
90	// We only need to allocate additional memory for a subset of results.
91	unsigned numTrailingResults = OpResult::getNumTrailing(numResults: resultTypes.size());
92	unsigned numInlineResults = OpResult::getNumInline(numResults: resultTypes.size());
93	unsigned numSuccessors = successors.size();
94	unsigned numOperands = operands.size();
95	unsigned numResults = resultTypes.size();
96	int opPropertiesAllocSize = llvm::alignTo<8>(Value: name.getOpPropertyByteSize());
97
98	// If the operation is known to have no operands, don't allocate an operand
99	// storage.
100	bool needsOperandStorage =
101	operands.empty() ? !name.hasTrait<OpTrait::ZeroOperands>() : true;
102
103	// Compute the byte size for the operation and the operand storage. This takes
104	// into account the size of the operation, its trailing objects, and its
105	// prefixed objects.
106	size_t byteSize =
107	totalSizeToAlloc<detail::OperandStorage, detail::OpProperties,
108	BlockOperand, Region, OpOperand>(
109	Counts: needsOperandStorage ? 1 : 0, Counts: opPropertiesAllocSize, Counts: numSuccessors,
110	Counts: numRegions, Counts: numOperands);
111	size_t prefixByteSize = llvm::alignTo(
112	Value: Operation::prefixAllocSize(numOutOfLineResults: numTrailingResults, numInlineResults),
113	Align: alignof(Operation));
114	char *mallocMem = reinterpret_cast<char *>(malloc(size: byteSize + prefixByteSize));
115	void *rawMem = mallocMem + prefixByteSize;
116
117	// Create the new Operation.
118	Operation *op = ::new (rawMem) Operation(
119	location, name, numResults, numSuccessors, numRegions,
120	opPropertiesAllocSize, attributes, properties, needsOperandStorage);
121
122	assert((numSuccessors == 0 || op->mightHaveTrait<OpTrait::IsTerminator>()) &&
123	"unexpected successors in a non-terminator operation");
124
125	// Initialize the results.
126	auto resultTypeIt = resultTypes.begin();
127	for (unsigned i = 0; i < numInlineResults; ++i, ++resultTypeIt)
128	new (op->getInlineOpResult(resultNumber: i)) detail::InlineOpResult(*resultTypeIt, i);
129	for (unsigned i = 0; i < numTrailingResults; ++i, ++resultTypeIt) {
130	new (op->getOutOfLineOpResult(resultNumber: i))
131	detail::OutOfLineOpResult(*resultTypeIt, i);
132	}
133
134	// Initialize the regions.
135	for (unsigned i = 0; i != numRegions; ++i)
136	new (&op->getRegion(index: i)) Region(op);
137
138	// Initialize the operands.
139	if (needsOperandStorage) {
140	new (&op->getOperandStorage()) detail::OperandStorage(
141	op, op->getTrailingObjects<OpOperand>(), operands);
142	}
143
144	// Initialize the successors.
145	auto blockOperands = op->getBlockOperands();
146	for (unsigned i = 0; i != numSuccessors; ++i)
147	new (&blockOperands[i]) BlockOperand(op, successors[i]);
148
149	// This must be done after properties are initalized.
150	op->setAttrs(attributes);
151
152	return op;
153	}
154
155	Operation::Operation(Location location, OperationName name, unsigned numResults,
156	unsigned numSuccessors, unsigned numRegions,
157	int fullPropertiesStorageSize, DictionaryAttr attributes,
158	OpaqueProperties properties, bool hasOperandStorage)
159	: location(location), numResults(numResults), numSuccs(numSuccessors),
160	numRegions(numRegions), hasOperandStorage(hasOperandStorage),
161	propertiesStorageSize((fullPropertiesStorageSize + 7) / 8), name(name) {
162	assert(attributes && "unexpected null attribute dictionary");
163	assert(fullPropertiesStorageSize <= propertiesCapacity &&
164	"Properties size overflow");
165	#ifndef NDEBUG
166	if (!getDialect() && !getContext()->allowsUnregisteredDialects())
167	llvm::report_fatal_error(
168	reason: name.getStringRef() +
169	" created with unregistered dialect. If this is intended, please call "
170	"allowUnregisteredDialects() on the MLIRContext, or use "
171	"-allow-unregistered-dialect with the MLIR tool used.");
172	#endif
173	if (fullPropertiesStorageSize)
174	name.initOpProperties(storage: getPropertiesStorage(), init: properties);
175	}
176
177	// Operations are deleted through the destroy() member because they are
178	// allocated via malloc.
179	Operation::~Operation() {
180	assert(block == nullptr && "operation destroyed but still in a block");
181	#ifndef NDEBUG
182	if (!use_empty()) {
183	{
184	InFlightDiagnostic diag =
185	emitOpError(message: "operation destroyed but still has uses");
186	for (Operation *user : getUsers())
187	diag.attachNote(noteLoc: user->getLoc()) << "- use: " << *user << "\n";
188	}
189	llvm::report_fatal_error(reason: "operation destroyed but still has uses");
190	}
191	#endif
192	// Explicitly run the destructors for the operands.
193	if (hasOperandStorage)
194	getOperandStorage().~OperandStorage();
195
196	// Explicitly run the destructors for the successors.
197	for (auto &successor : getBlockOperands())
198	successor.~BlockOperand();
199
200	// Explicitly destroy the regions.
201	for (auto &region : getRegions())
202	region.~Region();
203	if (propertiesStorageSize)
204	name.destroyOpProperties(properties: getPropertiesStorage());
205	}
206
207	/// Destroy this operation or one of its subclasses.
208	void Operation::destroy() {
209	// Operations may have additional prefixed allocation, which needs to be
210	// accounted for here when computing the address to free.
211	char *rawMem = reinterpret_cast<char *>(this) -
212	llvm::alignTo(Value: prefixAllocSize(), Align: alignof(Operation));
213	this->~Operation();
214	free(ptr: rawMem);
215	}
216
217	/// Return true if this operation is a proper ancestor of the `other`
218	/// operation.
219	bool Operation::isProperAncestor(Operation *other) {
220	while ((other = other->getParentOp()))
221	if (this == other)
222	return true;
223	return false;
224	}
225
226	/// Replace any uses of 'from' with 'to' within this operation.
227	void Operation::replaceUsesOfWith(Value from, Value to) {
228	if (from == to)
229	return;
230	for (auto &operand : getOpOperands())
231	if (operand.get() == from)
232	operand.set(to);
233	}
234
235	/// Replace the current operands of this operation with the ones provided in
236	/// 'operands'.
237	void Operation::setOperands(ValueRange operands) {
238	if (LLVM_LIKELY(hasOperandStorage))
239	return getOperandStorage().setOperands(owner: this, values: operands);
240	assert(operands.empty() && "setting operands without an operand storage");
241	}
242
243	/// Replace the operands beginning at 'start' and ending at 'start' + 'length'
244	/// with the ones provided in 'operands'. 'operands' may be smaller or larger
245	/// than the range pointed to by 'start'+'length'.
246	void Operation::setOperands(unsigned start, unsigned length,
247	ValueRange operands) {
248	assert((start + length) <= getNumOperands() &&
249	"invalid operand range specified");
250	if (LLVM_LIKELY(hasOperandStorage))
251	return getOperandStorage().setOperands(owner: this, start, length, operands);
252	assert(operands.empty() && "setting operands without an operand storage");
253	}
254
255	/// Insert the given operands into the operand list at the given 'index'.
256	void Operation::insertOperands(unsigned index, ValueRange operands) {
257	if (LLVM_LIKELY(hasOperandStorage))
258	return setOperands(start: index, /*length=*/0, operands);
259	assert(operands.empty() && "inserting operands without an operand storage");
260	}
261
262	//===----------------------------------------------------------------------===//
263	// Diagnostics
264	//===----------------------------------------------------------------------===//
265
266	/// Emit an error about fatal conditions with this operation, reporting up to
267	/// any diagnostic handlers that may be listening.
268	InFlightDiagnostic Operation::emitError(const Twine &message) {
269	InFlightDiagnostic diag = mlir::emitError(loc: getLoc(), message);
270	if (getContext()->shouldPrintOpOnDiagnostic()) {
271	diag.attachNote(noteLoc: getLoc())
272	.append(arg: "see current operation: ")
273	.appendOp(op&: *this, flags: OpPrintingFlags().printGenericOpForm());
274	}
275	return diag;
276	}
277
278	/// Emit a warning about this operation, reporting up to any diagnostic
279	/// handlers that may be listening.
280	InFlightDiagnostic Operation::emitWarning(const Twine &message) {
281	InFlightDiagnostic diag = mlir::emitWarning(loc: getLoc(), message);
282	if (getContext()->shouldPrintOpOnDiagnostic())
283	diag.attachNote(noteLoc: getLoc()) << "see current operation: " << *this;
284	return diag;
285	}
286
287	/// Emit a remark about this operation, reporting up to any diagnostic
288	/// handlers that may be listening.
289	InFlightDiagnostic Operation::emitRemark(const Twine &message) {
290	InFlightDiagnostic diag = mlir::emitRemark(loc: getLoc(), message);
291	if (getContext()->shouldPrintOpOnDiagnostic())
292	diag.attachNote(noteLoc: getLoc()) << "see current operation: " << *this;
293	return diag;
294	}
295
296	DictionaryAttr Operation::getAttrDictionary() {
297	if (getPropertiesStorageSize()) {
298	NamedAttrList attrsList = attrs;
299	getName().populateInherentAttrs(op: this, attrs&: attrsList);
300	return attrsList.getDictionary(getContext());
301	}
302	return attrs;
303	}
304
305	void Operation::setAttrs(DictionaryAttr newAttrs) {
306	assert(newAttrs && "expected valid attribute dictionary");
307	if (getPropertiesStorageSize()) {
308	// We're spliting the providing DictionaryAttr by removing the inherentAttr
309	// which will be stored in the properties.
310	SmallVector<NamedAttribute> discardableAttrs;
311	discardableAttrs.reserve(N: newAttrs.size());
312	for (NamedAttribute attr : newAttrs) {
313	if (getInherentAttr(attr.getName()))
314	setInherentAttr(attr.getName(), attr.getValue());
315	else
316	discardableAttrs.push_back(attr);
317	}
318	if (discardableAttrs.size() != newAttrs.size())
319	newAttrs = DictionaryAttr::get(getContext(), discardableAttrs);
320	}
321	attrs = newAttrs;
322	}
323	void Operation::setAttrs(ArrayRef<NamedAttribute> newAttrs) {
324	if (getPropertiesStorageSize()) {
325	// We're spliting the providing array of attributes by removing the
326	// inherentAttr which will be stored in the properties.
327	SmallVector<NamedAttribute> discardableAttrs;
328	discardableAttrs.reserve(N: newAttrs.size());
329	for (NamedAttribute attr : newAttrs) {
330	if (getInherentAttr(attr.getName()))
331	setInherentAttr(attr.getName(), attr.getValue());
332	else
333	discardableAttrs.push_back(Elt: attr);
334	}
335	attrs = DictionaryAttr::get(getContext(), discardableAttrs);
336	return;
337	}
338	attrs = DictionaryAttr::get(getContext(), newAttrs);
339	}
340
341	std::optional<Attribute> Operation::getInherentAttr(StringRef name) {
342	return getName().getInherentAttr(op: this, name);
343	}
344
345	void Operation::setInherentAttr(StringAttr name, Attribute value) {
346	getName().setInherentAttr(op: this, name: name, value);
347	}
348
349	Attribute Operation::getPropertiesAsAttribute() {
350	std::optional<RegisteredOperationName> info = getRegisteredInfo();
351	if (LLVM_UNLIKELY(!info))
352	return *getPropertiesStorage().as<Attribute *>();
353	return info->getOpPropertiesAsAttribute(op: this);
354	}
355	LogicalResult Operation::setPropertiesFromAttribute(
356	Attribute attr, function_ref<InFlightDiagnostic()> emitError) {
357	std::optional<RegisteredOperationName> info = getRegisteredInfo();
358	if (LLVM_UNLIKELY(!info)) {
359	*getPropertiesStorage().as<Attribute *>() = attr;
360	return success();
361	}
362	return info->setOpPropertiesFromAttribute(
363	opName: this->getName(), properties: this->getPropertiesStorage(), attr, emitError);
364	}
365
366	void Operation::copyProperties(OpaqueProperties rhs) {
367	name.copyOpProperties(lhs: getPropertiesStorage(), rhs);
368	}
369
370	llvm::hash_code Operation::hashProperties() {
371	return name.hashOpProperties(properties: getPropertiesStorage());
372	}
373
374	//===----------------------------------------------------------------------===//
375	// Operation Ordering
376	//===----------------------------------------------------------------------===//
377
378	constexpr unsigned Operation::kInvalidOrderIdx;
379	constexpr unsigned Operation::kOrderStride;
380
381	/// Given an operation 'other' that is within the same parent block, return
382	/// whether the current operation is before 'other' in the operation list
383	/// of the parent block.
384	/// Note: This function has an average complexity of O(1), but worst case may
385	/// take O(N) where N is the number of operations within the parent block.
386	bool Operation::isBeforeInBlock(Operation *other) {
387	assert(block && "Operations without parent blocks have no order.");
388	assert(other && other->block == block &&
389	"Expected other operation to have the same parent block.");
390	// If the order of the block is already invalid, directly recompute the
391	// parent.
392	if (!block->isOpOrderValid()) {
393	block->recomputeOpOrder();
394	} else {
395	// Update the order either operation if necessary.
396	updateOrderIfNecessary();
397	other->updateOrderIfNecessary();
398	}
399
400	return orderIndex < other->orderIndex;
401	}
402
403	/// Update the order index of this operation of this operation if necessary,
404	/// potentially recomputing the order of the parent block.
405	void Operation::updateOrderIfNecessary() {
406	assert(block && "expected valid parent");
407
408	// If the order is valid for this operation there is nothing to do.
409	if (hasValidOrder() || llvm::hasSingleElement(C&: *block))
410	return;
411	Operation *blockFront = &block->front();
412	Operation *blockBack = &block->back();
413
414	// This method is expected to only be invoked on blocks with more than one
415	// operation.
416	assert(blockFront != blockBack && "expected more than one operation");
417
418	// If the operation is at the end of the block.
419	if (this == blockBack) {
420	Operation *prevNode = getPrevNode();
421	if (!prevNode->hasValidOrder())
422	return block->recomputeOpOrder();
423
424	// Add the stride to the previous operation.
425	orderIndex = prevNode->orderIndex + kOrderStride;
426	return;
427	}
428
429	// If this is the first operation try to use the next operation to compute the
430	// ordering.
431	if (this == blockFront) {
432	Operation *nextNode = getNextNode();
433	if (!nextNode->hasValidOrder())
434	return block->recomputeOpOrder();
435	// There is no order to give this operation.
436	if (nextNode->orderIndex == 0)
437	return block->recomputeOpOrder();
438
439	// If we can't use the stride, just take the middle value left. This is safe
440	// because we know there is at least one valid index to assign to.
441	if (nextNode->orderIndex <= kOrderStride)
442	orderIndex = (nextNode->orderIndex / 2);
443	else
444	orderIndex = kOrderStride;
445	return;
446	}
447
448	// Otherwise, this operation is between two others. Place this operation in
449	// the middle of the previous and next if possible.
450	Operation *prevNode = getPrevNode(), *nextNode = getNextNode();
451	if (!prevNode->hasValidOrder() || !nextNode->hasValidOrder())
452	return block->recomputeOpOrder();
453	unsigned prevOrder = prevNode->orderIndex, nextOrder = nextNode->orderIndex;
454
455	// Check to see if there is a valid order between the two.
456	if (prevOrder + 1 == nextOrder)
457	return block->recomputeOpOrder();
458	orderIndex = prevOrder + ((nextOrder - prevOrder) / 2);
459	}
460
461	//===----------------------------------------------------------------------===//
462	// ilist_traits for Operation
463	//===----------------------------------------------------------------------===//
464
465	auto llvm::ilist_detail::SpecificNodeAccess<
466	typename llvm::ilist_detail::compute_node_options<
467	::mlir::Operation>::type>::getNodePtr(pointer n) -> node_type * {
468	return NodeAccess::getNodePtr<OptionsT>(N: n);
469	}
470
471	auto llvm::ilist_detail::SpecificNodeAccess<
472	typename llvm::ilist_detail::compute_node_options<
473	::mlir::Operation>::type>::getNodePtr(const_pointer n)
474	-> const node_type * {
475	return NodeAccess::getNodePtr<OptionsT>(N: n);
476	}
477
478	auto llvm::ilist_detail::SpecificNodeAccess<
479	typename llvm::ilist_detail::compute_node_options<
480	::mlir::Operation>::type>::getValuePtr(node_type *n) -> pointer {
481	return NodeAccess::getValuePtr<OptionsT>(N: n);
482	}
483
484	auto llvm::ilist_detail::SpecificNodeAccess<
485	typename llvm::ilist_detail::compute_node_options<
486	::mlir::Operation>::type>::getValuePtr(const node_type *n)
487	-> const_pointer {
488	return NodeAccess::getValuePtr<OptionsT>(N: n);
489	}
490
491	void llvm::ilist_traits<::mlir::Operation>::deleteNode(Operation *op) {
492	op->destroy();
493	}
494
495	Block *llvm::ilist_traits<::mlir::Operation>::getContainingBlock() {
496	size_t offset(size_t(&((Block *)nullptr->*Block::getSublistAccess(nullptr))));
497	iplist<Operation> *anchor(static_cast<iplist<Operation> *>(this));
498	return reinterpret_cast<Block *>(reinterpret_cast<char *>(anchor) - offset);
499	}
500
501	/// This is a trait method invoked when an operation is added to a block. We
502	/// keep the block pointer up to date.
503	void llvm::ilist_traits<::mlir::Operation>::addNodeToList(Operation *op) {
504	assert(!op->getBlock() && "already in an operation block!");
505	op->block = getContainingBlock();
506
507	// Invalidate the order on the operation.
508	op->orderIndex = Operation::kInvalidOrderIdx;
509	}
510
511	/// This is a trait method invoked when an operation is removed from a block.
512	/// We keep the block pointer up to date.
513	void llvm::ilist_traits<::mlir::Operation>::removeNodeFromList(Operation *op) {
514	assert(op->block && "not already in an operation block!");
515	op->block = nullptr;
516	}
517
518	/// This is a trait method invoked when an operation is moved from one block
519	/// to another. We keep the block pointer up to date.
520	void llvm::ilist_traits<::mlir::Operation>::transferNodesFromList(
521	ilist_traits<Operation> &otherList, op_iterator first, op_iterator last) {
522	Block *curParent = getContainingBlock();
523
524	// Invalidate the ordering of the parent block.
525	curParent->invalidateOpOrder();
526
527	// If we are transferring operations within the same block, the block
528	// pointer doesn't need to be updated.
529	if (curParent == otherList.getContainingBlock())
530	return;
531
532	// Update the 'block' member of each operation.
533	for (; first != last; ++first)
534	first->block = curParent;
535	}
536
537	/// Remove this operation (and its descendants) from its Block and delete
538	/// all of them.
539	void Operation::erase() {
540	if (auto *parent = getBlock())
541	parent->getOperations().erase(IT: this);
542	else
543	destroy();
544	}
545
546	/// Remove the operation from its parent block, but don't delete it.
547	void Operation::remove() {
548	if (Block *parent = getBlock())
549	parent->getOperations().remove(IT: this);
550	}
551
552	/// Unlink this operation from its current block and insert it right before
553	/// `existingOp` which may be in the same or another block in the same
554	/// function.
555	void Operation::moveBefore(Operation *existingOp) {
556	moveBefore(existingOp->getBlock(), existingOp->getIterator());
557	}
558
559	/// Unlink this operation from its current basic block and insert it right
560	/// before `iterator` in the specified basic block.
561	void Operation::moveBefore(Block *block,
562	llvm::iplist<Operation>::iterator iterator) {
563	assert(getBlock() &&
564	"cannot move an operation that isn't contained in a block");
565	block->getOperations().splice(iterator, getBlock()->getOperations(),
566	getIterator());
567	}
568
569	/// Unlink this operation from its current block and insert it right after
570	/// `existingOp` which may be in the same or another block in the same function.
571	void Operation::moveAfter(Operation *existingOp) {
572	moveAfter(existingOp->getBlock(), existingOp->getIterator());
573	}
574
575	/// Unlink this operation from its current block and insert it right after
576	/// `iterator` in the specified block.
577	void Operation::moveAfter(Block *block,
578	llvm::iplist<Operation>::iterator iterator) {
579	assert(iterator != block->end() && "cannot move after end of block");
580	moveBefore(block, iterator: std::next(x: iterator));
581	}
582
583	/// This drops all operand uses from this operation, which is an essential
584	/// step in breaking cyclic dependences between references when they are to
585	/// be deleted.
586	void Operation::dropAllReferences() {
587	for (auto &op : getOpOperands())
588	op.drop();
589
590	for (auto &region : getRegions())
591	region.dropAllReferences();
592
593	for (auto &dest : getBlockOperands())
594	dest.drop();
595	}
596
597	/// This drops all uses of any values defined by this operation or its nested
598	/// regions, wherever they are located.
599	void Operation::dropAllDefinedValueUses() {
600	dropAllUses();
601
602	for (auto &region : getRegions())
603	for (auto &block : region)
604	block.dropAllDefinedValueUses();
605	}
606
607	void Operation::setSuccessor(Block *block, unsigned index) {
608	assert(index < getNumSuccessors());
609	getBlockOperands()[index].set(block);
610	}
611
612	#ifndef NDEBUG
613	/// Assert that the folded results (in case of values) have the same type as
614	/// the results of the given op.
615	static void checkFoldResultTypes(Operation *op,
616	SmallVectorImpl<OpFoldResult> &results) {
617	if (results.empty())
618	return;
619
620	for (auto [ofr, opResult] : llvm::zip_equal(t&: results, u: op->getResults())) {
621	if (auto value = dyn_cast<Value>(Val&: ofr)) {
622	if (value.getType() != opResult.getType()) {
623	op->emitOpError() << "folder produced a value of incorrect type: "
624	<< value.getType()
625	<< ", expected: " << opResult.getType();
626	assert(false && "incorrect fold result type");
627	}
628	}
629	}
630	}
631	#endif // NDEBUG
632
633	/// Attempt to fold this operation using the Op's registered foldHook.
634	LogicalResult Operation::fold(ArrayRef<Attribute> operands,
635	SmallVectorImpl<OpFoldResult> &results) {
636	// If we have a registered operation definition matching this one, use it to
637	// try to constant fold the operation.
638	if (succeeded(Result: name.foldHook(op: this, operands, results))) {
639	#ifndef NDEBUG
640	checkFoldResultTypes(op: this, results);
641	#endif // NDEBUG
642	return success();
643	}
644
645	// Otherwise, fall back on the dialect hook to handle it.
646	Dialect *dialect = getDialect();
647	if (!dialect)
648	return failure();
649
650	auto *interface = dyn_cast<DialectFoldInterface>(Val: dialect);
651	if (!interface)
652	return failure();
653
654	LogicalResult status = interface->fold(op: this, operands, results);
655	#ifndef NDEBUG
656	if (succeeded(Result: status))
657	checkFoldResultTypes(op: this, results);
658	#endif // NDEBUG
659	return status;
660	}
661
662	LogicalResult Operation::fold(SmallVectorImpl<OpFoldResult> &results) {
663	// Check if any operands are constants.
664	SmallVector<Attribute> constants;
665	constants.assign(NumElts: getNumOperands(), Elt: Attribute());
666	for (unsigned i = 0, e = getNumOperands(); i != e; ++i)
667	matchPattern(value: getOperand(idx: i), pattern: m_Constant(bind_value: &constants[i]));
668	return fold(operands: constants, results);
669	}
670
671	/// Emit an error with the op name prefixed, like "'dim' op " which is
672	/// convenient for verifiers.
673	InFlightDiagnostic Operation::emitOpError(const Twine &message) {
674	return emitError() << "'" << getName() << "' op " << message;
675	}
676
677	//===----------------------------------------------------------------------===//
678	// Operation Cloning
679	//===----------------------------------------------------------------------===//
680
681	Operation::CloneOptions::CloneOptions()
682	: cloneRegionsFlag(false), cloneOperandsFlag(false) {}
683
684	Operation::CloneOptions::CloneOptions(bool cloneRegions, bool cloneOperands)
685	: cloneRegionsFlag(cloneRegions), cloneOperandsFlag(cloneOperands) {}
686
687	Operation::CloneOptions Operation::CloneOptions::all() {
688	return CloneOptions().cloneRegions().cloneOperands();
689	}
690
691	Operation::CloneOptions &Operation::CloneOptions::cloneRegions(bool enable) {
692	cloneRegionsFlag = enable;
693	return *this;
694	}
695
696	Operation::CloneOptions &Operation::CloneOptions::cloneOperands(bool enable) {
697	cloneOperandsFlag = enable;
698	return *this;
699	}
700
701	/// Create a deep copy of this operation but keep the operation regions empty.
702	/// Operands are remapped using `mapper` (if present), and `mapper` is updated
703	/// to contain the results. The `mapResults` flag specifies whether the results
704	/// of the cloned operation should be added to the map.
705	Operation *Operation::cloneWithoutRegions(IRMapping &mapper) {
706	return clone(mapper, options: CloneOptions::all().cloneRegions(enable: false));
707	}
708
709	Operation *Operation::cloneWithoutRegions() {
710	IRMapping mapper;
711	return cloneWithoutRegions(mapper);
712	}
713
714	/// Create a deep copy of this operation, remapping any operands that use
715	/// values outside of the operation using the map that is provided (leaving
716	/// them alone if no entry is present). Replaces references to cloned
717	/// sub-operations to the corresponding operation that is copied, and adds
718	/// those mappings to the map.
719	Operation *Operation::clone(IRMapping &mapper, CloneOptions options) {
720	SmallVector<Value, 8> operands;
721	SmallVector<Block *, 2> successors;
722
723	// Remap the operands.
724	if (options.shouldCloneOperands()) {
725	operands.reserve(N: getNumOperands());
726	for (auto opValue : getOperands())
727	operands.push_back(Elt: mapper.lookupOrDefault(from: opValue));
728	}
729
730	// Remap the successors.
731	successors.reserve(N: getNumSuccessors());
732	for (Block *successor : getSuccessors())
733	successors.push_back(Elt: mapper.lookupOrDefault(from: successor));
734
735	// Create the new operation.
736	auto *newOp = create(getLoc(), getName(), getResultTypes(), operands, attrs,
737	getPropertiesStorage(), successors, getNumRegions());
738	mapper.map(this, newOp);
739
740	// Clone the regions.
741	if (options.shouldCloneRegions()) {
742	for (unsigned i = 0; i != numRegions; ++i)
743	getRegion(index: i).cloneInto(&newOp->getRegion(i), mapper);
744	}
745
746	// Remember the mapping of any results.
747	for (unsigned i = 0, e = getNumResults(); i != e; ++i)
748	mapper.map(getResult(idx: i), newOp->getResult(i));
749
750	return newOp;
751	}
752
753	Operation *Operation::clone(CloneOptions options) {
754	IRMapping mapper;
755	return clone(mapper, options);
756	}
757
758	//===----------------------------------------------------------------------===//
759	// OpState trait class.
760	//===----------------------------------------------------------------------===//
761
762	// The fallback for the parser is to try for a dialect operation parser.
763	// Otherwise, reject the custom assembly form.
764	ParseResult OpState::parse(OpAsmParser &parser, OperationState &result) {
765	if (auto parseFn = result.name.getDialect()->getParseOperationHook(
766	opName: result.name.getStringRef()))
767	return (*parseFn)(parser, result);
768	return parser.emitError(loc: parser.getNameLoc(), message: "has no custom assembly form");
769	}
770
771	// The fallback for the printer is to try for a dialect operation printer.
772	// Otherwise, it prints the generic form.
773	void OpState::print(Operation *op, OpAsmPrinter &p, StringRef defaultDialect) {
774	if (auto printFn = op->getDialect()->getOperationPrinter(op)) {
775	printOpName(op, p, defaultDialect);
776	printFn(op, p);
777	} else {
778	p.printGenericOp(op);
779	}
780	}
781
782	/// Print an operation name, eliding the dialect prefix if necessary and doesn't
783	/// lead to ambiguities.
784	void OpState::printOpName(Operation *op, OpAsmPrinter &p,
785	StringRef defaultDialect) {
786	StringRef name = op->getName().getStringRef();
787	if (name.starts_with(Prefix: (defaultDialect + ".").str()) && name.count(C: '.') == 1)
788	name = name.drop_front(N: defaultDialect.size() + 1);
789	p.getStream() << name;
790	}
791
792	/// Parse properties as a Attribute.
793	ParseResult OpState::genericParseProperties(OpAsmParser &parser,
794	Attribute &result) {
795	if (succeeded(Result: parser.parseOptionalLess())) { // The less is optional.
796	if (parser.parseAttribute(result) || parser.parseGreater())
797	return failure();
798	}
799	return success();
800	}
801
802	/// Print the properties as a Attribute with names not included within
803	/// 'elidedProps'
804	void OpState::genericPrintProperties(OpAsmPrinter &p, Attribute properties,
805	ArrayRef<StringRef> elidedProps) {
806	if (!properties)
807	return;
808	auto dictAttr = dyn_cast_or_null<::mlir::DictionaryAttr>(properties);
809	if (dictAttr && !elidedProps.empty()) {
810	ArrayRef<NamedAttribute> attrs = dictAttr.getValue();
811	llvm::SmallDenseSet<StringRef> elidedAttrsSet(elidedProps.begin(),
812	elidedProps.end());
813	bool atLeastOneAttr = llvm::any_of(Range&: attrs, P: [&](NamedAttribute attr) {
814	return !elidedAttrsSet.contains(attr.getName().strref());
815	});
816	if (atLeastOneAttr) {
817	p << "<";
818	p.printOptionalAttrDict(attrs: dictAttr.getValue(), elidedAttrs: elidedProps);
819	p << ">";
820	}
821	} else {
822	p << "<" << properties << ">";
823	}
824	}
825
826	/// Emit an error about fatal conditions with this operation, reporting up to
827	/// any diagnostic handlers that may be listening.
828	InFlightDiagnostic OpState::emitError(const Twine &message) {
829	return getOperation()->emitError(message);
830	}
831
832	/// Emit an error with the op name prefixed, like "'dim' op " which is
833	/// convenient for verifiers.
834	InFlightDiagnostic OpState::emitOpError(const Twine &message) {
835	return getOperation()->emitOpError(message);
836	}
837
838	/// Emit a warning about this operation, reporting up to any diagnostic
839	/// handlers that may be listening.
840	InFlightDiagnostic OpState::emitWarning(const Twine &message) {
841	return getOperation()->emitWarning(message);
842	}
843
844	/// Emit a remark about this operation, reporting up to any diagnostic
845	/// handlers that may be listening.
846	InFlightDiagnostic OpState::emitRemark(const Twine &message) {
847	return getOperation()->emitRemark(message);
848	}
849
850	//===----------------------------------------------------------------------===//
851	// Op Trait implementations
852	//===----------------------------------------------------------------------===//
853
854	LogicalResult
855	OpTrait::impl::foldCommutative(Operation *op, ArrayRef<Attribute> operands,
856	SmallVectorImpl<OpFoldResult> &results) {
857	// Nothing to fold if there are not at least 2 operands.
858	if (op->getNumOperands() < 2)
859	return failure();
860	// Move all constant operands to the end.
861	OpOperand *operandsBegin = op->getOpOperands().begin();
862	auto isNonConstant = [&](OpOperand &o) {
863	return !static_cast<bool>(operands[std::distance(first: operandsBegin, last: &o)]);
864	};
865	auto *firstConstantIt = llvm::find_if_not(Range: op->getOpOperands(), P: isNonConstant);
866	auto *newConstantIt = std::stable_partition(
867	first: firstConstantIt, last: op->getOpOperands().end(), pred: isNonConstant);
868	// Return success if the op was modified.
869	return success(IsSuccess: firstConstantIt != newConstantIt);
870	}
871
872	OpFoldResult OpTrait::impl::foldIdempotent(Operation *op) {
873	if (op->getNumOperands() == 1) {
874	auto *argumentOp = op->getOperand(idx: 0).getDefiningOp();
875	if (argumentOp && op->getName() == argumentOp->getName()) {
876	// Replace the outer operation output with the inner operation.
877	return op->getOperand(idx: 0);
878	}
879	} else if (op->getOperand(idx: 0) == op->getOperand(idx: 1)) {
880	return op->getOperand(idx: 0);
881	}
882
883	return {};
884	}
885
886	OpFoldResult OpTrait::impl::foldInvolution(Operation *op) {
887	auto *argumentOp = op->getOperand(idx: 0).getDefiningOp();
888	if (argumentOp && op->getName() == argumentOp->getName()) {
889	// Replace the outer involutions output with inner's input.
890	return argumentOp->getOperand(idx: 0);
891	}
892
893	return {};
894	}
895
896	LogicalResult OpTrait::impl::verifyZeroOperands(Operation *op) {
897	if (op->getNumOperands() != 0)
898	return op->emitOpError() << "requires zero operands";
899	return success();
900	}
901
902	LogicalResult OpTrait::impl::verifyOneOperand(Operation *op) {
903	if (op->getNumOperands() != 1)
904	return op->emitOpError() << "requires a single operand";
905	return success();
906	}
907
908	LogicalResult OpTrait::impl::verifyNOperands(Operation *op,
909	unsigned numOperands) {
910	if (op->getNumOperands() != numOperands) {
911	return op->emitOpError() << "expected " << numOperands
912	<< " operands, but found " << op->getNumOperands();
913	}
914	return success();
915	}
916
917	LogicalResult OpTrait::impl::verifyAtLeastNOperands(Operation *op,
918	unsigned numOperands) {
919	if (op->getNumOperands() < numOperands)
920	return op->emitOpError()
921	<< "expected " << numOperands << " or more operands, but found "
922	<< op->getNumOperands();
923	return success();
924	}
925
926	/// If this is a vector type, or a tensor type, return the scalar element type
927	/// that it is built around, otherwise return the type unmodified.
928	static Type getTensorOrVectorElementType(Type type) {
929	if (auto vec = llvm::dyn_cast<VectorType>(type))
930	return vec.getElementType();
931
932	// Look through tensor<vector<...>> to find the underlying element type.
933	if (auto tensor = llvm::dyn_cast<TensorType>(Val&: type))
934	return getTensorOrVectorElementType(type: tensor.getElementType());
935	return type;
936	}
937
938	LogicalResult OpTrait::impl::verifyIsIdempotent(Operation *op) {
939	// FIXME: Add back check for no side effects on operation.
940	// Currently adding it would cause the shared library build
941	// to fail since there would be a dependency of IR on SideEffectInterfaces
942	// which is cyclical.
943	return success();
944	}
945
946	LogicalResult OpTrait::impl::verifyIsInvolution(Operation *op) {
947	// FIXME: Add back check for no side effects on operation.
948	// Currently adding it would cause the shared library build
949	// to fail since there would be a dependency of IR on SideEffectInterfaces
950	// which is cyclical.
951	return success();
952	}
953
954	LogicalResult
955	OpTrait::impl::verifyOperandsAreSignlessIntegerLike(Operation *op) {
956	for (auto opType : op->getOperandTypes()) {
957	auto type = getTensorOrVectorElementType(type: opType);
958	if (!type.isSignlessIntOrIndex())
959	return op->emitOpError() << "requires an integer or index type";
960	}
961	return success();
962	}
963
964	LogicalResult OpTrait::impl::verifyOperandsAreFloatLike(Operation *op) {
965	for (auto opType : op->getOperandTypes()) {
966	auto type = getTensorOrVectorElementType(type: opType);
967	if (!llvm::isa<FloatType>(Val: type))
968	return op->emitOpError(message: "requires a float type");
969	}
970	return success();
971	}
972
973	LogicalResult OpTrait::impl::verifySameTypeOperands(Operation *op) {
974	// Zero or one operand always have the "same" type.
975	unsigned nOperands = op->getNumOperands();
976	if (nOperands < 2)
977	return success();
978
979	auto type = op->getOperand(idx: 0).getType();
980	for (auto opType : llvm::drop_begin(RangeOrContainer: op->getOperandTypes(), N: 1))
981	if (opType != type)
982	return op->emitOpError() << "requires all operands to have the same type";
983	return success();
984	}
985
986	LogicalResult OpTrait::impl::verifyZeroRegions(Operation *op) {
987	if (op->getNumRegions() != 0)
988	return op->emitOpError() << "requires zero regions";
989	return success();
990	}
991
992	LogicalResult OpTrait::impl::verifyOneRegion(Operation *op) {
993	if (op->getNumRegions() != 1)
994	return op->emitOpError() << "requires one region";
995	return success();
996	}
997
998	LogicalResult OpTrait::impl::verifyNRegions(Operation *op,
999	unsigned numRegions) {
1000	if (op->getNumRegions() != numRegions)
1001	return op->emitOpError() << "expected " << numRegions << " regions";
1002	return success();
1003	}
1004
1005	LogicalResult OpTrait::impl::verifyAtLeastNRegions(Operation *op,
1006	unsigned numRegions) {
1007	if (op->getNumRegions() < numRegions)
1008	return op->emitOpError() << "expected " << numRegions << " or more regions";
1009	return success();
1010	}
1011
1012	LogicalResult OpTrait::impl::verifyZeroResults(Operation *op) {
1013	if (op->getNumResults() != 0)
1014	return op->emitOpError() << "requires zero results";
1015	return success();
1016	}
1017
1018	LogicalResult OpTrait::impl::verifyOneResult(Operation *op) {
1019	if (op->getNumResults() != 1)
1020	return op->emitOpError() << "requires one result";
1021	return success();
1022	}
1023
1024	LogicalResult OpTrait::impl::verifyNResults(Operation *op,
1025	unsigned numOperands) {
1026	if (op->getNumResults() != numOperands)
1027	return op->emitOpError() << "expected " << numOperands << " results";
1028	return success();
1029	}
1030
1031	LogicalResult OpTrait::impl::verifyAtLeastNResults(Operation *op,
1032	unsigned numOperands) {
1033	if (op->getNumResults() < numOperands)
1034	return op->emitOpError()
1035	<< "expected " << numOperands << " or more results";
1036	return success();
1037	}
1038
1039	LogicalResult OpTrait::impl::verifySameOperandsShape(Operation *op) {
1040	if (failed(Result: verifyAtLeastNOperands(op, numOperands: 1)))
1041	return failure();
1042
1043	if (failed(Result: verifyCompatibleShapes(types: op->getOperandTypes())))
1044	return op->emitOpError() << "requires the same shape for all operands";
1045
1046	return success();
1047	}
1048
1049	LogicalResult OpTrait::impl::verifySameOperandsAndResultShape(Operation *op) {
1050	if (failed(Result: verifyAtLeastNOperands(op, numOperands: 1)) ||
1051	failed(Result: verifyAtLeastNResults(op, numOperands: 1)))
1052	return failure();
1053
1054	SmallVector<Type, 8> types(op->getOperandTypes());
1055	types.append(RHS: llvm::to_vector<4>(Range: op->getResultTypes()));
1056
1057	if (failed(Result: verifyCompatibleShapes(types)))
1058	return op->emitOpError()
1059	<< "requires the same shape for all operands and results";
1060
1061	return success();
1062	}
1063
1064	LogicalResult OpTrait::impl::verifySameOperandsElementType(Operation *op) {
1065	if (failed(Result: verifyAtLeastNOperands(op, numOperands: 1)))
1066	return failure();
1067	auto elementType = getElementTypeOrSelf(val: op->getOperand(idx: 0));
1068
1069	for (auto operand : llvm::drop_begin(RangeOrContainer: op->getOperands(), N: 1)) {
1070	if (getElementTypeOrSelf(val: operand) != elementType)
1071	return op->emitOpError(message: "requires the same element type for all operands");
1072	}
1073
1074	return success();
1075	}
1076
1077	LogicalResult
1078	OpTrait::impl::verifySameOperandsAndResultElementType(Operation *op) {
1079	if (failed(Result: verifyAtLeastNOperands(op, numOperands: 1)) ||
1080	failed(Result: verifyAtLeastNResults(op, numOperands: 1)))
1081	return failure();
1082
1083	auto elementType = getElementTypeOrSelf(val: op->getResult(idx: 0));
1084
1085	// Verify result element type matches first result's element type.
1086	for (auto result : llvm::drop_begin(RangeOrContainer: op->getResults(), N: 1)) {
1087	if (getElementTypeOrSelf(val: result) != elementType)
1088	return op->emitOpError(
1089	message: "requires the same element type for all operands and results");
1090	}
1091
1092	// Verify operand's element type matches first result's element type.
1093	for (auto operand : op->getOperands()) {
1094	if (getElementTypeOrSelf(val: operand) != elementType)
1095	return op->emitOpError(
1096	message: "requires the same element type for all operands and results");
1097	}
1098
1099	return success();
1100	}
1101
1102	LogicalResult OpTrait::impl::verifySameOperandsAndResultType(Operation *op) {
1103	if (failed(Result: verifyAtLeastNOperands(op, numOperands: 1)) ||
1104	failed(Result: verifyAtLeastNResults(op, numOperands: 1)))
1105	return failure();
1106
1107	auto type = op->getResult(idx: 0).getType();
1108	auto elementType = getElementTypeOrSelf(type);
1109	Attribute encoding = nullptr;
1110	if (auto rankedType = dyn_cast<RankedTensorType>(type))
1111	encoding = rankedType.getEncoding();
1112	for (auto resultType : llvm::drop_begin(RangeOrContainer: op->getResultTypes())) {
1113	if (getElementTypeOrSelf(type: resultType) != elementType ||
1114	failed(Result: verifyCompatibleShape(type1: resultType, type2: type)))
1115	return op->emitOpError()
1116	<< "requires the same type for all operands and results";
1117	if (encoding)
1118	if (auto rankedType = dyn_cast<RankedTensorType>(resultType);
1119	encoding != rankedType.getEncoding())
1120	return op->emitOpError()
1121	<< "requires the same encoding for all operands and results";
1122	}
1123	for (auto opType : op->getOperandTypes()) {
1124	if (getElementTypeOrSelf(type: opType) != elementType ||
1125	failed(Result: verifyCompatibleShape(type1: opType, type2: type)))
1126	return op->emitOpError()
1127	<< "requires the same type for all operands and results";
1128	if (encoding)
1129	if (auto rankedType = dyn_cast<RankedTensorType>(opType);
1130	encoding != rankedType.getEncoding())
1131	return op->emitOpError()
1132	<< "requires the same encoding for all operands and results";
1133	}
1134	return success();
1135	}
1136
1137	LogicalResult OpTrait::impl::verifySameOperandsAndResultRank(Operation *op) {
1138	if (failed(Result: verifyAtLeastNOperands(op, numOperands: 1)))
1139	return failure();
1140
1141	// delegate function that returns true if type is a shaped type with known
1142	// rank
1143	auto hasRank = [](const Type type) {
1144	if (auto shapedType = dyn_cast<ShapedType>(type))
1145	return shapedType.hasRank();
1146
1147	return false;
1148	};
1149
1150	auto rankedOperandTypes =
1151	llvm::make_filter_range(Range: op->getOperandTypes(), Pred: hasRank);
1152	auto rankedResultTypes =
1153	llvm::make_filter_range(Range: op->getResultTypes(), Pred: hasRank);
1154
1155	// If all operands and results are unranked, then no further verification.
1156	if (rankedOperandTypes.empty() && rankedResultTypes.empty())
1157	return success();
1158
1159	// delegate function that returns rank of shaped type with known rank
1160	auto getRank = [](const Type type) {
1161	return cast<ShapedType>(type).getRank();
1162	};
1163
1164	auto rank = !rankedOperandTypes.empty() ? getRank(*rankedOperandTypes.begin())
1165	: getRank(*rankedResultTypes.begin());
1166
1167	for (const auto type : rankedOperandTypes) {
1168	if (rank != getRank(type)) {
1169	return op->emitOpError(message: "operands don't have matching ranks");
1170	}
1171	}
1172
1173	for (const auto type : rankedResultTypes) {
1174	if (rank != getRank(type)) {
1175	return op->emitOpError(message: "result type has different rank than operands");
1176	}
1177	}
1178
1179	return success();
1180	}
1181
1182	LogicalResult OpTrait::impl::verifyIsTerminator(Operation *op) {
1183	Block *block = op->getBlock();
1184	// Verify that the operation is at the end of the respective parent block.
1185	if (!block || &block->back() != op)
1186	return op->emitOpError(message: "must be the last operation in the parent block");
1187	return success();
1188	}
1189
1190	static LogicalResult verifyTerminatorSuccessors(Operation *op) {
1191	auto *parent = op->getParentRegion();
1192
1193	// Verify that the operands lines up with the BB arguments in the successor.
1194	for (Block *succ : op->getSuccessors())
1195	if (succ->getParent() != parent)
1196	return op->emitError(message: "reference to block defined in another region");
1197	return success();
1198	}
1199
1200	LogicalResult OpTrait::impl::verifyZeroSuccessors(Operation *op) {
1201	if (op->getNumSuccessors() != 0) {
1202	return op->emitOpError(message: "requires 0 successors but found ")
1203	<< op->getNumSuccessors();
1204	}
1205	return success();
1206	}
1207
1208	LogicalResult OpTrait::impl::verifyOneSuccessor(Operation *op) {
1209	if (op->getNumSuccessors() != 1) {
1210	return op->emitOpError(message: "requires 1 successor but found ")
1211	<< op->getNumSuccessors();
1212	}
1213	return verifyTerminatorSuccessors(op);
1214	}
1215	LogicalResult OpTrait::impl::verifyNSuccessors(Operation *op,
1216	unsigned numSuccessors) {
1217	if (op->getNumSuccessors() != numSuccessors) {
1218	return op->emitOpError(message: "requires ")
1219	<< numSuccessors << " successors but found "
1220	<< op->getNumSuccessors();
1221	}
1222	return verifyTerminatorSuccessors(op);
1223	}
1224	LogicalResult OpTrait::impl::verifyAtLeastNSuccessors(Operation *op,
1225	unsigned numSuccessors) {
1226	if (op->getNumSuccessors() < numSuccessors) {
1227	return op->emitOpError(message: "requires at least ")
1228	<< numSuccessors << " successors but found "
1229	<< op->getNumSuccessors();
1230	}
1231	return verifyTerminatorSuccessors(op);
1232	}
1233
1234	LogicalResult OpTrait::impl::verifyResultsAreBoolLike(Operation *op) {
1235	for (auto resultType : op->getResultTypes()) {
1236	auto elementType = getTensorOrVectorElementType(type: resultType);
1237	bool isBoolType = elementType.isInteger(width: 1);
1238	if (!isBoolType)
1239	return op->emitOpError() << "requires a bool result type";
1240	}
1241
1242	return success();
1243	}
1244
1245	LogicalResult OpTrait::impl::verifyResultsAreFloatLike(Operation *op) {
1246	for (auto resultType : op->getResultTypes())
1247	if (!llvm::isa<FloatType>(Val: getTensorOrVectorElementType(type: resultType)))
1248	return op->emitOpError() << "requires a floating point type";
1249
1250	return success();
1251	}
1252
1253	LogicalResult
1254	OpTrait::impl::verifyResultsAreSignlessIntegerLike(Operation *op) {
1255	for (auto resultType : op->getResultTypes())
1256	if (!getTensorOrVectorElementType(type: resultType).isSignlessIntOrIndex())
1257	return op->emitOpError() << "requires an integer or index type";
1258	return success();
1259	}
1260
1261	LogicalResult OpTrait::impl::verifyValueSizeAttr(Operation *op,
1262	StringRef attrName,
1263	StringRef valueGroupName,
1264	size_t expectedCount) {
1265	auto sizeAttr = op->getAttrOfType<DenseI32ArrayAttr>(attrName);
1266	if (!sizeAttr)
1267	return op->emitOpError(message: "requires dense i32 array attribute '")
1268	<< attrName << "'";
1269
1270	ArrayRef<int32_t> sizes = sizeAttr.asArrayRef();
1271	if (llvm::any_of(Range&: sizes, P: [](int32_t element) { return element < 0; }))
1272	return op->emitOpError(message: "'")
1273	<< attrName << "' attribute cannot have negative elements";
1274
1275	size_t totalCount =
1276	std::accumulate(first: sizes.begin(), last: sizes.end(), init: 0,
1277	binary_op: [](unsigned all, int32_t one) { return all + one; });
1278
1279	if (totalCount != expectedCount)
1280	return op->emitOpError()
1281	<< valueGroupName << " count (" << expectedCount
1282	<< ") does not match with the total size (" << totalCount
1283	<< ") specified in attribute '" << attrName << "'";
1284	return success();
1285	}
1286
1287	LogicalResult OpTrait::impl::verifyOperandSizeAttr(Operation *op,
1288	StringRef attrName) {
1289	return verifyValueSizeAttr(op, attrName, valueGroupName: "operand", expectedCount: op->getNumOperands());
1290	}
1291
1292	LogicalResult OpTrait::impl::verifyResultSizeAttr(Operation *op,
1293	StringRef attrName) {
1294	return verifyValueSizeAttr(op, attrName, valueGroupName: "result", expectedCount: op->getNumResults());
1295	}
1296
1297	LogicalResult OpTrait::impl::verifyNoRegionArguments(Operation *op) {
1298	for (Region &region : op->getRegions()) {
1299	if (region.empty())
1300	continue;
1301
1302	if (region.getNumArguments() != 0) {
1303	if (op->getNumRegions() > 1)
1304	return op->emitOpError(message: "region #")
1305	<< region.getRegionNumber() << " should have no arguments";
1306	return op->emitOpError(message: "region should have no arguments");
1307	}
1308	}
1309	return success();
1310	}
1311
1312	LogicalResult OpTrait::impl::verifyElementwise(Operation *op) {
1313	auto isMappableType = llvm::IsaPred<VectorType, TensorType>;
1314	auto resultMappableTypes =
1315	llvm::filter_to_vector<1>(C: op->getResultTypes(), Pred&: isMappableType);
1316	auto operandMappableTypes =
1317	llvm::filter_to_vector<2>(C: op->getOperandTypes(), Pred&: isMappableType);
1318
1319	// If the op only has scalar operand/result types, then we have nothing to
1320	// check.
1321	if (resultMappableTypes.empty() && operandMappableTypes.empty())
1322	return success();
1323
1324	if (!resultMappableTypes.empty() && operandMappableTypes.empty())
1325	return op->emitOpError(message: "if a result is non-scalar, then at least one "
1326	"operand must be non-scalar");
1327
1328	assert(!operandMappableTypes.empty());
1329
1330	if (resultMappableTypes.empty())
1331	return op->emitOpError(message: "if an operand is non-scalar, then there must be at "
1332	"least one non-scalar result");
1333
1334	if (resultMappableTypes.size() != op->getNumResults())
1335	return op->emitOpError(
1336	message: "if an operand is non-scalar, then all results must be non-scalar");
1337
1338	SmallVector<Type, 4> types = llvm::to_vector<2>(
1339	Range: llvm::concat<Type>(Ranges&: operandMappableTypes, Ranges&: resultMappableTypes));
1340	TypeID expectedBaseTy = types.front().getTypeID();
1341	if (!llvm::all_of(Range&: types,
1342	P: [&](Type t) { return t.getTypeID() == expectedBaseTy; }) ||
1343	failed(Result: verifyCompatibleShapes(types))) {
1344	return op->emitOpError() << "all non-scalar operands/results must have the "
1345	"same shape and base type";
1346	}
1347
1348	return success();
1349	}
1350
1351	/// Check for any values used by operations regions attached to the
1352	/// specified "IsIsolatedFromAbove" operation defined outside of it.
1353	LogicalResult OpTrait::impl::verifyIsIsolatedFromAbove(Operation *isolatedOp) {
1354	assert(isolatedOp->hasTrait<OpTrait::IsIsolatedFromAbove>() &&
1355	"Intended to check IsolatedFromAbove ops");
1356
1357	// List of regions to analyze. Each region is processed independently, with
1358	// respect to the common `limit` region, so we can look at them in any order.
1359	// Therefore, use a simple vector and push/pop back the current region.
1360	SmallVector<Region *, 8> pendingRegions;
1361	for (auto &region : isolatedOp->getRegions()) {
1362	pendingRegions.push_back(Elt: &region);
1363
1364	// Traverse all operations in the region.
1365	while (!pendingRegions.empty()) {
1366	for (Operation &op : pendingRegions.pop_back_val()->getOps()) {
1367	for (Value operand : op.getOperands()) {
1368	// Check that any value that is used by an operation is defined in the
1369	// same region as either an operation result.
1370	auto *operandRegion = operand.getParentRegion();
1371	if (!operandRegion)
1372	return op.emitError(message: "operation's operand is unlinked");
1373	if (!region.isAncestor(other: operandRegion)) {
1374	return op.emitOpError(message: "using value defined outside the region")
1375	.attachNote(noteLoc: isolatedOp->getLoc())
1376	<< "required by region isolation constraints";
1377	}
1378	}
1379
1380	// Schedule any regions in the operation for further checking. Don't
1381	// recurse into other IsolatedFromAbove ops, because they will check
1382	// themselves.
1383	if (op.getNumRegions() &&
1384	!op.hasTrait<OpTrait::IsIsolatedFromAbove>()) {
1385	for (Region &subRegion : op.getRegions())
1386	pendingRegions.push_back(Elt: &subRegion);
1387	}
1388	}
1389	}
1390	}
1391
1392	return success();
1393	}
1394
1395	bool OpTrait::hasElementwiseMappableTraits(Operation *op) {
1396	return op->hasTrait<Elementwise>() && op->hasTrait<Scalarizable>() &&
1397	op->hasTrait<Vectorizable>() && op->hasTrait<Tensorizable>();
1398	}
1399
1400	//===----------------------------------------------------------------------===//
1401	// Misc. utils
1402	//===----------------------------------------------------------------------===//
1403
1404	/// Insert an operation, generated by `buildTerminatorOp`, at the end of the
1405	/// region's only block if it does not have a terminator already. If the region
1406	/// is empty, insert a new block first. `buildTerminatorOp` should return the
1407	/// terminator operation to insert.
1408	void impl::ensureRegionTerminator(
1409	Region &region, OpBuilder &builder, Location loc,
1410	function_ref<Operation *(OpBuilder &, Location)> buildTerminatorOp) {
1411	OpBuilder::InsertionGuard guard(builder);
1412	if (region.empty())
1413	builder.createBlock(parent: &region);
1414
1415	Block &block = region.back();
1416	if (!block.empty() && block.back().hasTrait<OpTrait::IsTerminator>())
1417	return;
1418
1419	builder.setInsertionPointToEnd(&block);
1420	builder.insert(op: buildTerminatorOp(builder, loc));
1421	}
1422
1423	/// Create a simple OpBuilder and forward to the OpBuilder version of this
1424	/// function.
1425	void impl::ensureRegionTerminator(
1426	Region &region, Builder &builder, Location loc,
1427	function_ref<Operation *(OpBuilder &, Location)> buildTerminatorOp) {
1428	OpBuilder opBuilder(builder.getContext());
1429	ensureRegionTerminator(region, builder&: opBuilder, loc, buildTerminatorOp);
1430	}
1431