1	//===- Operation.cpp - Operation support code -----------------------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8
9	#include "mlir/IR/Operation.h"
10	#include "mlir/IR/Attributes.h"
11	#include "mlir/IR/BuiltinAttributes.h"
12	#include "mlir/IR/BuiltinTypes.h"
13	#include "mlir/IR/Dialect.h"
14	#include "mlir/IR/IRMapping.h"
15	#include "mlir/IR/Matchers.h"
16	#include "mlir/IR/OpImplementation.h"
17	#include "mlir/IR/OperationSupport.h"
18	#include "mlir/IR/PatternMatch.h"
19	#include "mlir/IR/TypeUtilities.h"
20	#include "mlir/Interfaces/FoldInterfaces.h"
21	#include "llvm/ADT/SmallVector.h"
22	#include "llvm/Support/ErrorHandling.h"
23	#include <numeric>
24	#include <optional>
25
26	using namespace mlir;
27
28	//===----------------------------------------------------------------------===//
29	// Operation
30	//===----------------------------------------------------------------------===//
31
32	/// Create a new Operation from operation state.
33	Operation *Operation::create(const OperationState &state) {
34	Operation *op =
35	create(location: state.location, name: state.name, resultTypes: state.types, operands: state.operands,
36	attributes: state.attributes.getDictionary(context: state.getContext()),
37	properties: state.properties, successors: state.successors, regions: state.regions);
38	if (LLVM_UNLIKELY(state.propertiesAttr)) {
39	assert(!state.properties);
40	LogicalResult result =
41	op->setPropertiesFromAttribute(attr: state.propertiesAttr,
42	/*diagnostic=*/emitError: nullptr);
43	assert(result.succeeded() && "invalid properties in op creation");
44	(void)result;
45	}
46	return op;
47	}
48
49	/// Create a new Operation with the specific fields.
50	Operation *Operation::create(Location location, OperationName name,
51	TypeRange resultTypes, ValueRange operands,
52	NamedAttrList &&attributes,
53	OpaqueProperties properties, BlockRange successors,
54	RegionRange regions) {
55	unsigned numRegions = regions.size();
56	Operation *op =
57	create(location, name, resultTypes, operands, attributes: std::move(attributes),
58	properties, successors, numRegions);
59	for (unsigned i = 0; i < numRegions; ++i)
60	if (regions[i])
61	op->getRegion(index: i).takeBody(other&: *regions[i]);
62	return op;
63	}
64
65	/// Create a new Operation with the specific fields.
66	Operation *Operation::create(Location location, OperationName name,
67	TypeRange resultTypes, ValueRange operands,
68	NamedAttrList &&attributes,
69	OpaqueProperties properties, BlockRange successors,
70	unsigned numRegions) {
71	// Populate default attributes.
72	name.populateDefaultAttrs(attrs&: attributes);
73
74	return create(location, name, resultTypes, operands,
75	attributes: attributes.getDictionary(context: location.getContext()), properties,
76	successors, numRegions);
77	}
78
79	/// Overload of create that takes an existing DictionaryAttr to avoid
80	/// unnecessarily uniquing a list of attributes.
81	Operation *Operation::create(Location location, OperationName name,
82	TypeRange resultTypes, ValueRange operands,
83	DictionaryAttr attributes,
84	OpaqueProperties properties, BlockRange successors,
85	unsigned numRegions) {
86	assert(llvm::all_of(resultTypes, [](Type t) { return t; }) &&
87	"unexpected null result type");
88
89	// We only need to allocate additional memory for a subset of results.
90	unsigned numTrailingResults = OpResult::getNumTrailing(numResults: resultTypes.size());
91	unsigned numInlineResults = OpResult::getNumInline(numResults: resultTypes.size());
92	unsigned numSuccessors = successors.size();
93	unsigned numOperands = operands.size();
94	unsigned numResults = resultTypes.size();
95	int opPropertiesAllocSize = llvm::alignTo<8>(Value: name.getOpPropertyByteSize());
96
97	// If the operation is known to have no operands, don't allocate an operand
98	// storage.
99	bool needsOperandStorage =
100	operands.empty() ? !name.hasTrait<OpTrait::ZeroOperands>() : true;
101
102	// Compute the byte size for the operation and the operand storage. This takes
103	// into account the size of the operation, its trailing objects, and its
104	// prefixed objects.
105	size_t byteSize =
106	totalSizeToAlloc<detail::OperandStorage, detail::OpProperties,
107	BlockOperand, Region, OpOperand>(
108	Counts: needsOperandStorage ? 1 : 0, Counts: opPropertiesAllocSize, Counts: numSuccessors,
109	Counts: numRegions, Counts: numOperands);
110	size_t prefixByteSize = llvm::alignTo(
111	Value: Operation::prefixAllocSize(numOutOfLineResults: numTrailingResults, numInlineResults),
112	Align: alignof(Operation));
113	char *mallocMem = reinterpret_cast<char *>(malloc(size: byteSize + prefixByteSize));
114	void *rawMem = mallocMem + prefixByteSize;
115
116	// Create the new Operation.
117	Operation *op = ::new (rawMem) Operation(
118	location, name, numResults, numSuccessors, numRegions,
119	opPropertiesAllocSize, attributes, properties, needsOperandStorage);
120
121	assert((numSuccessors == 0 || op->mightHaveTrait<OpTrait::IsTerminator>()) &&
122	"unexpected successors in a non-terminator operation");
123
124	// Initialize the results.
125	auto resultTypeIt = resultTypes.begin();
126	for (unsigned i = 0; i < numInlineResults; ++i, ++resultTypeIt)
127	new (op->getInlineOpResult(resultNumber: i)) detail::InlineOpResult(*resultTypeIt, i);
128	for (unsigned i = 0; i < numTrailingResults; ++i, ++resultTypeIt) {
129	new (op->getOutOfLineOpResult(resultNumber: i))
130	detail::OutOfLineOpResult(*resultTypeIt, i);
131	}
132
133	// Initialize the regions.
134	for (unsigned i = 0; i != numRegions; ++i)
135	new (&op->getRegion(index: i)) Region(op);
136
137	// Initialize the operands.
138	if (needsOperandStorage) {
139	new (&op->getOperandStorage()) detail::OperandStorage(
140	op, op->getTrailingObjects<OpOperand>(), operands);
141	}
142
143	// Initialize the successors.
144	auto blockOperands = op->getBlockOperands();
145	for (unsigned i = 0; i != numSuccessors; ++i)
146	new (&blockOperands[i]) BlockOperand(op, successors[i]);
147
148	// This must be done after properties are initalized.
149	op->setAttrs(attributes);
150
151	return op;
152	}
153
154	Operation::Operation(Location location, OperationName name, unsigned numResults,
155	unsigned numSuccessors, unsigned numRegions,
156	int fullPropertiesStorageSize, DictionaryAttr attributes,
157	OpaqueProperties properties, bool hasOperandStorage)
158	: location(location), numResults(numResults), numSuccs(numSuccessors),
159	numRegions(numRegions), hasOperandStorage(hasOperandStorage),
160	propertiesStorageSize((fullPropertiesStorageSize + 7) / 8), name(name) {
161	assert(attributes && "unexpected null attribute dictionary");
162	assert(fullPropertiesStorageSize <= propertiesCapacity &&
163	"Properties size overflow");
164	#ifndef NDEBUG
165	if (!getDialect() && !getContext()->allowsUnregisteredDialects())
166	llvm::report_fatal_error(
167	name.getStringRef() +
168	" created with unregistered dialect. If this is intended, please call "
169	"allowUnregisteredDialects() on the MLIRContext, or use "
170	"-allow-unregistered-dialect with the MLIR tool used.");
171	#endif
172	if (fullPropertiesStorageSize)
173	name.initOpProperties(storage: getPropertiesStorage(), init: properties);
174	}
175
176	// Operations are deleted through the destroy() member because they are
177	// allocated via malloc.
178	Operation::~Operation() {
179	assert(block == nullptr && "operation destroyed but still in a block");
180	#ifndef NDEBUG
181	if (!use_empty()) {
182	{
183	InFlightDiagnostic diag =
184	emitOpError("operation destroyed but still has uses");
185	for (Operation *user : getUsers())
186	diag.attachNote(user->getLoc()) << "- use: " << *user << "\n";
187	}
188	llvm::report_fatal_error("operation destroyed but still has uses");
189	}
190	#endif
191	// Explicitly run the destructors for the operands.
192	if (hasOperandStorage)
193	getOperandStorage().~OperandStorage();
194
195	// Explicitly run the destructors for the successors.
196	for (auto &successor : getBlockOperands())
197	successor.~BlockOperand();
198
199	// Explicitly destroy the regions.
200	for (auto &region : getRegions())
201	region.~Region();
202	if (propertiesStorageSize)
203	name.destroyOpProperties(properties: getPropertiesStorage());
204	}
205
206	/// Destroy this operation or one of its subclasses.
207	void Operation::destroy() {
208	// Operations may have additional prefixed allocation, which needs to be
209	// accounted for here when computing the address to free.
210	char *rawMem = reinterpret_cast<char *>(this) -
211	llvm::alignTo(Value: prefixAllocSize(), Align: alignof(Operation));
212	this->~Operation();
213	free(ptr: rawMem);
214	}
215
216	/// Return true if this operation is a proper ancestor of the `other`
217	/// operation.
218	bool Operation::isProperAncestor(Operation *other) {
219	while ((other = other->getParentOp()))
220	if (this == other)
221	return true;
222	return false;
223	}
224
225	/// Replace any uses of 'from' with 'to' within this operation.
226	void Operation::replaceUsesOfWith(Value from, Value to) {
227	if (from == to)
228	return;
229	for (auto &operand : getOpOperands())
230	if (operand.get() == from)
231	operand.set(to);
232	}
233
234	/// Replace the current operands of this operation with the ones provided in
235	/// 'operands'.
236	void Operation::setOperands(ValueRange operands) {
237	if (LLVM_LIKELY(hasOperandStorage))
238	return getOperandStorage().setOperands(owner: this, values: operands);
239	assert(operands.empty() && "setting operands without an operand storage");
240	}
241
242	/// Replace the operands beginning at 'start' and ending at 'start' + 'length'
243	/// with the ones provided in 'operands'. 'operands' may be smaller or larger
244	/// than the range pointed to by 'start'+'length'.
245	void Operation::setOperands(unsigned start, unsigned length,
246	ValueRange operands) {
247	assert((start + length) <= getNumOperands() &&
248	"invalid operand range specified");
249	if (LLVM_LIKELY(hasOperandStorage))
250	return getOperandStorage().setOperands(owner: this, start, length, operands);
251	assert(operands.empty() && "setting operands without an operand storage");
252	}
253
254	/// Insert the given operands into the operand list at the given 'index'.
255	void Operation::insertOperands(unsigned index, ValueRange operands) {
256	if (LLVM_LIKELY(hasOperandStorage))
257	return setOperands(start: index, /*length=*/0, operands);
258	assert(operands.empty() && "inserting operands without an operand storage");
259	}
260
261	//===----------------------------------------------------------------------===//
262	// Diagnostics
263	//===----------------------------------------------------------------------===//
264
265	/// Emit an error about fatal conditions with this operation, reporting up to
266	/// any diagnostic handlers that may be listening.
267	InFlightDiagnostic Operation::emitError(const Twine &message) {
268	InFlightDiagnostic diag = mlir::emitError(loc: getLoc(), message);
269	if (getContext()->shouldPrintOpOnDiagnostic()) {
270	diag.attachNote(noteLoc: getLoc())
271	.append(arg: "see current operation: ")
272	.appendOp(op&: *this, flags: OpPrintingFlags().printGenericOpForm());
273	}
274	return diag;
275	}
276
277	/// Emit a warning about this operation, reporting up to any diagnostic
278	/// handlers that may be listening.
279	InFlightDiagnostic Operation::emitWarning(const Twine &message) {
280	InFlightDiagnostic diag = mlir::emitWarning(loc: getLoc(), message);
281	if (getContext()->shouldPrintOpOnDiagnostic())
282	diag.attachNote(noteLoc: getLoc()) << "see current operation: " << *this;
283	return diag;
284	}
285
286	/// Emit a remark about this operation, reporting up to any diagnostic
287	/// handlers that may be listening.
288	InFlightDiagnostic Operation::emitRemark(const Twine &message) {
289	InFlightDiagnostic diag = mlir::emitRemark(loc: getLoc(), message);
290	if (getContext()->shouldPrintOpOnDiagnostic())
291	diag.attachNote(noteLoc: getLoc()) << "see current operation: " << *this;
292	return diag;
293	}
294
295	DictionaryAttr Operation::getAttrDictionary() {
296	if (getPropertiesStorageSize()) {
297	NamedAttrList attrsList = attrs;
298	getName().populateInherentAttrs(op: this, attrs&: attrsList);
299	return attrsList.getDictionary(context: getContext());
300	}
301	return attrs;
302	}
303
304	void Operation::setAttrs(DictionaryAttr newAttrs) {
305	assert(newAttrs && "expected valid attribute dictionary");
306	if (getPropertiesStorageSize()) {
307	// We're spliting the providing DictionaryAttr by removing the inherentAttr
308	// which will be stored in the properties.
309	SmallVector<NamedAttribute> discardableAttrs;
310	discardableAttrs.reserve(N: newAttrs.size());
311	for (NamedAttribute attr : newAttrs) {
312	if (getInherentAttr(name: attr.getName()))
313	setInherentAttr(name: attr.getName(), value: attr.getValue());
314	else
315	discardableAttrs.push_back(Elt: attr);
316	}
317	if (discardableAttrs.size() != newAttrs.size())
318	newAttrs = DictionaryAttr::get(context: getContext(), value: discardableAttrs);
319	}
320	attrs = newAttrs;
321	}
322	void Operation::setAttrs(ArrayRef<NamedAttribute> newAttrs) {
323	if (getPropertiesStorageSize()) {
324	// We're spliting the providing array of attributes by removing the
325	// inherentAttr which will be stored in the properties.
326	SmallVector<NamedAttribute> discardableAttrs;
327	discardableAttrs.reserve(N: newAttrs.size());
328	for (NamedAttribute attr : newAttrs) {
329	if (getInherentAttr(name: attr.getName()))
330	setInherentAttr(name: attr.getName(), value: attr.getValue());
331	else
332	discardableAttrs.push_back(Elt: attr);
333	}
334	attrs = DictionaryAttr::get(context: getContext(), value: discardableAttrs);
335	return;
336	}
337	attrs = DictionaryAttr::get(context: getContext(), value: newAttrs);
338	}
339
340	std::optional<Attribute> Operation::getInherentAttr(StringRef name) {
341	return getName().getInherentAttr(op: this, name);
342	}
343
344	void Operation::setInherentAttr(StringAttr name, Attribute value) {
345	getName().setInherentAttr(op: this, name, value);
346	}
347
348	Attribute Operation::getPropertiesAsAttribute() {
349	std::optional<RegisteredOperationName> info = getRegisteredInfo();
350	if (LLVM_UNLIKELY(!info))
351	return *getPropertiesStorage().as<Attribute *>();
352	return info->getOpPropertiesAsAttribute(op: this);
353	}
354	LogicalResult Operation::setPropertiesFromAttribute(
355	Attribute attr, function_ref<InFlightDiagnostic()> emitError) {
356	std::optional<RegisteredOperationName> info = getRegisteredInfo();
357	if (LLVM_UNLIKELY(!info)) {
358	*getPropertiesStorage().as<Attribute *>() = attr;
359	return success();
360	}
361	return info->setOpPropertiesFromAttribute(
362	opName: this->getName(), properties: this->getPropertiesStorage(), attr, emitError);
363	}
364
365	void Operation::copyProperties(OpaqueProperties rhs) {
366	name.copyOpProperties(lhs: getPropertiesStorage(), rhs);
367	}
368
369	llvm::hash_code Operation::hashProperties() {
370	return name.hashOpProperties(properties: getPropertiesStorage());
371	}
372
373	//===----------------------------------------------------------------------===//
374	// Operation Ordering
375	//===----------------------------------------------------------------------===//
376
377	constexpr unsigned Operation::kInvalidOrderIdx;
378	constexpr unsigned Operation::kOrderStride;
379
380	/// Given an operation 'other' that is within the same parent block, return
381	/// whether the current operation is before 'other' in the operation list
382	/// of the parent block.
383	/// Note: This function has an average complexity of O(1), but worst case may
384	/// take O(N) where N is the number of operations within the parent block.
385	bool Operation::isBeforeInBlock(Operation *other) {
386	assert(block && "Operations without parent blocks have no order.");
387	assert(other && other->block == block &&
388	"Expected other operation to have the same parent block.");
389	// If the order of the block is already invalid, directly recompute the
390	// parent.
391	if (!block->isOpOrderValid()) {
392	block->recomputeOpOrder();
393	} else {
394	// Update the order either operation if necessary.
395	updateOrderIfNecessary();
396	other->updateOrderIfNecessary();
397	}
398
399	return orderIndex < other->orderIndex;
400	}
401
402	/// Update the order index of this operation of this operation if necessary,
403	/// potentially recomputing the order of the parent block.
404	void Operation::updateOrderIfNecessary() {
405	assert(block && "expected valid parent");
406
407	// If the order is valid for this operation there is nothing to do.
408	if (hasValidOrder() || llvm::hasSingleElement(C&: *block))
409	return;
410	Operation *blockFront = &block->front();
411	Operation *blockBack = &block->back();
412
413	// This method is expected to only be invoked on blocks with more than one
414	// operation.
415	assert(blockFront != blockBack && "expected more than one operation");
416
417	// If the operation is at the end of the block.
418	if (this == blockBack) {
419	Operation *prevNode = getPrevNode();
420	if (!prevNode->hasValidOrder())
421	return block->recomputeOpOrder();
422
423	// Add the stride to the previous operation.
424	orderIndex = prevNode->orderIndex + kOrderStride;
425	return;
426	}
427
428	// If this is the first operation try to use the next operation to compute the
429	// ordering.
430	if (this == blockFront) {
431	Operation *nextNode = getNextNode();
432	if (!nextNode->hasValidOrder())
433	return block->recomputeOpOrder();
434	// There is no order to give this operation.
435	if (nextNode->orderIndex == 0)
436	return block->recomputeOpOrder();
437
438	// If we can't use the stride, just take the middle value left. This is safe
439	// because we know there is at least one valid index to assign to.
440	if (nextNode->orderIndex <= kOrderStride)
441	orderIndex = (nextNode->orderIndex / 2);
442	else
443	orderIndex = kOrderStride;
444	return;
445	}
446
447	// Otherwise, this operation is between two others. Place this operation in
448	// the middle of the previous and next if possible.
449	Operation *prevNode = getPrevNode(), *nextNode = getNextNode();
450	if (!prevNode->hasValidOrder() || !nextNode->hasValidOrder())
451	return block->recomputeOpOrder();
452	unsigned prevOrder = prevNode->orderIndex, nextOrder = nextNode->orderIndex;
453
454	// Check to see if there is a valid order between the two.
455	if (prevOrder + 1 == nextOrder)
456	return block->recomputeOpOrder();
457	orderIndex = prevOrder + ((nextOrder - prevOrder) / 2);
458	}
459
460	//===----------------------------------------------------------------------===//
461	// ilist_traits for Operation
462	//===----------------------------------------------------------------------===//
463
464	auto llvm::ilist_detail::SpecificNodeAccess<
465	typename llvm::ilist_detail::compute_node_options<
466	::mlir::Operation>::type>::getNodePtr(pointer n) -> node_type * {
467	return NodeAccess::getNodePtr<OptionsT>(N: n);
468	}
469
470	auto llvm::ilist_detail::SpecificNodeAccess<
471	typename llvm::ilist_detail::compute_node_options<
472	::mlir::Operation>::type>::getNodePtr(const_pointer n)
473	-> const node_type * {
474	return NodeAccess::getNodePtr<OptionsT>(N: n);
475	}
476
477	auto llvm::ilist_detail::SpecificNodeAccess<
478	typename llvm::ilist_detail::compute_node_options<
479	::mlir::Operation>::type>::getValuePtr(node_type *n) -> pointer {
480	return NodeAccess::getValuePtr<OptionsT>(N: n);
481	}
482
483	auto llvm::ilist_detail::SpecificNodeAccess<
484	typename llvm::ilist_detail::compute_node_options<
485	::mlir::Operation>::type>::getValuePtr(const node_type *n)
486	-> const_pointer {
487	return NodeAccess::getValuePtr<OptionsT>(N: n);
488	}
489
490	void llvm::ilist_traits<::mlir::Operation>::deleteNode(Operation *op) {
491	op->destroy();
492	}
493
494	Block *llvm::ilist_traits<::mlir::Operation>::getContainingBlock() {
495	size_t offset(size_t(&((Block *)nullptr->*Block::getSublistAccess(nullptr))));
496	iplist<Operation> *anchor(static_cast<iplist<Operation> *>(this));
497	return reinterpret_cast<Block *>(reinterpret_cast<char *>(anchor) - offset);
498	}
499
500	/// This is a trait method invoked when an operation is added to a block. We
501	/// keep the block pointer up to date.
502	void llvm::ilist_traits<::mlir::Operation>::addNodeToList(Operation *op) {
503	assert(!op->getBlock() && "already in an operation block!");
504	op->block = getContainingBlock();
505
506	// Invalidate the order on the operation.
507	op->orderIndex = Operation::kInvalidOrderIdx;
508	}
509
510	/// This is a trait method invoked when an operation is removed from a block.
511	/// We keep the block pointer up to date.
512	void llvm::ilist_traits<::mlir::Operation>::removeNodeFromList(Operation *op) {
513	assert(op->block && "not already in an operation block!");
514	op->block = nullptr;
515	}
516
517	/// This is a trait method invoked when an operation is moved from one block
518	/// to another. We keep the block pointer up to date.
519	void llvm::ilist_traits<::mlir::Operation>::transferNodesFromList(
520	ilist_traits<Operation> &otherList, op_iterator first, op_iterator last) {
521	Block *curParent = getContainingBlock();
522
523	// Invalidate the ordering of the parent block.
524	curParent->invalidateOpOrder();
525
526	// If we are transferring operations within the same block, the block
527	// pointer doesn't need to be updated.
528	if (curParent == otherList.getContainingBlock())
529	return;
530
531	// Update the 'block' member of each operation.
532	for (; first != last; ++first)
533	first->block = curParent;
534	}
535
536	/// Remove this operation (and its descendants) from its Block and delete
537	/// all of them.
538	void Operation::erase() {
539	if (auto *parent = getBlock())
540	parent->getOperations().erase(IT: this);
541	else
542	destroy();
543	}
544
545	/// Remove the operation from its parent block, but don't delete it.
546	void Operation::remove() {
547	if (Block *parent = getBlock())
548	parent->getOperations().remove(IT: this);
549	}
550
551	/// Unlink this operation from its current block and insert it right before
552	/// `existingOp` which may be in the same or another block in the same
553	/// function.
554	void Operation::moveBefore(Operation *existingOp) {
555	moveBefore(block: existingOp->getBlock(), iterator: existingOp->getIterator());
556	}
557
558	/// Unlink this operation from its current basic block and insert it right
559	/// before `iterator` in the specified basic block.
560	void Operation::moveBefore(Block *block,
561	llvm::iplist<Operation>::iterator iterator) {
562	assert(getBlock() &&
563	"cannot move an operation that isn't contained in a block");
564	block->getOperations().splice(where: iterator, L2&: getBlock()->getOperations(),
565	first: getIterator());
566	}
567
568	/// Unlink this operation from its current block and insert it right after
569	/// `existingOp` which may be in the same or another block in the same function.
570	void Operation::moveAfter(Operation *existingOp) {
571	moveAfter(block: existingOp->getBlock(), iterator: existingOp->getIterator());
572	}
573
574	/// Unlink this operation from its current block and insert it right after
575	/// `iterator` in the specified block.
576	void Operation::moveAfter(Block *block,
577	llvm::iplist<Operation>::iterator iterator) {
578	assert(iterator != block->end() && "cannot move after end of block");
579	moveBefore(block, iterator: std::next(x: iterator));
580	}
581
582	/// This drops all operand uses from this operation, which is an essential
583	/// step in breaking cyclic dependences between references when they are to
584	/// be deleted.
585	void Operation::dropAllReferences() {
586	for (auto &op : getOpOperands())
587	op.drop();
588
589	for (auto &region : getRegions())
590	region.dropAllReferences();
591
592	for (auto &dest : getBlockOperands())
593	dest.drop();
594	}
595
596	/// This drops all uses of any values defined by this operation or its nested
597	/// regions, wherever they are located.
598	void Operation::dropAllDefinedValueUses() {
599	dropAllUses();
600
601	for (auto &region : getRegions())
602	for (auto &block : region)
603	block.dropAllDefinedValueUses();
604	}
605
606	void Operation::setSuccessor(Block *block, unsigned index) {
607	assert(index < getNumSuccessors());
608	getBlockOperands()[index].set(block);
609	}
610
611	#ifndef NDEBUG
612	/// Assert that the folded results (in case of values) have the same type as
613	/// the results of the given op.
614	static void checkFoldResultTypes(Operation *op,
615	SmallVectorImpl<OpFoldResult> &results) {
616	if (results.empty())
617	return;
618
619	for (auto [ofr, opResult] : llvm::zip_equal(results, op->getResults())) {
620	if (auto value = dyn_cast<Value>(ofr)) {
621	if (value.getType() != opResult.getType()) {
622	op->emitOpError() << "folder produced a value of incorrect type: "
623	<< value.getType()
624	<< ", expected: " << opResult.getType();
625	assert(false && "incorrect fold result type");
626	}
627	}
628	}
629	}
630	#endif // NDEBUG
631
632	/// Attempt to fold this operation using the Op's registered foldHook.
633	LogicalResult Operation::fold(ArrayRef<Attribute> operands,
634	SmallVectorImpl<OpFoldResult> &results) {
635	// If we have a registered operation definition matching this one, use it to
636	// try to constant fold the operation.
637	if (succeeded(Result: name.foldHook(op: this, operands, results))) {
638	#ifndef NDEBUG
639	checkFoldResultTypes(this, results);
640	#endif // NDEBUG
641	return success();
642	}
643
644	// Otherwise, fall back on the dialect hook to handle it.
645	Dialect *dialect = getDialect();
646	if (!dialect)
647	return failure();
648
649	auto *interface = dyn_cast<DialectFoldInterface>(Val: dialect);
650	if (!interface)
651	return failure();
652
653	LogicalResult status = interface->fold(op: this, operands, results);
654	#ifndef NDEBUG
655	if (succeeded(status))
656	checkFoldResultTypes(this, results);
657	#endif // NDEBUG
658	return status;
659	}
660
661	LogicalResult Operation::fold(SmallVectorImpl<OpFoldResult> &results) {
662	// Check if any operands are constants.
663	SmallVector<Attribute> constants;
664	constants.assign(NumElts: getNumOperands(), Elt: Attribute());
665	for (unsigned i = 0, e = getNumOperands(); i != e; ++i)
666	matchPattern(value: getOperand(idx: i), pattern: m_Constant(bind_value: &constants[i]));
667	return fold(operands: constants, results);
668	}
669
670	/// Emit an error with the op name prefixed, like "'dim' op " which is
671	/// convenient for verifiers.
672	InFlightDiagnostic Operation::emitOpError(const Twine &message) {
673	return emitError() << "'" << getName() << "' op " << message;
674	}
675
676	//===----------------------------------------------------------------------===//
677	// Operation Cloning
678	//===----------------------------------------------------------------------===//
679
680	Operation::CloneOptions::CloneOptions()
681	: cloneRegionsFlag(false), cloneOperandsFlag(false) {}
682
683	Operation::CloneOptions::CloneOptions(bool cloneRegions, bool cloneOperands)
684	: cloneRegionsFlag(cloneRegions), cloneOperandsFlag(cloneOperands) {}
685
686	Operation::CloneOptions Operation::CloneOptions::all() {
687	return CloneOptions().cloneRegions().cloneOperands();
688	}
689
690	Operation::CloneOptions &Operation::CloneOptions::cloneRegions(bool enable) {
691	cloneRegionsFlag = enable;
692	return *this;
693	}
694
695	Operation::CloneOptions &Operation::CloneOptions::cloneOperands(bool enable) {
696	cloneOperandsFlag = enable;
697	return *this;
698	}
699
700	/// Create a deep copy of this operation but keep the operation regions empty.
701	/// Operands are remapped using `mapper` (if present), and `mapper` is updated
702	/// to contain the results. The `mapResults` flag specifies whether the results
703	/// of the cloned operation should be added to the map.
704	Operation *Operation::cloneWithoutRegions(IRMapping &mapper) {
705	return clone(mapper, options: CloneOptions::all().cloneRegions(enable: false));
706	}
707
708	Operation *Operation::cloneWithoutRegions() {
709	IRMapping mapper;
710	return cloneWithoutRegions(mapper);
711	}
712
713	/// Create a deep copy of this operation, remapping any operands that use
714	/// values outside of the operation using the map that is provided (leaving
715	/// them alone if no entry is present). Replaces references to cloned
716	/// sub-operations to the corresponding operation that is copied, and adds
717	/// those mappings to the map.
718	Operation *Operation::clone(IRMapping &mapper, CloneOptions options) {
719	SmallVector<Value, 8> operands;
720	SmallVector<Block *, 2> successors;
721
722	// Remap the operands.
723	if (options.shouldCloneOperands()) {
724	operands.reserve(N: getNumOperands());
725	for (auto opValue : getOperands())
726	operands.push_back(Elt: mapper.lookupOrDefault(from: opValue));
727	}
728
729	// Remap the successors.
730	successors.reserve(N: getNumSuccessors());
731	for (Block *successor : getSuccessors())
732	successors.push_back(Elt: mapper.lookupOrDefault(from: successor));
733
734	// Create the new operation.
735	auto *newOp = create(location: getLoc(), name: getName(), resultTypes: getResultTypes(), operands, attributes: attrs,
736	properties: getPropertiesStorage(), successors, numRegions: getNumRegions());
737	mapper.map(from: this, to: newOp);
738
739	// Clone the regions.
740	if (options.shouldCloneRegions()) {
741	for (unsigned i = 0; i != numRegions; ++i)
742	getRegion(index: i).cloneInto(dest: &newOp->getRegion(index: i), mapper);
743	}
744
745	// Remember the mapping of any results.
746	for (unsigned i = 0, e = getNumResults(); i != e; ++i)
747	mapper.map(from: getResult(idx: i), to: newOp->getResult(idx: i));
748
749	return newOp;
750	}
751
752	Operation *Operation::clone(CloneOptions options) {
753	IRMapping mapper;
754	return clone(mapper, options);
755	}
756
757	//===----------------------------------------------------------------------===//
758	// OpState trait class.
759	//===----------------------------------------------------------------------===//
760
761	// The fallback for the parser is to try for a dialect operation parser.
762	// Otherwise, reject the custom assembly form.
763	ParseResult OpState::parse(OpAsmParser &parser, OperationState &result) {
764	if (auto parseFn = result.name.getDialect()->getParseOperationHook(
765	opName: result.name.getStringRef()))
766	return (*parseFn)(parser, result);
767	return parser.emitError(loc: parser.getNameLoc(), message: "has no custom assembly form");
768	}
769
770	// The fallback for the printer is to try for a dialect operation printer.
771	// Otherwise, it prints the generic form.
772	void OpState::print(Operation *op, OpAsmPrinter &p, StringRef defaultDialect) {
773	if (auto printFn = op->getDialect()->getOperationPrinter(op)) {
774	printOpName(op, p, defaultDialect);
775	printFn(op, p);
776	} else {
777	p.printGenericOp(op);
778	}
779	}
780
781	/// Print an operation name, eliding the dialect prefix if necessary and doesn't
782	/// lead to ambiguities.
783	void OpState::printOpName(Operation *op, OpAsmPrinter &p,
784	StringRef defaultDialect) {
785	StringRef name = op->getName().getStringRef();
786	if (name.starts_with(Prefix: (defaultDialect + ".").str()) && name.count(C: '.') == 1)
787	name = name.drop_front(N: defaultDialect.size() + 1);
788	p.getStream() << name;
789	}
790
791	/// Parse properties as a Attribute.
792	ParseResult OpState::genericParseProperties(OpAsmParser &parser,
793	Attribute &result) {
794	if (succeeded(Result: parser.parseOptionalLess())) { // The less is optional.
795	if (parser.parseAttribute(result) || parser.parseGreater())
796	return failure();
797	}
798	return success();
799	}
800
801	/// Print the properties as a Attribute with names not included within
802	/// 'elidedProps'
803	void OpState::genericPrintProperties(OpAsmPrinter &p, Attribute properties,
804	ArrayRef<StringRef> elidedProps) {
805	if (!properties)
806	return;
807	auto dictAttr = dyn_cast_or_null<::mlir::DictionaryAttr>(Val&: properties);
808	if (dictAttr && !elidedProps.empty()) {
809	ArrayRef<NamedAttribute> attrs = dictAttr.getValue();
810	llvm::SmallDenseSet<StringRef> elidedAttrsSet(elidedProps.begin(),
811	elidedProps.end());
812	auto filteredAttrs =
813	llvm::make_filter_range(Range&: attrs, Pred: [&](NamedAttribute attr) {
814	return !elidedAttrsSet.contains(V: attr.getName().strref());
815	});
816	if (!filteredAttrs.empty()) {
817	p << "<{";
818	interleaveComma(c: filteredAttrs, os&: p, each_fn: [&](NamedAttribute attr) {
819	p.printNamedAttribute(attr);
820	});
821	p << "}>";
822	}
823	} else {
824	p << "<" << properties << ">";
825	}
826	}
827
828	/// Emit an error about fatal conditions with this operation, reporting up to
829	/// any diagnostic handlers that may be listening.
830	InFlightDiagnostic OpState::emitError(const Twine &message) {
831	return getOperation()->emitError(message);
832	}
833
834	/// Emit an error with the op name prefixed, like "'dim' op " which is
835	/// convenient for verifiers.
836	InFlightDiagnostic OpState::emitOpError(const Twine &message) {
837	return getOperation()->emitOpError(message);
838	}
839
840	/// Emit a warning about this operation, reporting up to any diagnostic
841	/// handlers that may be listening.
842	InFlightDiagnostic OpState::emitWarning(const Twine &message) {
843	return getOperation()->emitWarning(message);
844	}
845
846	/// Emit a remark about this operation, reporting up to any diagnostic
847	/// handlers that may be listening.
848	InFlightDiagnostic OpState::emitRemark(const Twine &message) {
849	return getOperation()->emitRemark(message);
850	}
851
852	//===----------------------------------------------------------------------===//
853	// Op Trait implementations
854	//===----------------------------------------------------------------------===//
855
856	LogicalResult
857	OpTrait::impl::foldCommutative(Operation *op, ArrayRef<Attribute> operands,
858	SmallVectorImpl<OpFoldResult> &results) {
859	// Nothing to fold if there are not at least 2 operands.
860	if (op->getNumOperands() < 2)
861	return failure();
862	// Move all constant operands to the end.
863	OpOperand *operandsBegin = op->getOpOperands().begin();
864	auto isNonConstant = [&](OpOperand &o) {
865	return !static_cast<bool>(operands[std::distance(first: operandsBegin, last: &o)]);
866	};
867	auto *firstConstantIt = llvm::find_if_not(Range: op->getOpOperands(), P: isNonConstant);
868	auto *newConstantIt = std::stable_partition(
869	first: firstConstantIt, last: op->getOpOperands().end(), pred: isNonConstant);
870	// Return success if the op was modified.
871	return success(IsSuccess: firstConstantIt != newConstantIt);
872	}
873
874	OpFoldResult OpTrait::impl::foldIdempotent(Operation *op) {
875	if (op->getNumOperands() == 1) {
876	auto *argumentOp = op->getOperand(idx: 0).getDefiningOp();
877	if (argumentOp && op->getName() == argumentOp->getName()) {
878	// Replace the outer operation output with the inner operation.
879	return op->getOperand(idx: 0);
880	}
881	} else if (op->getOperand(idx: 0) == op->getOperand(idx: 1)) {
882	return op->getOperand(idx: 0);
883	}
884
885	return {};
886	}
887
888	OpFoldResult OpTrait::impl::foldInvolution(Operation *op) {
889	auto *argumentOp = op->getOperand(idx: 0).getDefiningOp();
890	if (argumentOp && op->getName() == argumentOp->getName()) {
891	// Replace the outer involutions output with inner's input.
892	return argumentOp->getOperand(idx: 0);
893	}
894
895	return {};
896	}
897
898	LogicalResult OpTrait::impl::verifyZeroOperands(Operation *op) {
899	if (op->getNumOperands() != 0)
900	return op->emitOpError() << "requires zero operands";
901	return success();
902	}
903
904	LogicalResult OpTrait::impl::verifyOneOperand(Operation *op) {
905	if (op->getNumOperands() != 1)
906	return op->emitOpError() << "requires a single operand";
907	return success();
908	}
909
910	LogicalResult OpTrait::impl::verifyNOperands(Operation *op,
911	unsigned numOperands) {
912	if (op->getNumOperands() != numOperands) {
913	return op->emitOpError() << "expected " << numOperands
914	<< " operands, but found " << op->getNumOperands();
915	}
916	return success();
917	}
918
919	LogicalResult OpTrait::impl::verifyAtLeastNOperands(Operation *op,
920	unsigned numOperands) {
921	if (op->getNumOperands() < numOperands)
922	return op->emitOpError()
923	<< "expected " << numOperands << " or more operands, but found "
924	<< op->getNumOperands();
925	return success();
926	}
927
928	/// If this is a vector type, or a tensor type, return the scalar element type
929	/// that it is built around, otherwise return the type unmodified.
930	static Type getTensorOrVectorElementType(Type type) {
931	if (auto vec = llvm::dyn_cast<VectorType>(Val&: type))
932	return vec.getElementType();
933
934	// Look through tensor<vector<...>> to find the underlying element type.
935	if (auto tensor = llvm::dyn_cast<TensorType>(Val&: type))
936	return getTensorOrVectorElementType(type: tensor.getElementType());
937	return type;
938	}
939
940	LogicalResult OpTrait::impl::verifyIsIdempotent(Operation *op) {
941	// FIXME: Add back check for no side effects on operation.
942	// Currently adding it would cause the shared library build
943	// to fail since there would be a dependency of IR on SideEffectInterfaces
944	// which is cyclical.
945	return success();
946	}
947
948	LogicalResult OpTrait::impl::verifyIsInvolution(Operation *op) {
949	// FIXME: Add back check for no side effects on operation.
950	// Currently adding it would cause the shared library build
951	// to fail since there would be a dependency of IR on SideEffectInterfaces
952	// which is cyclical.
953	return success();
954	}
955
956	LogicalResult
957	OpTrait::impl::verifyOperandsAreSignlessIntegerLike(Operation *op) {
958	for (auto opType : op->getOperandTypes()) {
959	auto type = getTensorOrVectorElementType(type: opType);
960	if (!type.isSignlessIntOrIndex())
961	return op->emitOpError() << "requires an integer or index type";
962	}
963	return success();
964	}
965
966	LogicalResult OpTrait::impl::verifyOperandsAreFloatLike(Operation *op) {
967	for (auto opType : op->getOperandTypes()) {
968	auto type = getTensorOrVectorElementType(type: opType);
969	if (!llvm::isa<FloatType>(Val: type))
970	return op->emitOpError(message: "requires a float type");
971	}
972	return success();
973	}
974
975	LogicalResult OpTrait::impl::verifySameTypeOperands(Operation *op) {
976	// Zero or one operand always have the "same" type.
977	unsigned nOperands = op->getNumOperands();
978	if (nOperands < 2)
979	return success();
980
981	auto type = op->getOperand(idx: 0).getType();
982	for (auto opType : llvm::drop_begin(RangeOrContainer: op->getOperandTypes(), N: 1))
983	if (opType != type)
984	return op->emitOpError() << "requires all operands to have the same type";
985	return success();
986	}
987
988	LogicalResult OpTrait::impl::verifyZeroRegions(Operation *op) {
989	if (op->getNumRegions() != 0)
990	return op->emitOpError() << "requires zero regions";
991	return success();
992	}
993
994	LogicalResult OpTrait::impl::verifyOneRegion(Operation *op) {
995	if (op->getNumRegions() != 1)
996	return op->emitOpError() << "requires one region";
997	return success();
998	}
999
1000	LogicalResult OpTrait::impl::verifyNRegions(Operation *op,
1001	unsigned numRegions) {
1002	if (op->getNumRegions() != numRegions)
1003	return op->emitOpError() << "expected " << numRegions << " regions";
1004	return success();
1005	}
1006
1007	LogicalResult OpTrait::impl::verifyAtLeastNRegions(Operation *op,
1008	unsigned numRegions) {
1009	if (op->getNumRegions() < numRegions)
1010	return op->emitOpError() << "expected " << numRegions << " or more regions";
1011	return success();
1012	}
1013
1014	LogicalResult OpTrait::impl::verifyZeroResults(Operation *op) {
1015	if (op->getNumResults() != 0)
1016	return op->emitOpError() << "requires zero results";
1017	return success();
1018	}
1019
1020	LogicalResult OpTrait::impl::verifyOneResult(Operation *op) {
1021	if (op->getNumResults() != 1)
1022	return op->emitOpError() << "requires one result";
1023	return success();
1024	}
1025
1026	LogicalResult OpTrait::impl::verifyNResults(Operation *op,
1027	unsigned numOperands) {
1028	if (op->getNumResults() != numOperands)
1029	return op->emitOpError() << "expected " << numOperands << " results";
1030	return success();
1031	}
1032
1033	LogicalResult OpTrait::impl::verifyAtLeastNResults(Operation *op,
1034	unsigned numOperands) {
1035	if (op->getNumResults() < numOperands)
1036	return op->emitOpError()
1037	<< "expected " << numOperands << " or more results";
1038	return success();
1039	}
1040
1041	LogicalResult OpTrait::impl::verifySameOperandsShape(Operation *op) {
1042	if (failed(Result: verifyAtLeastNOperands(op, numOperands: 1)))
1043	return failure();
1044
1045	if (failed(Result: verifyCompatibleShapes(types: op->getOperandTypes())))
1046	return op->emitOpError() << "requires the same shape for all operands";
1047
1048	return success();
1049	}
1050
1051	LogicalResult OpTrait::impl::verifySameOperandsAndResultShape(Operation *op) {
1052	if (failed(Result: verifyAtLeastNOperands(op, numOperands: 1)) ||
1053	failed(Result: verifyAtLeastNResults(op, numOperands: 1)))
1054	return failure();
1055
1056	SmallVector<Type, 8> types(op->getOperandTypes());
1057	types.append(RHS: llvm::to_vector<4>(Range: op->getResultTypes()));
1058
1059	if (failed(Result: verifyCompatibleShapes(types)))
1060	return op->emitOpError()
1061	<< "requires the same shape for all operands and results";
1062
1063	return success();
1064	}
1065
1066	LogicalResult OpTrait::impl::verifySameOperandsElementType(Operation *op) {
1067	if (failed(Result: verifyAtLeastNOperands(op, numOperands: 1)))
1068	return failure();
1069	auto elementType = getElementTypeOrSelf(val: op->getOperand(idx: 0));
1070
1071	for (auto operand : llvm::drop_begin(RangeOrContainer: op->getOperands(), N: 1)) {
1072	if (getElementTypeOrSelf(val: operand) != elementType)
1073	return op->emitOpError(message: "requires the same element type for all operands");
1074	}
1075
1076	return success();
1077	}
1078
1079	LogicalResult
1080	OpTrait::impl::verifySameOperandsAndResultElementType(Operation *op) {
1081	if (failed(Result: verifyAtLeastNOperands(op, numOperands: 1)) ||
1082	failed(Result: verifyAtLeastNResults(op, numOperands: 1)))
1083	return failure();
1084
1085	auto elementType = getElementTypeOrSelf(val: op->getResult(idx: 0));
1086
1087	// Verify result element type matches first result's element type.
1088	for (auto result : llvm::drop_begin(RangeOrContainer: op->getResults(), N: 1)) {
1089	if (getElementTypeOrSelf(val: result) != elementType)
1090	return op->emitOpError(
1091	message: "requires the same element type for all operands and results");
1092	}
1093
1094	// Verify operand's element type matches first result's element type.
1095	for (auto operand : op->getOperands()) {
1096	if (getElementTypeOrSelf(val: operand) != elementType)
1097	return op->emitOpError(
1098	message: "requires the same element type for all operands and results");
1099	}
1100
1101	return success();
1102	}
1103
1104	LogicalResult OpTrait::impl::verifySameOperandsAndResultType(Operation *op) {
1105	if (failed(Result: verifyAtLeastNOperands(op, numOperands: 1)) ||
1106	failed(Result: verifyAtLeastNResults(op, numOperands: 1)))
1107	return failure();
1108
1109	auto type = op->getResult(idx: 0).getType();
1110	auto elementType = getElementTypeOrSelf(type);
1111	Attribute encoding = nullptr;
1112	if (auto rankedType = dyn_cast<RankedTensorType>(Val&: type))
1113	encoding = rankedType.getEncoding();
1114	for (auto resultType : llvm::drop_begin(RangeOrContainer: op->getResultTypes())) {
1115	if (getElementTypeOrSelf(type: resultType) != elementType ||
1116	failed(Result: verifyCompatibleShape(type1: resultType, type2: type)))
1117	return op->emitOpError()
1118	<< "requires the same type for all operands and results";
1119	if (encoding)
1120	if (auto rankedType = dyn_cast<RankedTensorType>(Val&: resultType);
1121	encoding != rankedType.getEncoding())
1122	return op->emitOpError()
1123	<< "requires the same encoding for all operands and results";
1124	}
1125	for (auto opType : op->getOperandTypes()) {
1126	if (getElementTypeOrSelf(type: opType) != elementType ||
1127	failed(Result: verifyCompatibleShape(type1: opType, type2: type)))
1128	return op->emitOpError()
1129	<< "requires the same type for all operands and results";
1130	if (encoding)
1131	if (auto rankedType = dyn_cast<RankedTensorType>(Val&: opType);
1132	encoding != rankedType.getEncoding())
1133	return op->emitOpError()
1134	<< "requires the same encoding for all operands and results";
1135	}
1136	return success();
1137	}
1138
1139	LogicalResult OpTrait::impl::verifySameOperandsAndResultRank(Operation *op) {
1140	if (failed(Result: verifyAtLeastNOperands(op, numOperands: 1)))
1141	return failure();
1142
1143	// delegate function that returns true if type is a shaped type with known
1144	// rank
1145	auto hasRank = [](const Type type) {
1146	if (auto shapedType = dyn_cast<ShapedType>(Val: type))
1147	return shapedType.hasRank();
1148
1149	return false;
1150	};
1151
1152	auto rankedOperandTypes =
1153	llvm::make_filter_range(Range: op->getOperandTypes(), Pred: hasRank);
1154	auto rankedResultTypes =
1155	llvm::make_filter_range(Range: op->getResultTypes(), Pred: hasRank);
1156
1157	// If all operands and results are unranked, then no further verification.
1158	if (rankedOperandTypes.empty() && rankedResultTypes.empty())
1159	return success();
1160
1161	// delegate function that returns rank of shaped type with known rank
1162	auto getRank = [](const Type type) {
1163	return cast<ShapedType>(Val: type).getRank();
1164	};
1165
1166	auto rank = !rankedOperandTypes.empty() ? getRank(*rankedOperandTypes.begin())
1167	: getRank(*rankedResultTypes.begin());
1168
1169	for (const auto type : rankedOperandTypes) {
1170	if (rank != getRank(type)) {
1171	return op->emitOpError(message: "operands don't have matching ranks");
1172	}
1173	}
1174
1175	for (const auto type : rankedResultTypes) {
1176	if (rank != getRank(type)) {
1177	return op->emitOpError(message: "result type has different rank than operands");
1178	}
1179	}
1180
1181	return success();
1182	}
1183
1184	LogicalResult OpTrait::impl::verifyIsTerminator(Operation *op) {
1185	Block *block = op->getBlock();
1186	// Verify that the operation is at the end of the respective parent block.
1187	if (!block || &block->back() != op)
1188	return op->emitOpError(message: "must be the last operation in the parent block");
1189	return success();
1190	}
1191
1192	static LogicalResult verifyTerminatorSuccessors(Operation *op) {
1193	auto *parent = op->getParentRegion();
1194
1195	// Verify that the operands lines up with the BB arguments in the successor.
1196	for (Block *succ : op->getSuccessors())
1197	if (succ->getParent() != parent)
1198	return op->emitError(message: "reference to block defined in another region");
1199	return success();
1200	}
1201
1202	LogicalResult OpTrait::impl::verifyZeroSuccessors(Operation *op) {
1203	if (op->getNumSuccessors() != 0) {
1204	return op->emitOpError(message: "requires 0 successors but found ")
1205	<< op->getNumSuccessors();
1206	}
1207	return success();
1208	}
1209
1210	LogicalResult OpTrait::impl::verifyOneSuccessor(Operation *op) {
1211	if (op->getNumSuccessors() != 1) {
1212	return op->emitOpError(message: "requires 1 successor but found ")
1213	<< op->getNumSuccessors();
1214	}
1215	return verifyTerminatorSuccessors(op);
1216	}
1217	LogicalResult OpTrait::impl::verifyNSuccessors(Operation *op,
1218	unsigned numSuccessors) {
1219	if (op->getNumSuccessors() != numSuccessors) {
1220	return op->emitOpError(message: "requires ")
1221	<< numSuccessors << " successors but found "
1222	<< op->getNumSuccessors();
1223	}
1224	return verifyTerminatorSuccessors(op);
1225	}
1226	LogicalResult OpTrait::impl::verifyAtLeastNSuccessors(Operation *op,
1227	unsigned numSuccessors) {
1228	if (op->getNumSuccessors() < numSuccessors) {
1229	return op->emitOpError(message: "requires at least ")
1230	<< numSuccessors << " successors but found "
1231	<< op->getNumSuccessors();
1232	}
1233	return verifyTerminatorSuccessors(op);
1234	}
1235
1236	LogicalResult OpTrait::impl::verifyResultsAreBoolLike(Operation *op) {
1237	for (auto resultType : op->getResultTypes()) {
1238	auto elementType = getTensorOrVectorElementType(type: resultType);
1239	bool isBoolType = elementType.isInteger(width: 1);
1240	if (!isBoolType)
1241	return op->emitOpError() << "requires a bool result type";
1242	}
1243
1244	return success();
1245	}
1246
1247	LogicalResult OpTrait::impl::verifyResultsAreFloatLike(Operation *op) {
1248	for (auto resultType : op->getResultTypes())
1249	if (!llvm::isa<FloatType>(Val: getTensorOrVectorElementType(type: resultType)))
1250	return op->emitOpError() << "requires a floating point type";
1251
1252	return success();
1253	}
1254
1255	LogicalResult
1256	OpTrait::impl::verifyResultsAreSignlessIntegerLike(Operation *op) {
1257	for (auto resultType : op->getResultTypes())
1258	if (!getTensorOrVectorElementType(type: resultType).isSignlessIntOrIndex())
1259	return op->emitOpError() << "requires an integer or index type";
1260	return success();
1261	}
1262
1263	LogicalResult OpTrait::impl::verifyValueSizeAttr(Operation *op,
1264	StringRef attrName,
1265	StringRef valueGroupName,
1266	size_t expectedCount) {
1267	auto sizeAttr = op->getAttrOfType<DenseI32ArrayAttr>(name: attrName);
1268	if (!sizeAttr)
1269	return op->emitOpError(message: "requires dense i32 array attribute '")
1270	<< attrName << "'";
1271
1272	ArrayRef<int32_t> sizes = sizeAttr.asArrayRef();
1273	if (llvm::any_of(Range&: sizes, P: [](int32_t element) { return element < 0; }))
1274	return op->emitOpError(message: "'")
1275	<< attrName << "' attribute cannot have negative elements";
1276
1277	size_t totalCount =
1278	std::accumulate(first: sizes.begin(), last: sizes.end(), init: 0,
1279	binary_op: [](unsigned all, int32_t one) { return all + one; });
1280
1281	if (totalCount != expectedCount)
1282	return op->emitOpError()
1283	<< valueGroupName << " count (" << expectedCount
1284	<< ") does not match with the total size (" << totalCount
1285	<< ") specified in attribute '" << attrName << "'";
1286	return success();
1287	}
1288
1289	LogicalResult OpTrait::impl::verifyOperandSizeAttr(Operation *op,
1290	StringRef attrName) {
1291	return verifyValueSizeAttr(op, attrName, valueGroupName: "operand", expectedCount: op->getNumOperands());
1292	}
1293
1294	LogicalResult OpTrait::impl::verifyResultSizeAttr(Operation *op,
1295	StringRef attrName) {
1296	return verifyValueSizeAttr(op, attrName, valueGroupName: "result", expectedCount: op->getNumResults());
1297	}
1298
1299	LogicalResult OpTrait::impl::verifyNoRegionArguments(Operation *op) {
1300	for (Region &region : op->getRegions()) {
1301	if (region.empty())
1302	continue;
1303
1304	if (region.getNumArguments() != 0) {
1305	if (op->getNumRegions() > 1)
1306	return op->emitOpError(message: "region #")
1307	<< region.getRegionNumber() << " should have no arguments";
1308	return op->emitOpError(message: "region should have no arguments");
1309	}
1310	}
1311	return success();
1312	}
1313
1314	LogicalResult OpTrait::impl::verifyElementwise(Operation *op) {
1315	auto isMappableType = llvm::IsaPred<VectorType, TensorType>;
1316	auto resultMappableTypes =
1317	llvm::filter_to_vector<1>(C: op->getResultTypes(), Pred&: isMappableType);
1318	auto operandMappableTypes =
1319	llvm::filter_to_vector<2>(C: op->getOperandTypes(), Pred&: isMappableType);
1320
1321	// If the op only has scalar operand/result types, then we have nothing to
1322	// check.
1323	if (resultMappableTypes.empty() && operandMappableTypes.empty())
1324	return success();
1325
1326	if (!resultMappableTypes.empty() && operandMappableTypes.empty())
1327	return op->emitOpError(message: "if a result is non-scalar, then at least one "
1328	"operand must be non-scalar");
1329
1330	assert(!operandMappableTypes.empty());
1331
1332	if (resultMappableTypes.empty())
1333	return op->emitOpError(message: "if an operand is non-scalar, then there must be at "
1334	"least one non-scalar result");
1335
1336	if (resultMappableTypes.size() != op->getNumResults())
1337	return op->emitOpError(
1338	message: "if an operand is non-scalar, then all results must be non-scalar");
1339
1340	SmallVector<Type, 4> types = llvm::to_vector<2>(
1341	Range: llvm::concat<Type>(Ranges&: operandMappableTypes, Ranges&: resultMappableTypes));
1342	TypeID expectedBaseTy = types.front().getTypeID();
1343	if (!llvm::all_of(Range&: types,
1344	P: [&](Type t) { return t.getTypeID() == expectedBaseTy; }) ||
1345	failed(Result: verifyCompatibleShapes(types))) {
1346	return op->emitOpError() << "all non-scalar operands/results must have the "
1347	"same shape and base type";
1348	}
1349
1350	return success();
1351	}
1352
1353	/// Check for any values used by operations regions attached to the
1354	/// specified "IsIsolatedFromAbove" operation defined outside of it.
1355	LogicalResult OpTrait::impl::verifyIsIsolatedFromAbove(Operation *isolatedOp) {
1356	assert(isolatedOp->hasTrait<OpTrait::IsIsolatedFromAbove>() &&
1357	"Intended to check IsolatedFromAbove ops");
1358
1359	// List of regions to analyze. Each region is processed independently, with
1360	// respect to the common `limit` region, so we can look at them in any order.
1361	// Therefore, use a simple vector and push/pop back the current region.
1362	SmallVector<Region *, 8> pendingRegions;
1363	for (auto &region : isolatedOp->getRegions()) {
1364	pendingRegions.push_back(Elt: &region);
1365
1366	// Traverse all operations in the region.
1367	while (!pendingRegions.empty()) {
1368	for (Operation &op : pendingRegions.pop_back_val()->getOps()) {
1369	for (Value operand : op.getOperands()) {
1370	// Check that any value that is used by an operation is defined in the
1371	// same region as either an operation result.
1372	auto *operandRegion = operand.getParentRegion();
1373	if (!operandRegion)
1374	return op.emitError(message: "operation's operand is unlinked");
1375	if (!region.isAncestor(other: operandRegion)) {
1376	return op.emitOpError(message: "using value defined outside the region")
1377	.attachNote(noteLoc: isolatedOp->getLoc())
1378	<< "required by region isolation constraints";
1379	}
1380	}
1381
1382	// Schedule any regions in the operation for further checking. Don't
1383	// recurse into other IsolatedFromAbove ops, because they will check
1384	// themselves.
1385	if (op.getNumRegions() &&
1386	!op.hasTrait<OpTrait::IsIsolatedFromAbove>()) {
1387	for (Region &subRegion : op.getRegions())
1388	pendingRegions.push_back(Elt: &subRegion);
1389	}
1390	}
1391	}
1392	}
1393
1394	return success();
1395	}
1396
1397	bool OpTrait::hasElementwiseMappableTraits(Operation *op) {
1398	return op->hasTrait<Elementwise>() && op->hasTrait<Scalarizable>() &&
1399	op->hasTrait<Vectorizable>() && op->hasTrait<Tensorizable>();
1400	}
1401
1402	//===----------------------------------------------------------------------===//
1403	// Misc. utils
1404	//===----------------------------------------------------------------------===//
1405
1406	/// Insert an operation, generated by `buildTerminatorOp`, at the end of the
1407	/// region's only block if it does not have a terminator already. If the region
1408	/// is empty, insert a new block first. `buildTerminatorOp` should return the
1409	/// terminator operation to insert.
1410	void impl::ensureRegionTerminator(
1411	Region &region, OpBuilder &builder, Location loc,
1412	function_ref<Operation *(OpBuilder &, Location)> buildTerminatorOp) {
1413	OpBuilder::InsertionGuard guard(builder);
1414	if (region.empty())
1415	builder.createBlock(parent: &region);
1416
1417	Block &block = region.back();
1418	if (!block.empty() && block.back().hasTrait<OpTrait::IsTerminator>())
1419	return;
1420
1421	builder.setInsertionPointToEnd(&block);
1422	builder.insert(op: buildTerminatorOp(builder, loc));
1423	}
1424
1425	/// Create a simple OpBuilder and forward to the OpBuilder version of this
1426	/// function.
1427	void impl::ensureRegionTerminator(
1428	Region &region, Builder &builder, Location loc,
1429	function_ref<Operation *(OpBuilder &, Location)> buildTerminatorOp) {
1430	OpBuilder opBuilder(builder.getContext());
1431	ensureRegionTerminator(region, builder&: opBuilder, loc, buildTerminatorOp);
1432	}
1433