1//===- Pass.cpp - Pass infrastructure implementation ----------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements common pass infrastructure.
10//
11//===----------------------------------------------------------------------===//
12
13#include "mlir/Pass/Pass.h"
14#include "PassDetail.h"
15#include "mlir/IR/Diagnostics.h"
16#include "mlir/IR/Dialect.h"
17#include "mlir/IR/OpDefinition.h"
18#include "mlir/IR/Threading.h"
19#include "mlir/IR/Verifier.h"
20#include "mlir/Support/FileUtilities.h"
21#include "mlir/Support/IndentedOstream.h"
22#include "llvm/ADT/Hashing.h"
23#include "llvm/ADT/STLExtras.h"
24#include "llvm/ADT/ScopeExit.h"
25#include "llvm/Support/CommandLine.h"
26#include "llvm/Support/CrashRecoveryContext.h"
27#include "llvm/Support/Mutex.h"
28#include "llvm/Support/Signals.h"
29#include "llvm/Support/Threading.h"
30#include "llvm/Support/ToolOutputFile.h"
31#include <optional>
32
33using namespace mlir;
34using namespace mlir::detail;
35
36//===----------------------------------------------------------------------===//
37// PassExecutionAction
38//===----------------------------------------------------------------------===//
39
40PassExecutionAction::PassExecutionAction(ArrayRef<IRUnit> irUnits,
41 const Pass &pass)
42 : Base(irUnits), pass(pass) {}
43
44void PassExecutionAction::print(raw_ostream &os) const {
45 os << llvm::formatv(Fmt: "`{0}` running `{1}` on Operation `{2}`", Vals: tag,
46 Vals: pass.getName(), Vals: getOp()->getName());
47}
48
49Operation *PassExecutionAction::getOp() const {
50 ArrayRef<IRUnit> irUnits = getContextIRUnits();
51 return irUnits.empty() ? nullptr
52 : llvm::dyn_cast_if_present<Operation *>(Val: irUnits[0]);
53}
54
55MLIR_DEFINE_EXPLICIT_TYPE_ID(::mlir::PassExecutionAction)
56
57//===----------------------------------------------------------------------===//
58// Pass
59//===----------------------------------------------------------------------===//
60
61/// Out of line virtual method to ensure vtables and metadata are emitted to a
62/// single .o file.
63void Pass::anchor() {}
64
65/// Attempt to initialize the options of this pass from the given string.
66LogicalResult Pass::initializeOptions(
67 StringRef options,
68 function_ref<LogicalResult(const Twine &)> errorHandler) {
69 std::string errStr;
70 llvm::raw_string_ostream os(errStr);
71 if (failed(Result: passOptions.parseFromString(options, errorStream&: os))) {
72 return errorHandler(errStr);
73 }
74 return success();
75}
76
77/// Copy the option values from 'other', which is another instance of this
78/// pass.
79void Pass::copyOptionValuesFrom(const Pass *other) {
80 passOptions.copyOptionValuesFrom(other: other->passOptions);
81}
82
83/// Prints out the pass in the textual representation of pipelines. If this is
84/// an adaptor pass, print its pass managers. When `pretty` is true, the
85/// printed pipeline is formatted for readability.
86void Pass::printAsTextualPipeline(raw_ostream &os, bool pretty) {
87 // Special case for adaptors to print its pass managers.
88 if (auto *adaptor = dyn_cast<OpToOpPassAdaptor>(Val: this)) {
89 llvm::interleave(
90 c: adaptor->getPassManagers(),
91 each_fn: [&](OpPassManager &pm) { pm.printAsTextualPipeline(os, pretty); },
92 between_fn: [&] {
93 os << ",";
94 if (pretty)
95 os << "\n";
96 });
97 return;
98 }
99 // Otherwise, print the pass argument followed by its options. If the pass
100 // doesn't have an argument, print the name of the pass to give some indicator
101 // of what pass was run.
102 StringRef argument = getArgument();
103 if (!argument.empty())
104 os << argument;
105 else
106 os << "unknown<" << getName() << ">";
107 passOptions.print(os);
108}
109
110//===----------------------------------------------------------------------===//
111// OpPassManagerImpl
112//===----------------------------------------------------------------------===//
113
114namespace mlir {
115namespace detail {
116struct OpPassManagerImpl {
117 OpPassManagerImpl(OperationName opName, OpPassManager::Nesting nesting)
118 : name(opName.getStringRef().str()), opName(opName),
119 initializationGeneration(0), nesting(nesting) {}
120 OpPassManagerImpl(StringRef name, OpPassManager::Nesting nesting)
121 : name(name == OpPassManager::getAnyOpAnchorName() ? "" : name.str()),
122 initializationGeneration(0), nesting(nesting) {}
123 OpPassManagerImpl(OpPassManager::Nesting nesting)
124 : initializationGeneration(0), nesting(nesting) {}
125 OpPassManagerImpl(const OpPassManagerImpl &rhs)
126 : name(rhs.name), opName(rhs.opName),
127 initializationGeneration(rhs.initializationGeneration),
128 nesting(rhs.nesting) {
129 for (const std::unique_ptr<Pass> &pass : rhs.passes) {
130 std::unique_ptr<Pass> newPass = pass->clone();
131 newPass->threadingSibling = pass.get();
132 passes.push_back(x: std::move(newPass));
133 }
134 }
135
136 /// Merge the passes of this pass manager into the one provided.
137 void mergeInto(OpPassManagerImpl &rhs);
138
139 /// Nest a new operation pass manager for the given operation kind under this
140 /// pass manager.
141 OpPassManager &nest(OperationName nestedName) {
142 return nest(nested: OpPassManager(nestedName, nesting));
143 }
144 OpPassManager &nest(StringRef nestedName) {
145 return nest(nested: OpPassManager(nestedName, nesting));
146 }
147 OpPassManager &nestAny() { return nest(nested: OpPassManager(nesting)); }
148
149 /// Nest the given pass manager under this pass manager.
150 OpPassManager &nest(OpPassManager &&nested);
151
152 /// Add the given pass to this pass manager. If this pass has a concrete
153 /// operation type, it must be the same type as this pass manager.
154 void addPass(std::unique_ptr<Pass> pass);
155
156 /// Clear the list of passes in this pass manager, other options are
157 /// preserved.
158 void clear();
159
160 /// Finalize the pass list in preparation for execution. This includes
161 /// coalescing adjacent pass managers when possible, verifying scheduled
162 /// passes, etc.
163 LogicalResult finalizePassList(MLIRContext *ctx);
164
165 /// Return the operation name of this pass manager.
166 std::optional<OperationName> getOpName(MLIRContext &context) {
167 if (!name.empty() && !opName)
168 opName = OperationName(name, &context);
169 return opName;
170 }
171 std::optional<StringRef> getOpName() const {
172 return name.empty() ? std::optional<StringRef>()
173 : std::optional<StringRef>(name);
174 }
175
176 /// Return the name used to anchor this pass manager. This is either the name
177 /// of an operation, or the result of `getAnyOpAnchorName()` in the case of an
178 /// op-agnostic pass manager.
179 StringRef getOpAnchorName() const {
180 return getOpName().value_or(u: OpPassManager::getAnyOpAnchorName());
181 }
182
183 /// Indicate if the current pass manager can be scheduled on the given
184 /// operation type.
185 bool canScheduleOn(MLIRContext &context, OperationName opName);
186
187 /// The name of the operation that passes of this pass manager operate on.
188 std::string name;
189
190 /// The cached OperationName (internalized in the context) for the name of the
191 /// operation that passes of this pass manager operate on.
192 std::optional<OperationName> opName;
193
194 /// The set of passes to run as part of this pass manager.
195 std::vector<std::unique_ptr<Pass>> passes;
196
197 /// The current initialization generation of this pass manager. This is used
198 /// to indicate when a pass manager should be reinitialized.
199 unsigned initializationGeneration;
200
201 /// Control the implicit nesting of passes that mismatch the name set for this
202 /// OpPassManager.
203 OpPassManager::Nesting nesting;
204};
205} // namespace detail
206} // namespace mlir
207
208void OpPassManagerImpl::mergeInto(OpPassManagerImpl &rhs) {
209 assert(name == rhs.name && "merging unrelated pass managers");
210 for (auto &pass : passes)
211 rhs.passes.push_back(x: std::move(pass));
212 passes.clear();
213}
214
215OpPassManager &OpPassManagerImpl::nest(OpPassManager &&nested) {
216 auto *adaptor = new OpToOpPassAdaptor(std::move(nested));
217 addPass(pass: std::unique_ptr<Pass>(adaptor));
218 return adaptor->getPassManagers().front();
219}
220
221void OpPassManagerImpl::addPass(std::unique_ptr<Pass> pass) {
222 // If this pass runs on a different operation than this pass manager, then
223 // implicitly nest a pass manager for this operation if enabled.
224 std::optional<StringRef> pmOpName = getOpName();
225 std::optional<StringRef> passOpName = pass->getOpName();
226 if (pmOpName && passOpName && *pmOpName != *passOpName) {
227 if (nesting == OpPassManager::Nesting::Implicit)
228 return nest(nestedName: *passOpName).addPass(pass: std::move(pass));
229 llvm::report_fatal_error(reason: llvm::Twine("Can't add pass '") + pass->getName() +
230 "' restricted to '" + *passOpName +
231 "' on a PassManager intended to run on '" +
232 getOpAnchorName() + "', did you intend to nest?");
233 }
234
235 passes.emplace_back(args: std::move(pass));
236}
237
238void OpPassManagerImpl::clear() { passes.clear(); }
239
240LogicalResult OpPassManagerImpl::finalizePassList(MLIRContext *ctx) {
241 auto finalizeAdaptor = [ctx](OpToOpPassAdaptor *adaptor) {
242 for (auto &pm : adaptor->getPassManagers())
243 if (failed(Result: pm.getImpl().finalizePassList(ctx)))
244 return failure();
245 return success();
246 };
247
248 // Walk the pass list and merge adjacent adaptors.
249 OpToOpPassAdaptor *lastAdaptor = nullptr;
250 for (auto &pass : passes) {
251 // Check to see if this pass is an adaptor.
252 if (auto *currentAdaptor = dyn_cast<OpToOpPassAdaptor>(Val: pass.get())) {
253 // If it is the first adaptor in a possible chain, remember it and
254 // continue.
255 if (!lastAdaptor) {
256 lastAdaptor = currentAdaptor;
257 continue;
258 }
259
260 // Otherwise, try to merge into the existing adaptor and delete the
261 // current one. If merging fails, just remember this as the last adaptor.
262 if (succeeded(Result: currentAdaptor->tryMergeInto(ctx, rhs&: *lastAdaptor)))
263 pass.reset();
264 else
265 lastAdaptor = currentAdaptor;
266 } else if (lastAdaptor) {
267 // If this pass isn't an adaptor, finalize it and forget the last adaptor.
268 if (failed(Result: finalizeAdaptor(lastAdaptor)))
269 return failure();
270 lastAdaptor = nullptr;
271 }
272 }
273
274 // If there was an adaptor at the end of the manager, finalize it as well.
275 if (lastAdaptor && failed(Result: finalizeAdaptor(lastAdaptor)))
276 return failure();
277
278 // Now that the adaptors have been merged, erase any empty slots corresponding
279 // to the merged adaptors that were nulled-out in the loop above.
280 llvm::erase_if(C&: passes, P: std::logical_not<std::unique_ptr<Pass>>());
281
282 // If this is a op-agnostic pass manager, there is nothing left to do.
283 std::optional<OperationName> rawOpName = getOpName(context&: *ctx);
284 if (!rawOpName)
285 return success();
286
287 // Otherwise, verify that all of the passes are valid for the current
288 // operation anchor.
289 std::optional<RegisteredOperationName> opName =
290 rawOpName->getRegisteredInfo();
291 for (std::unique_ptr<Pass> &pass : passes) {
292 if (opName && !pass->canScheduleOn(opName: *opName)) {
293 return emitError(UnknownLoc::get(ctx))
294 << "unable to schedule pass '" << pass->getName()
295 << "' on a PassManager intended to run on '" << getOpAnchorName()
296 << "'!";
297 }
298 }
299 return success();
300}
301
302bool OpPassManagerImpl::canScheduleOn(MLIRContext &context,
303 OperationName opName) {
304 // If this pass manager is op-specific, we simply check if the provided
305 // operation name is the same as this one.
306 std::optional<OperationName> pmOpName = getOpName(context);
307 if (pmOpName)
308 return pmOpName == opName;
309
310 // Otherwise, this is an op-agnostic pass manager. Check that the operation
311 // can be scheduled on all passes within the manager.
312 std::optional<RegisteredOperationName> registeredInfo =
313 opName.getRegisteredInfo();
314 if (!registeredInfo ||
315 !registeredInfo->hasTrait<OpTrait::IsIsolatedFromAbove>())
316 return false;
317 return llvm::all_of(Range&: passes, P: [&](const std::unique_ptr<Pass> &pass) {
318 return pass->canScheduleOn(opName: *registeredInfo);
319 });
320}
321
322//===----------------------------------------------------------------------===//
323// OpPassManager
324//===----------------------------------------------------------------------===//
325
326OpPassManager::OpPassManager(Nesting nesting)
327 : impl(new OpPassManagerImpl(nesting)) {}
328OpPassManager::OpPassManager(StringRef name, Nesting nesting)
329 : impl(new OpPassManagerImpl(name, nesting)) {}
330OpPassManager::OpPassManager(OperationName name, Nesting nesting)
331 : impl(new OpPassManagerImpl(name, nesting)) {}
332OpPassManager::OpPassManager(OpPassManager &&rhs) { *this = std::move(rhs); }
333OpPassManager::OpPassManager(const OpPassManager &rhs) { *this = rhs; }
334OpPassManager &OpPassManager::operator=(const OpPassManager &rhs) {
335 impl = std::make_unique<OpPassManagerImpl>(args&: *rhs.impl);
336 return *this;
337}
338OpPassManager &OpPassManager::operator=(OpPassManager &&rhs) {
339 impl = std::move(rhs.impl);
340 return *this;
341}
342
343OpPassManager::~OpPassManager() = default;
344
345OpPassManager::pass_iterator OpPassManager::begin() {
346 return MutableArrayRef<std::unique_ptr<Pass>>{impl->passes}.begin();
347}
348OpPassManager::pass_iterator OpPassManager::end() {
349 return MutableArrayRef<std::unique_ptr<Pass>>{impl->passes}.end();
350}
351
352OpPassManager::const_pass_iterator OpPassManager::begin() const {
353 return ArrayRef<std::unique_ptr<Pass>>{impl->passes}.begin();
354}
355OpPassManager::const_pass_iterator OpPassManager::end() const {
356 return ArrayRef<std::unique_ptr<Pass>>{impl->passes}.end();
357}
358
359/// Nest a new operation pass manager for the given operation kind under this
360/// pass manager.
361OpPassManager &OpPassManager::nest(OperationName nestedName) {
362 return impl->nest(nestedName);
363}
364OpPassManager &OpPassManager::nest(StringRef nestedName) {
365 return impl->nest(nestedName);
366}
367OpPassManager &OpPassManager::nestAny() { return impl->nestAny(); }
368
369/// Add the given pass to this pass manager. If this pass has a concrete
370/// operation type, it must be the same type as this pass manager.
371void OpPassManager::addPass(std::unique_ptr<Pass> pass) {
372 impl->addPass(pass: std::move(pass));
373}
374
375void OpPassManager::clear() { impl->clear(); }
376
377/// Returns the number of passes held by this manager.
378size_t OpPassManager::size() const { return impl->passes.size(); }
379
380/// Returns the internal implementation instance.
381OpPassManagerImpl &OpPassManager::getImpl() { return *impl; }
382
383/// Return the operation name that this pass manager operates on.
384std::optional<StringRef> OpPassManager::getOpName() const {
385 return impl->getOpName();
386}
387
388/// Return the operation name that this pass manager operates on.
389std::optional<OperationName>
390OpPassManager::getOpName(MLIRContext &context) const {
391 return impl->getOpName(context);
392}
393
394StringRef OpPassManager::getOpAnchorName() const {
395 return impl->getOpAnchorName();
396}
397
398/// Prints out the passes of the pass manager as the textual representation
399/// of pipelines. When `pretty` is true, the printed pipeline is formatted for
400/// readability.
401void printAsTextualPipeline(
402 raw_indented_ostream &os, StringRef anchorName,
403 const llvm::iterator_range<OpPassManager::pass_iterator> &passes,
404 bool pretty = false) {
405 os << anchorName << "(";
406 if (pretty) {
407 os << "\n";
408 os.indent();
409 }
410 llvm::interleave(
411 c: passes,
412 each_fn: [&](mlir::Pass &pass) { pass.printAsTextualPipeline(os, pretty); },
413 between_fn: [&]() {
414 os << ",";
415 if (pretty)
416 os << "\n";
417 });
418 if (pretty) {
419 os << "\n";
420 os.unindent();
421 }
422 os << ")";
423}
424void printAsTextualPipeline(
425 raw_ostream &os, StringRef anchorName,
426 const llvm::iterator_range<OpPassManager::pass_iterator> &passes,
427 bool pretty) {
428 raw_indented_ostream indentedOS(os);
429 printAsTextualPipeline(os&: indentedOS, anchorName, passes, pretty);
430}
431void OpPassManager::printAsTextualPipeline(raw_ostream &os, bool pretty) const {
432 StringRef anchorName = getOpAnchorName();
433 raw_indented_ostream indentedOS(os);
434 ::printAsTextualPipeline(
435 os&: indentedOS, anchorName,
436 passes: {MutableArrayRef<std::unique_ptr<Pass>>{impl->passes}.begin(),
437 MutableArrayRef<std::unique_ptr<Pass>>{impl->passes}.end()},
438 pretty);
439}
440
441void OpPassManager::dump() {
442 llvm::errs() << "Pass Manager with " << impl->passes.size() << " passes:\n";
443 printAsTextualPipeline(os&: llvm::errs(), /*pretty=*/true);
444 llvm::errs() << "\n";
445}
446
447static void registerDialectsForPipeline(const OpPassManager &pm,
448 DialectRegistry &dialects) {
449 for (const Pass &pass : pm.getPasses())
450 pass.getDependentDialects(registry&: dialects);
451}
452
453void OpPassManager::getDependentDialects(DialectRegistry &dialects) const {
454 registerDialectsForPipeline(pm: *this, dialects);
455}
456
457void OpPassManager::setNesting(Nesting nesting) { impl->nesting = nesting; }
458
459OpPassManager::Nesting OpPassManager::getNesting() { return impl->nesting; }
460
461LogicalResult OpPassManager::initialize(MLIRContext *context,
462 unsigned newInitGeneration) {
463 if (impl->initializationGeneration == newInitGeneration)
464 return success();
465 impl->initializationGeneration = newInitGeneration;
466 for (Pass &pass : getPasses()) {
467 // If this pass isn't an adaptor, directly initialize it.
468 auto *adaptor = dyn_cast<OpToOpPassAdaptor>(Val: &pass);
469 if (!adaptor) {
470 if (failed(Result: pass.initialize(context)))
471 return failure();
472 continue;
473 }
474
475 // Otherwise, initialize each of the adaptors pass managers.
476 for (OpPassManager &adaptorPM : adaptor->getPassManagers())
477 if (failed(Result: adaptorPM.initialize(context, newInitGeneration)))
478 return failure();
479 }
480 return success();
481}
482
483llvm::hash_code OpPassManager::hash() {
484 llvm::hash_code hashCode{};
485 for (Pass &pass : getPasses()) {
486 // If this pass isn't an adaptor, directly hash it.
487 auto *adaptor = dyn_cast<OpToOpPassAdaptor>(Val: &pass);
488 if (!adaptor) {
489 hashCode = llvm::hash_combine(args: hashCode, args: &pass);
490 continue;
491 }
492 // Otherwise, hash recursively each of the adaptors pass managers.
493 for (OpPassManager &adaptorPM : adaptor->getPassManagers())
494 llvm::hash_combine(args: hashCode, args: adaptorPM.hash());
495 }
496 return hashCode;
497}
498
499//===----------------------------------------------------------------------===//
500// OpToOpPassAdaptor
501//===----------------------------------------------------------------------===//
502
503LogicalResult OpToOpPassAdaptor::run(Pass *pass, Operation *op,
504 AnalysisManager am, bool verifyPasses,
505 unsigned parentInitGeneration) {
506 std::optional<RegisteredOperationName> opInfo = op->getRegisteredInfo();
507 if (!opInfo)
508 return op->emitOpError()
509 << "trying to schedule a pass on an unregistered operation";
510 if (!opInfo->hasTrait<OpTrait::IsIsolatedFromAbove>())
511 return op->emitOpError() << "trying to schedule a pass on an operation not "
512 "marked as 'IsolatedFromAbove'";
513 if (!pass->canScheduleOn(opName: *op->getName().getRegisteredInfo()))
514 return op->emitOpError()
515 << "trying to schedule a pass on an unsupported operation";
516
517 // Initialize the pass state with a callback for the pass to dynamically
518 // execute a pipeline on the currently visited operation.
519 PassInstrumentor *pi = am.getPassInstrumentor();
520 PassInstrumentation::PipelineParentInfo parentInfo = {.parentThreadID: llvm::get_threadid(),
521 .parentPass: pass};
522 auto dynamicPipelineCallback = [&](OpPassManager &pipeline,
523 Operation *root) -> LogicalResult {
524 if (!op->isAncestor(other: root))
525 return root->emitOpError()
526 << "Trying to schedule a dynamic pipeline on an "
527 "operation that isn't "
528 "nested under the current operation the pass is processing";
529 assert(
530 pipeline.getImpl().canScheduleOn(*op->getContext(), root->getName()));
531
532 // Before running, finalize the passes held by the pipeline.
533 if (failed(Result: pipeline.getImpl().finalizePassList(ctx: root->getContext())))
534 return failure();
535
536 // Initialize the user provided pipeline and execute the pipeline.
537 if (failed(Result: pipeline.initialize(context: root->getContext(), newInitGeneration: parentInitGeneration)))
538 return failure();
539 AnalysisManager nestedAm = root == op ? am : am.nest(op: root);
540 return OpToOpPassAdaptor::runPipeline(pm&: pipeline, op: root, am: nestedAm,
541 verifyPasses, parentInitGeneration,
542 instrumentor: pi, parentInfo: &parentInfo);
543 };
544 pass->passState.emplace(args&: op, args&: am, args&: dynamicPipelineCallback);
545
546 // Instrument before the pass has run.
547 if (pi)
548 pi->runBeforePass(pass, op);
549
550 bool passFailed = false;
551 op->getContext()->executeAction<PassExecutionAction>(
552 actionFn: [&]() {
553 // Invoke the virtual runOnOperation method.
554 if (auto *adaptor = dyn_cast<OpToOpPassAdaptor>(Val: pass))
555 adaptor->runOnOperation(verifyPasses);
556 else
557 pass->runOnOperation();
558 passFailed = pass->passState->irAndPassFailed.getInt();
559 },
560 irUnits: {op}, args&: *pass);
561
562 // Invalidate any non preserved analyses.
563 am.invalidate(pa: pass->passState->preservedAnalyses);
564
565 // When verifyPasses is specified, we run the verifier (unless the pass
566 // failed).
567 if (!passFailed && verifyPasses) {
568 bool runVerifierNow = true;
569
570 // If the pass is an adaptor pass, we don't run the verifier recursively
571 // because the nested operations should have already been verified after
572 // nested passes had run.
573 bool runVerifierRecursively = !isa<OpToOpPassAdaptor>(Val: pass);
574
575 // Reduce compile time by avoiding running the verifier if the pass didn't
576 // change the IR since the last time the verifier was run:
577 //
578 // 1) If the pass said that it preserved all analyses then it can't have
579 // permuted the IR.
580 //
581 // We run these checks in EXPENSIVE_CHECKS mode out of caution.
582#ifndef EXPENSIVE_CHECKS
583 runVerifierNow = !pass->passState->preservedAnalyses.isAll();
584#endif
585 if (runVerifierNow)
586 passFailed = failed(Result: verify(op, verifyRecursively: runVerifierRecursively));
587 }
588
589 // Instrument after the pass has run.
590 if (pi) {
591 if (passFailed)
592 pi->runAfterPassFailed(pass, op);
593 else
594 pi->runAfterPass(pass, op);
595 }
596
597 // Return if the pass signaled a failure.
598 return failure(IsFailure: passFailed);
599}
600
601/// Run the given operation and analysis manager on a provided op pass manager.
602LogicalResult OpToOpPassAdaptor::runPipeline(
603 OpPassManager &pm, Operation *op, AnalysisManager am, bool verifyPasses,
604 unsigned parentInitGeneration, PassInstrumentor *instrumentor,
605 const PassInstrumentation::PipelineParentInfo *parentInfo) {
606 assert((!instrumentor || parentInfo) &&
607 "expected parent info if instrumentor is provided");
608 auto scopeExit = llvm::make_scope_exit(F: [&] {
609 // Clear out any computed operation analyses. These analyses won't be used
610 // any more in this pipeline, and this helps reduce the current working set
611 // of memory. If preserving these analyses becomes important in the future
612 // we can re-evaluate this.
613 am.clear();
614 });
615
616 // Run the pipeline over the provided operation.
617 if (instrumentor) {
618 instrumentor->runBeforePipeline(name: pm.getOpName(context&: *op->getContext()),
619 parentInfo: *parentInfo);
620 }
621
622 for (Pass &pass : pm.getPasses())
623 if (failed(Result: run(pass: &pass, op, am, verifyPasses, parentInitGeneration)))
624 return failure();
625
626 if (instrumentor) {
627 instrumentor->runAfterPipeline(name: pm.getOpName(context&: *op->getContext()),
628 parentInfo: *parentInfo);
629 }
630 return success();
631}
632
633/// Find an operation pass manager with the given anchor name, or nullptr if one
634/// does not exist.
635static OpPassManager *
636findPassManagerWithAnchor(MutableArrayRef<OpPassManager> mgrs, StringRef name) {
637 auto *it = llvm::find_if(
638 Range&: mgrs, P: [&](OpPassManager &mgr) { return mgr.getOpAnchorName() == name; });
639 return it == mgrs.end() ? nullptr : &*it;
640}
641
642/// Find an operation pass manager that can operate on an operation of the given
643/// type, or nullptr if one does not exist.
644static OpPassManager *findPassManagerFor(MutableArrayRef<OpPassManager> mgrs,
645 OperationName name,
646 MLIRContext &context) {
647 auto *it = llvm::find_if(Range&: mgrs, P: [&](OpPassManager &mgr) {
648 return mgr.getImpl().canScheduleOn(context, opName: name);
649 });
650 return it == mgrs.end() ? nullptr : &*it;
651}
652
653OpToOpPassAdaptor::OpToOpPassAdaptor(OpPassManager &&mgr) {
654 mgrs.emplace_back(Args: std::move(mgr));
655}
656
657void OpToOpPassAdaptor::getDependentDialects(DialectRegistry &dialects) const {
658 for (auto &pm : mgrs)
659 pm.getDependentDialects(dialects);
660}
661
662LogicalResult OpToOpPassAdaptor::tryMergeInto(MLIRContext *ctx,
663 OpToOpPassAdaptor &rhs) {
664 // Functor used to check if a pass manager is generic, i.e. op-agnostic.
665 auto isGenericPM = [&](OpPassManager &pm) { return !pm.getOpName(); };
666
667 // Functor used to detect if the given generic pass manager will have a
668 // potential schedule conflict with the given `otherPMs`.
669 auto hasScheduleConflictWith = [&](OpPassManager &genericPM,
670 MutableArrayRef<OpPassManager> otherPMs) {
671 return llvm::any_of(Range&: otherPMs, P: [&](OpPassManager &pm) {
672 // If this is a non-generic pass manager, a conflict will arise if a
673 // non-generic pass manager's operation name can be scheduled on the
674 // generic passmanager.
675 if (std::optional<OperationName> pmOpName = pm.getOpName(context&: *ctx))
676 return genericPM.getImpl().canScheduleOn(context&: *ctx, opName: *pmOpName);
677 // Otherwise, this is a generic pass manager. We current can't determine
678 // when generic pass managers can be merged, so conservatively assume they
679 // conflict.
680 return true;
681 });
682 };
683
684 // Check that if either adaptor has a generic pass manager, that pm is
685 // compatible within any non-generic pass managers.
686 //
687 // Check the current adaptor.
688 auto *lhsGenericPMIt = llvm::find_if(Range&: mgrs, P: isGenericPM);
689 if (lhsGenericPMIt != mgrs.end() &&
690 hasScheduleConflictWith(*lhsGenericPMIt, rhs.mgrs))
691 return failure();
692 // Check the rhs adaptor.
693 auto *rhsGenericPMIt = llvm::find_if(Range&: rhs.mgrs, P: isGenericPM);
694 if (rhsGenericPMIt != rhs.mgrs.end() &&
695 hasScheduleConflictWith(*rhsGenericPMIt, mgrs))
696 return failure();
697
698 for (auto &pm : mgrs) {
699 // If an existing pass manager exists, then merge the given pass manager
700 // into it.
701 if (auto *existingPM =
702 findPassManagerWithAnchor(mgrs: rhs.mgrs, name: pm.getOpAnchorName())) {
703 pm.getImpl().mergeInto(rhs&: existingPM->getImpl());
704 } else {
705 // Otherwise, add the given pass manager to the list.
706 rhs.mgrs.emplace_back(Args: std::move(pm));
707 }
708 }
709 mgrs.clear();
710
711 // After coalescing, sort the pass managers within rhs by name.
712 auto compareFn = [](const OpPassManager &lhs, const OpPassManager &rhs) {
713 // Order op-specific pass managers first and op-agnostic pass managers last.
714 if (std::optional<StringRef> lhsName = lhs.getOpName()) {
715 if (std::optional<StringRef> rhsName = rhs.getOpName())
716 return *lhsName < *rhsName;
717 return true; // lhs(op-specific) < rhs(op-agnostic)
718 }
719 return false; // lhs(op-agnostic) > rhs(op-specific)
720 };
721 llvm::sort(C&: rhs.mgrs, Comp: compareFn);
722 return success();
723}
724
725/// Returns the adaptor pass name.
726std::string OpToOpPassAdaptor::getAdaptorName() {
727 std::string name = "Pipeline Collection : [";
728 llvm::raw_string_ostream os(name);
729 llvm::interleaveComma(c: getPassManagers(), os, each_fn: [&](OpPassManager &pm) {
730 os << '\'' << pm.getOpAnchorName() << '\'';
731 });
732 os << ']';
733 return name;
734}
735
736void OpToOpPassAdaptor::runOnOperation() {
737 llvm_unreachable(
738 "Unexpected call to Pass::runOnOperation() on OpToOpPassAdaptor");
739}
740
741/// Run the held pipeline over all nested operations.
742void OpToOpPassAdaptor::runOnOperation(bool verifyPasses) {
743 if (getContext().isMultithreadingEnabled())
744 runOnOperationAsyncImpl(verifyPasses);
745 else
746 runOnOperationImpl(verifyPasses);
747}
748
749/// Run this pass adaptor synchronously.
750void OpToOpPassAdaptor::runOnOperationImpl(bool verifyPasses) {
751 auto am = getAnalysisManager();
752 PassInstrumentation::PipelineParentInfo parentInfo = {.parentThreadID: llvm::get_threadid(),
753 .parentPass: this};
754 auto *instrumentor = am.getPassInstrumentor();
755 for (auto &region : getOperation()->getRegions()) {
756 for (auto &block : region) {
757 for (auto &op : block) {
758 auto *mgr = findPassManagerFor(mgrs, name: op.getName(), context&: *op.getContext());
759 if (!mgr)
760 continue;
761
762 // Run the held pipeline over the current operation.
763 unsigned initGeneration = mgr->impl->initializationGeneration;
764 if (failed(Result: runPipeline(pm&: *mgr, op: &op, am: am.nest(op: &op), verifyPasses,
765 parentInitGeneration: initGeneration, instrumentor, parentInfo: &parentInfo)))
766 signalPassFailure();
767 }
768 }
769 }
770}
771
772/// Utility functor that checks if the two ranges of pass managers have a size
773/// mismatch.
774static bool hasSizeMismatch(ArrayRef<OpPassManager> lhs,
775 ArrayRef<OpPassManager> rhs) {
776 return lhs.size() != rhs.size() ||
777 llvm::any_of(Range: llvm::seq<size_t>(Begin: 0, End: lhs.size()),
778 P: [&](size_t i) { return lhs[i].size() != rhs[i].size(); });
779}
780
781/// Run this pass adaptor synchronously.
782void OpToOpPassAdaptor::runOnOperationAsyncImpl(bool verifyPasses) {
783 AnalysisManager am = getAnalysisManager();
784 MLIRContext *context = &getContext();
785
786 // Create the async executors if they haven't been created, or if the main
787 // pipeline has changed.
788 if (asyncExecutors.empty() || hasSizeMismatch(lhs: asyncExecutors.front(), rhs: mgrs))
789 asyncExecutors.assign(NumElts: context->getThreadPool().getMaxConcurrency(), Elt: mgrs);
790
791 // This struct represents the information for a single operation to be
792 // scheduled on a pass manager.
793 struct OpPMInfo {
794 OpPMInfo(unsigned passManagerIdx, Operation *op, AnalysisManager am)
795 : passManagerIdx(passManagerIdx), op(op), am(am) {}
796
797 /// The index of the pass manager to schedule the operation on.
798 unsigned passManagerIdx;
799 /// The operation to schedule.
800 Operation *op;
801 /// The analysis manager for the operation.
802 AnalysisManager am;
803 };
804
805 // Run a prepass over the operation to collect the nested operations to
806 // execute over. This ensures that an analysis manager exists for each
807 // operation, as well as providing a queue of operations to execute over.
808 std::vector<OpPMInfo> opInfos;
809 DenseMap<OperationName, std::optional<unsigned>> knownOpPMIdx;
810 for (auto &region : getOperation()->getRegions()) {
811 for (Operation &op : region.getOps()) {
812 // Get the pass manager index for this operation type.
813 auto pmIdxIt = knownOpPMIdx.try_emplace(Key: op.getName(), Args: std::nullopt);
814 if (pmIdxIt.second) {
815 if (auto *mgr = findPassManagerFor(mgrs, name: op.getName(), context&: *context))
816 pmIdxIt.first->second = std::distance(first: mgrs.begin(), last: mgr);
817 }
818
819 // If this operation can be scheduled, add it to the list.
820 if (pmIdxIt.first->second)
821 opInfos.emplace_back(args&: *pmIdxIt.first->second, args: &op, args: am.nest(op: &op));
822 }
823 }
824
825 // Get the current thread for this adaptor.
826 PassInstrumentation::PipelineParentInfo parentInfo = {.parentThreadID: llvm::get_threadid(),
827 .parentPass: this};
828 auto *instrumentor = am.getPassInstrumentor();
829
830 // An atomic failure variable for the async executors.
831 std::vector<std::atomic<bool>> activePMs(asyncExecutors.size());
832 std::fill(first: activePMs.begin(), last: activePMs.end(), value: false);
833 std::atomic<bool> hasFailure = false;
834 parallelForEach(context, range&: opInfos, func: [&](OpPMInfo &opInfo) {
835 // Find an executor for this operation.
836 auto it = llvm::find_if(Range&: activePMs, P: [](std::atomic<bool> &isActive) {
837 bool expectedInactive = false;
838 return isActive.compare_exchange_strong(i1&: expectedInactive, i2: true);
839 });
840 unsigned pmIndex = it - activePMs.begin();
841
842 // Get the pass manager for this operation and execute it.
843 OpPassManager &pm = asyncExecutors[pmIndex][opInfo.passManagerIdx];
844 LogicalResult pipelineResult = runPipeline(
845 pm, op: opInfo.op, am: opInfo.am, verifyPasses,
846 parentInitGeneration: pm.impl->initializationGeneration, instrumentor, parentInfo: &parentInfo);
847 if (failed(Result: pipelineResult))
848 hasFailure.store(i: true);
849
850 // Reset the active bit for this pass manager.
851 activePMs[pmIndex].store(i: false);
852 });
853
854 // Signal a failure if any of the executors failed.
855 if (hasFailure)
856 signalPassFailure();
857}
858
859//===----------------------------------------------------------------------===//
860// PassManager
861//===----------------------------------------------------------------------===//
862
863PassManager::PassManager(MLIRContext *ctx, StringRef operationName,
864 Nesting nesting)
865 : OpPassManager(operationName, nesting), context(ctx), passTiming(false),
866 verifyPasses(true) {}
867
868PassManager::PassManager(OperationName operationName, Nesting nesting)
869 : OpPassManager(operationName, nesting),
870 context(operationName.getContext()), passTiming(false),
871 verifyPasses(true) {}
872
873PassManager::~PassManager() = default;
874
875void PassManager::enableVerifier(bool enabled) { verifyPasses = enabled; }
876
877/// Run the passes within this manager on the provided operation.
878LogicalResult PassManager::run(Operation *op) {
879 MLIRContext *context = getContext();
880 std::optional<OperationName> anchorOp = getOpName(context&: *context);
881 if (anchorOp && anchorOp != op->getName())
882 return emitError(loc: op->getLoc())
883 << "can't run '" << getOpAnchorName() << "' pass manager on '"
884 << op->getName() << "' op";
885
886 // Register all dialects for the current pipeline.
887 DialectRegistry dependentDialects;
888 getDependentDialects(dialects&: dependentDialects);
889 context->appendDialectRegistry(registry: dependentDialects);
890 for (StringRef name : dependentDialects.getDialectNames())
891 context->getOrLoadDialect(name);
892
893 // Before running, make sure to finalize the pipeline pass list.
894 if (failed(Result: getImpl().finalizePassList(ctx: context)))
895 return failure();
896
897 // Notify the context that we start running a pipeline for bookkeeping.
898 context->enterMultiThreadedExecution();
899
900 // Initialize all of the passes within the pass manager with a new generation.
901 llvm::hash_code newInitKey = context->getRegistryHash();
902 llvm::hash_code pipelineKey = hash();
903 if (newInitKey != initializationKey ||
904 pipelineKey != pipelineInitializationKey) {
905 if (failed(Result: initialize(context, newInitGeneration: impl->initializationGeneration + 1)))
906 return failure();
907 initializationKey = newInitKey;
908 pipelineKey = pipelineInitializationKey;
909 }
910
911 // Construct a top level analysis manager for the pipeline.
912 ModuleAnalysisManager am(op, instrumentor.get());
913
914 // If reproducer generation is enabled, run the pass manager with crash
915 // handling enabled.
916 LogicalResult result =
917 crashReproGenerator ? runWithCrashRecovery(op, am) : runPasses(op, am);
918
919 // Notify the context that the run is done.
920 context->exitMultiThreadedExecution();
921
922 // Dump all of the pass statistics if necessary.
923 if (passStatisticsMode)
924 dumpStatistics();
925 return result;
926}
927
928/// Add the provided instrumentation to the pass manager.
929void PassManager::addInstrumentation(std::unique_ptr<PassInstrumentation> pi) {
930 if (!instrumentor)
931 instrumentor = std::make_unique<PassInstrumentor>();
932
933 instrumentor->addInstrumentation(pi: std::move(pi));
934}
935
936LogicalResult PassManager::runPasses(Operation *op, AnalysisManager am) {
937 return OpToOpPassAdaptor::runPipeline(pm&: *this, op, am, verifyPasses,
938 parentInitGeneration: impl->initializationGeneration);
939}
940
941//===----------------------------------------------------------------------===//
942// AnalysisManager
943//===----------------------------------------------------------------------===//
944
945/// Get an analysis manager for the given operation, which must be a proper
946/// descendant of the current operation represented by this analysis manager.
947AnalysisManager AnalysisManager::nest(Operation *op) {
948 Operation *currentOp = impl->getOperation();
949 assert(currentOp->isProperAncestor(op) &&
950 "expected valid descendant operation");
951
952 // Check for the base case where the provided operation is immediately nested.
953 if (currentOp == op->getParentOp())
954 return nestImmediate(op);
955
956 // Otherwise, we need to collect all ancestors up to the current operation.
957 SmallVector<Operation *, 4> opAncestors;
958 do {
959 opAncestors.push_back(Elt: op);
960 op = op->getParentOp();
961 } while (op != currentOp);
962
963 AnalysisManager result = *this;
964 for (Operation *op : llvm::reverse(C&: opAncestors))
965 result = result.nestImmediate(op);
966 return result;
967}
968
969/// Get an analysis manager for the given immediately nested child operation.
970AnalysisManager AnalysisManager::nestImmediate(Operation *op) {
971 assert(impl->getOperation() == op->getParentOp() &&
972 "expected immediate child operation");
973
974 auto [it, inserted] = impl->childAnalyses.try_emplace(Key: op);
975 if (inserted)
976 it->second = std::make_unique<NestedAnalysisMap>(args&: op, args&: impl);
977 return {it->second.get()};
978}
979
980/// Invalidate any non preserved analyses.
981void detail::NestedAnalysisMap::invalidate(
982 const detail::PreservedAnalyses &pa) {
983 // If all analyses were preserved, then there is nothing to do here.
984 if (pa.isAll())
985 return;
986
987 // Invalidate the analyses for the current operation directly.
988 analyses.invalidate(pa);
989
990 // If no analyses were preserved, then just simply clear out the child
991 // analysis results.
992 if (pa.isNone()) {
993 childAnalyses.clear();
994 return;
995 }
996
997 // Otherwise, invalidate each child analysis map.
998 SmallVector<NestedAnalysisMap *, 8> mapsToInvalidate(1, this);
999 while (!mapsToInvalidate.empty()) {
1000 auto *map = mapsToInvalidate.pop_back_val();
1001 for (auto &analysisPair : map->childAnalyses) {
1002 analysisPair.second->invalidate(pa);
1003 if (!analysisPair.second->childAnalyses.empty())
1004 mapsToInvalidate.push_back(Elt: analysisPair.second.get());
1005 }
1006 }
1007}
1008
1009//===----------------------------------------------------------------------===//
1010// PassInstrumentation
1011//===----------------------------------------------------------------------===//
1012
1013PassInstrumentation::~PassInstrumentation() = default;
1014
1015void PassInstrumentation::runBeforePipeline(
1016 std::optional<OperationName> name, const PipelineParentInfo &parentInfo) {}
1017
1018void PassInstrumentation::runAfterPipeline(
1019 std::optional<OperationName> name, const PipelineParentInfo &parentInfo) {}
1020
1021//===----------------------------------------------------------------------===//
1022// PassInstrumentor
1023//===----------------------------------------------------------------------===//
1024
1025namespace mlir {
1026namespace detail {
1027struct PassInstrumentorImpl {
1028 /// Mutex to keep instrumentation access thread-safe.
1029 llvm::sys::SmartMutex<true> mutex;
1030
1031 /// Set of registered instrumentations.
1032 std::vector<std::unique_ptr<PassInstrumentation>> instrumentations;
1033};
1034} // namespace detail
1035} // namespace mlir
1036
1037PassInstrumentor::PassInstrumentor() : impl(new PassInstrumentorImpl()) {}
1038PassInstrumentor::~PassInstrumentor() = default;
1039
1040/// See PassInstrumentation::runBeforePipeline for details.
1041void PassInstrumentor::runBeforePipeline(
1042 std::optional<OperationName> name,
1043 const PassInstrumentation::PipelineParentInfo &parentInfo) {
1044 llvm::sys::SmartScopedLock<true> instrumentationLock(impl->mutex);
1045 for (auto &instr : impl->instrumentations)
1046 instr->runBeforePipeline(name, parentInfo);
1047}
1048
1049/// See PassInstrumentation::runAfterPipeline for details.
1050void PassInstrumentor::runAfterPipeline(
1051 std::optional<OperationName> name,
1052 const PassInstrumentation::PipelineParentInfo &parentInfo) {
1053 llvm::sys::SmartScopedLock<true> instrumentationLock(impl->mutex);
1054 for (auto &instr : llvm::reverse(C&: impl->instrumentations))
1055 instr->runAfterPipeline(name, parentInfo);
1056}
1057
1058/// See PassInstrumentation::runBeforePass for details.
1059void PassInstrumentor::runBeforePass(Pass *pass, Operation *op) {
1060 llvm::sys::SmartScopedLock<true> instrumentationLock(impl->mutex);
1061 for (auto &instr : impl->instrumentations)
1062 instr->runBeforePass(pass, op);
1063}
1064
1065/// See PassInstrumentation::runAfterPass for details.
1066void PassInstrumentor::runAfterPass(Pass *pass, Operation *op) {
1067 llvm::sys::SmartScopedLock<true> instrumentationLock(impl->mutex);
1068 for (auto &instr : llvm::reverse(C&: impl->instrumentations))
1069 instr->runAfterPass(pass, op);
1070}
1071
1072/// See PassInstrumentation::runAfterPassFailed for details.
1073void PassInstrumentor::runAfterPassFailed(Pass *pass, Operation *op) {
1074 llvm::sys::SmartScopedLock<true> instrumentationLock(impl->mutex);
1075 for (auto &instr : llvm::reverse(C&: impl->instrumentations))
1076 instr->runAfterPassFailed(pass, op);
1077}
1078
1079/// See PassInstrumentation::runBeforeAnalysis for details.
1080void PassInstrumentor::runBeforeAnalysis(StringRef name, TypeID id,
1081 Operation *op) {
1082 llvm::sys::SmartScopedLock<true> instrumentationLock(impl->mutex);
1083 for (auto &instr : impl->instrumentations)
1084 instr->runBeforeAnalysis(name, id, op);
1085}
1086
1087/// See PassInstrumentation::runAfterAnalysis for details.
1088void PassInstrumentor::runAfterAnalysis(StringRef name, TypeID id,
1089 Operation *op) {
1090 llvm::sys::SmartScopedLock<true> instrumentationLock(impl->mutex);
1091 for (auto &instr : llvm::reverse(C&: impl->instrumentations))
1092 instr->runAfterAnalysis(name, id, op);
1093}
1094
1095/// Add the given instrumentation to the collection.
1096void PassInstrumentor::addInstrumentation(
1097 std::unique_ptr<PassInstrumentation> pi) {
1098 llvm::sys::SmartScopedLock<true> instrumentationLock(impl->mutex);
1099 impl->instrumentations.emplace_back(args: std::move(pi));
1100}
1101

source code of mlir/lib/Pass/Pass.cpp