1 | //===- Construction of codegen pass pipelines ------------------*- C++ -*--===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | /// \file |
9 | /// |
10 | /// Interfaces for producing common pass manager configurations. |
11 | /// |
12 | //===----------------------------------------------------------------------===// |
13 | |
14 | #ifndef LLVM_PASSES_CODEGENPASSBUILDER_H |
15 | #define LLVM_PASSES_CODEGENPASSBUILDER_H |
16 | |
17 | #include "llvm/ADT/SmallVector.h" |
18 | #include "llvm/ADT/StringRef.h" |
19 | #include "llvm/Analysis/AliasAnalysis.h" |
20 | #include "llvm/Analysis/BasicAliasAnalysis.h" |
21 | #include "llvm/Analysis/ProfileSummaryInfo.h" |
22 | #include "llvm/Analysis/ScopedNoAliasAA.h" |
23 | #include "llvm/Analysis/TargetTransformInfo.h" |
24 | #include "llvm/Analysis/TypeBasedAliasAnalysis.h" |
25 | #include "llvm/CodeGen/AssignmentTrackingAnalysis.h" |
26 | #include "llvm/CodeGen/CallBrPrepare.h" |
27 | #include "llvm/CodeGen/CodeGenPrepare.h" |
28 | #include "llvm/CodeGen/DeadMachineInstructionElim.h" |
29 | #include "llvm/CodeGen/DwarfEHPrepare.h" |
30 | #include "llvm/CodeGen/ExpandMemCmp.h" |
31 | #include "llvm/CodeGen/ExpandReductions.h" |
32 | #include "llvm/CodeGen/FreeMachineFunction.h" |
33 | #include "llvm/CodeGen/GCMetadata.h" |
34 | #include "llvm/CodeGen/GlobalMerge.h" |
35 | #include "llvm/CodeGen/IndirectBrExpand.h" |
36 | #include "llvm/CodeGen/InterleavedAccess.h" |
37 | #include "llvm/CodeGen/InterleavedLoadCombine.h" |
38 | #include "llvm/CodeGen/JMCInstrumenter.h" |
39 | #include "llvm/CodeGen/LowerEmuTLS.h" |
40 | #include "llvm/CodeGen/MIRPrinter.h" |
41 | #include "llvm/CodeGen/MachinePassManager.h" |
42 | #include "llvm/CodeGen/PreISelIntrinsicLowering.h" |
43 | #include "llvm/CodeGen/ReplaceWithVeclib.h" |
44 | #include "llvm/CodeGen/SafeStack.h" |
45 | #include "llvm/CodeGen/SelectOptimize.h" |
46 | #include "llvm/CodeGen/ShadowStackGCLowering.h" |
47 | #include "llvm/CodeGen/SjLjEHPrepare.h" |
48 | #include "llvm/CodeGen/StackProtector.h" |
49 | #include "llvm/CodeGen/TargetPassConfig.h" |
50 | #include "llvm/CodeGen/UnreachableBlockElim.h" |
51 | #include "llvm/CodeGen/WasmEHPrepare.h" |
52 | #include "llvm/CodeGen/WinEHPrepare.h" |
53 | #include "llvm/IR/PassManager.h" |
54 | #include "llvm/IR/Verifier.h" |
55 | #include "llvm/IRPrinter/IRPrintingPasses.h" |
56 | #include "llvm/MC/MCAsmInfo.h" |
57 | #include "llvm/MC/MCTargetOptions.h" |
58 | #include "llvm/Support/CodeGen.h" |
59 | #include "llvm/Support/Debug.h" |
60 | #include "llvm/Support/Error.h" |
61 | #include "llvm/Support/ErrorHandling.h" |
62 | #include "llvm/Target/CGPassBuilderOption.h" |
63 | #include "llvm/Target/TargetMachine.h" |
64 | #include "llvm/Transforms/CFGuard.h" |
65 | #include "llvm/Transforms/Scalar/ConstantHoisting.h" |
66 | #include "llvm/Transforms/Scalar/LoopPassManager.h" |
67 | #include "llvm/Transforms/Scalar/LoopStrengthReduce.h" |
68 | #include "llvm/Transforms/Scalar/LowerConstantIntrinsics.h" |
69 | #include "llvm/Transforms/Scalar/MergeICmps.h" |
70 | #include "llvm/Transforms/Scalar/PartiallyInlineLibCalls.h" |
71 | #include "llvm/Transforms/Scalar/ScalarizeMaskedMemIntrin.h" |
72 | #include "llvm/Transforms/Utils/EntryExitInstrumenter.h" |
73 | #include "llvm/Transforms/Utils/LowerInvoke.h" |
74 | #include <cassert> |
75 | #include <type_traits> |
76 | #include <utility> |
77 | |
78 | namespace llvm { |
79 | |
80 | // FIXME: Dummy target independent passes definitions that have not yet been |
81 | // ported to new pass manager. Once they do, remove these. |
82 | #define DUMMY_FUNCTION_PASS(NAME, PASS_NAME) \ |
83 | struct PASS_NAME : public PassInfoMixin<PASS_NAME> { \ |
84 | template <typename... Ts> PASS_NAME(Ts &&...) {} \ |
85 | PreservedAnalyses run(Function &, FunctionAnalysisManager &) { \ |
86 | return PreservedAnalyses::all(); \ |
87 | } \ |
88 | }; |
89 | #define DUMMY_MACHINE_MODULE_PASS(NAME, PASS_NAME) \ |
90 | struct PASS_NAME : public PassInfoMixin<PASS_NAME> { \ |
91 | template <typename... Ts> PASS_NAME(Ts &&...) {} \ |
92 | PreservedAnalyses run(Module &, ModuleAnalysisManager &) { \ |
93 | return PreservedAnalyses::all(); \ |
94 | } \ |
95 | }; |
96 | #define DUMMY_MACHINE_FUNCTION_PASS(NAME, PASS_NAME) \ |
97 | struct PASS_NAME : public PassInfoMixin<PASS_NAME> { \ |
98 | template <typename... Ts> PASS_NAME(Ts &&...) {} \ |
99 | PreservedAnalyses run(MachineFunction &, \ |
100 | MachineFunctionAnalysisManager &) { \ |
101 | return PreservedAnalyses::all(); \ |
102 | } \ |
103 | }; |
104 | #include "llvm/Passes/MachinePassRegistry.def" |
105 | |
106 | /// This class provides access to building LLVM's passes. |
107 | /// |
108 | /// Its members provide the baseline state available to passes during their |
109 | /// construction. The \c MachinePassRegistry.def file specifies how to construct |
110 | /// all of the built-in passes, and those may reference these members during |
111 | /// construction. |
112 | template <typename DerivedT, typename TargetMachineT> class CodeGenPassBuilder { |
113 | public: |
114 | explicit CodeGenPassBuilder(TargetMachineT &TM, |
115 | const CGPassBuilderOption &Opts, |
116 | PassInstrumentationCallbacks *PIC) |
117 | : TM(TM), Opt(Opts), PIC(PIC) { |
118 | // Target could set CGPassBuilderOption::MISchedPostRA to true to achieve |
119 | // substitutePass(&PostRASchedulerID, &PostMachineSchedulerID) |
120 | |
121 | // Target should override TM.Options.EnableIPRA in their target-specific |
122 | // LLVMTM ctor. See TargetMachine::setGlobalISel for example. |
123 | if (Opt.EnableIPRA) |
124 | TM.Options.EnableIPRA = *Opt.EnableIPRA; |
125 | |
126 | if (Opt.EnableGlobalISelAbort) |
127 | TM.Options.GlobalISelAbort = *Opt.EnableGlobalISelAbort; |
128 | |
129 | if (!Opt.OptimizeRegAlloc) |
130 | Opt.OptimizeRegAlloc = getOptLevel() != CodeGenOptLevel::None; |
131 | } |
132 | |
133 | Error buildPipeline(ModulePassManager &MPM, raw_pwrite_stream &Out, |
134 | raw_pwrite_stream *DwoOut, |
135 | CodeGenFileType FileType) const; |
136 | |
137 | PassInstrumentationCallbacks *getPassInstrumentationCallbacks() const { |
138 | return PIC; |
139 | } |
140 | |
141 | protected: |
142 | template <typename PassT> |
143 | using is_module_pass_t = decltype(std::declval<PassT &>().run( |
144 | std::declval<Module &>(), std::declval<ModuleAnalysisManager &>())); |
145 | |
146 | template <typename PassT> |
147 | using is_function_pass_t = decltype(std::declval<PassT &>().run( |
148 | std::declval<Function &>(), std::declval<FunctionAnalysisManager &>())); |
149 | |
150 | template <typename PassT> |
151 | using is_machine_function_pass_t = decltype(std::declval<PassT &>().run( |
152 | std::declval<MachineFunction &>(), |
153 | std::declval<MachineFunctionAnalysisManager &>())); |
154 | |
155 | // Function object to maintain state while adding codegen IR passes. |
156 | // TODO: add a Function -> MachineFunction adaptor and merge |
157 | // AddIRPass/AddMachinePass so we can have a function pipeline that runs both |
158 | // function passes and machine function passes. |
159 | class AddIRPass { |
160 | public: |
161 | AddIRPass(ModulePassManager &MPM, const DerivedT &PB) : MPM(MPM), PB(PB) {} |
162 | ~AddIRPass() { |
163 | if (!FPM.isEmpty()) |
164 | MPM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: std::move(FPM))); |
165 | } |
166 | |
167 | template <typename PassT> |
168 | void operator()(PassT &&Pass, StringRef Name = PassT::name()) { |
169 | static_assert((is_detected<is_function_pass_t, PassT>::value || |
170 | is_detected<is_module_pass_t, PassT>::value) && |
171 | "Only module pass and function pass are supported." ); |
172 | |
173 | if (!PB.runBeforeAdding(Name)) |
174 | return; |
175 | |
176 | // Add Function Pass |
177 | if constexpr (is_detected<is_function_pass_t, PassT>::value) { |
178 | FPM.addPass(std::forward<PassT>(Pass)); |
179 | } else { |
180 | // Add Module Pass |
181 | if (!FPM.isEmpty()) { |
182 | MPM.addPass(Pass: createModuleToFunctionPassAdaptor(Pass: std::move(FPM))); |
183 | FPM = FunctionPassManager(); |
184 | } |
185 | |
186 | MPM.addPass(std::forward<PassT>(Pass)); |
187 | } |
188 | } |
189 | |
190 | private: |
191 | ModulePassManager &MPM; |
192 | FunctionPassManager FPM; |
193 | const DerivedT &PB; |
194 | }; |
195 | |
196 | // Function object to maintain state while adding codegen machine passes. |
197 | class AddMachinePass { |
198 | public: |
199 | AddMachinePass(ModulePassManager &MPM, const DerivedT &PB) |
200 | : MPM(MPM), PB(PB) {} |
201 | ~AddMachinePass() { |
202 | if (!MFPM.isEmpty()) |
203 | MPM.addPass(Pass: createModuleToMachineFunctionPassAdaptor(Pass: std::move(MFPM))); |
204 | } |
205 | |
206 | template <typename PassT> |
207 | void operator()(PassT &&Pass, bool Force = false, |
208 | StringRef Name = PassT::name()) { |
209 | static_assert((is_detected<is_machine_function_pass_t, PassT>::value || |
210 | is_detected<is_module_pass_t, PassT>::value) && |
211 | "Only module pass and function pass are supported." ); |
212 | |
213 | if (!Force && !PB.runBeforeAdding(Name)) |
214 | return; |
215 | |
216 | // Add Function Pass |
217 | if constexpr (is_detected<is_machine_function_pass_t, PassT>::value) { |
218 | MFPM.addPass(std::forward<PassT>(Pass)); |
219 | } else { |
220 | // Add Module Pass |
221 | if (!MFPM.isEmpty()) { |
222 | MPM.addPass( |
223 | Pass: createModuleToMachineFunctionPassAdaptor(Pass: std::move(MFPM))); |
224 | MFPM = MachineFunctionPassManager(); |
225 | } |
226 | |
227 | MPM.addPass(std::forward<PassT>(Pass)); |
228 | } |
229 | |
230 | for (auto &C : PB.AfterCallbacks) |
231 | C(Name, MFPM); |
232 | } |
233 | |
234 | private: |
235 | ModulePassManager &MPM; |
236 | MachineFunctionPassManager MFPM; |
237 | const DerivedT &PB; |
238 | }; |
239 | |
240 | TargetMachineT &TM; |
241 | CGPassBuilderOption Opt; |
242 | PassInstrumentationCallbacks *PIC; |
243 | |
244 | template <typename TMC> TMC &getTM() const { return static_cast<TMC &>(TM); } |
245 | CodeGenOptLevel getOptLevel() const { return TM.getOptLevel(); } |
246 | |
247 | /// Check whether or not GlobalISel should abort on error. |
248 | /// When this is disabled, GlobalISel will fall back on SDISel instead of |
249 | /// erroring out. |
250 | bool isGlobalISelAbortEnabled() const { |
251 | return TM.Options.GlobalISelAbort == GlobalISelAbortMode::Enable; |
252 | } |
253 | |
254 | /// Check whether or not a diagnostic should be emitted when GlobalISel |
255 | /// uses the fallback path. In other words, it will emit a diagnostic |
256 | /// when GlobalISel failed and isGlobalISelAbortEnabled is false. |
257 | bool reportDiagnosticWhenGlobalISelFallback() const { |
258 | return TM.Options.GlobalISelAbort == GlobalISelAbortMode::DisableWithDiag; |
259 | } |
260 | |
261 | /// addInstSelector - This method should install an instruction selector pass, |
262 | /// which converts from LLVM code to machine instructions. |
263 | Error addInstSelector(AddMachinePass &) const { |
264 | return make_error<StringError>(Args: "addInstSelector is not overridden" , |
265 | Args: inconvertibleErrorCode()); |
266 | } |
267 | |
268 | /// Target can override this to add GlobalMergePass before all IR passes. |
269 | void addGlobalMergePass(AddIRPass &) const {} |
270 | |
271 | /// Add passes that optimize instruction level parallelism for out-of-order |
272 | /// targets. These passes are run while the machine code is still in SSA |
273 | /// form, so they can use MachineTraceMetrics to control their heuristics. |
274 | /// |
275 | /// All passes added here should preserve the MachineDominatorTree, |
276 | /// MachineLoopInfo, and MachineTraceMetrics analyses. |
277 | void addILPOpts(AddMachinePass &) const {} |
278 | |
279 | /// This method may be implemented by targets that want to run passes |
280 | /// immediately before register allocation. |
281 | void addPreRegAlloc(AddMachinePass &) const {} |
282 | |
283 | /// addPreRewrite - Add passes to the optimized register allocation pipeline |
284 | /// after register allocation is complete, but before virtual registers are |
285 | /// rewritten to physical registers. |
286 | /// |
287 | /// These passes must preserve VirtRegMap and LiveIntervals, and when running |
288 | /// after RABasic or RAGreedy, they should take advantage of LiveRegMatrix. |
289 | /// When these passes run, VirtRegMap contains legal physreg assignments for |
290 | /// all virtual registers. |
291 | /// |
292 | /// Note if the target overloads addRegAssignAndRewriteOptimized, this may not |
293 | /// be honored. This is also not generally used for the fast variant, |
294 | /// where the allocation and rewriting are done in one pass. |
295 | void addPreRewrite(AddMachinePass &) const {} |
296 | |
297 | /// Add passes to be run immediately after virtual registers are rewritten |
298 | /// to physical registers. |
299 | void addPostRewrite(AddMachinePass &) const {} |
300 | |
301 | /// This method may be implemented by targets that want to run passes after |
302 | /// register allocation pass pipeline but before prolog-epilog insertion. |
303 | void addPostRegAlloc(AddMachinePass &) const {} |
304 | |
305 | /// This method may be implemented by targets that want to run passes after |
306 | /// prolog-epilog insertion and before the second instruction scheduling pass. |
307 | void addPreSched2(AddMachinePass &) const {} |
308 | |
309 | /// This pass may be implemented by targets that want to run passes |
310 | /// immediately before machine code is emitted. |
311 | void addPreEmitPass(AddMachinePass &) const {} |
312 | |
313 | /// Targets may add passes immediately before machine code is emitted in this |
314 | /// callback. This is called even later than `addPreEmitPass`. |
315 | // FIXME: Rename `addPreEmitPass` to something more sensible given its actual |
316 | // position and remove the `2` suffix here as this callback is what |
317 | // `addPreEmitPass` *should* be but in reality isn't. |
318 | void addPreEmitPass2(AddMachinePass &) const {} |
319 | |
320 | /// {{@ For GlobalISel |
321 | /// |
322 | |
323 | /// addPreISel - This method should add any "last minute" LLVM->LLVM |
324 | /// passes (which are run just before instruction selector). |
325 | void addPreISel(AddIRPass &) const { |
326 | llvm_unreachable("addPreISel is not overridden" ); |
327 | } |
328 | |
329 | /// This method should install an IR translator pass, which converts from |
330 | /// LLVM code to machine instructions with possibly generic opcodes. |
331 | Error addIRTranslator(AddMachinePass &) const { |
332 | return make_error<StringError>(Args: "addIRTranslator is not overridden" , |
333 | Args: inconvertibleErrorCode()); |
334 | } |
335 | |
336 | /// This method may be implemented by targets that want to run passes |
337 | /// immediately before legalization. |
338 | void addPreLegalizeMachineIR(AddMachinePass &) const {} |
339 | |
340 | /// This method should install a legalize pass, which converts the instruction |
341 | /// sequence into one that can be selected by the target. |
342 | Error addLegalizeMachineIR(AddMachinePass &) const { |
343 | return make_error<StringError>(Args: "addLegalizeMachineIR is not overridden" , |
344 | Args: inconvertibleErrorCode()); |
345 | } |
346 | |
347 | /// This method may be implemented by targets that want to run passes |
348 | /// immediately before the register bank selection. |
349 | void addPreRegBankSelect(AddMachinePass &) const {} |
350 | |
351 | /// This method should install a register bank selector pass, which |
352 | /// assigns register banks to virtual registers without a register |
353 | /// class or register banks. |
354 | Error addRegBankSelect(AddMachinePass &) const { |
355 | return make_error<StringError>(Args: "addRegBankSelect is not overridden" , |
356 | Args: inconvertibleErrorCode()); |
357 | } |
358 | |
359 | /// This method may be implemented by targets that want to run passes |
360 | /// immediately before the (global) instruction selection. |
361 | void addPreGlobalInstructionSelect(AddMachinePass &) const {} |
362 | |
363 | /// This method should install a (global) instruction selector pass, which |
364 | /// converts possibly generic instructions to fully target-specific |
365 | /// instructions, thereby constraining all generic virtual registers to |
366 | /// register classes. |
367 | Error addGlobalInstructionSelect(AddMachinePass &) const { |
368 | return make_error<StringError>( |
369 | Args: "addGlobalInstructionSelect is not overridden" , |
370 | Args: inconvertibleErrorCode()); |
371 | } |
372 | /// @}} |
373 | |
374 | /// High level function that adds all passes necessary to go from llvm IR |
375 | /// representation to the MI representation. |
376 | /// Adds IR based lowering and target specific optimization passes and finally |
377 | /// the core instruction selection passes. |
378 | void addISelPasses(AddIRPass &) const; |
379 | |
380 | /// Add the actual instruction selection passes. This does not include |
381 | /// preparation passes on IR. |
382 | Error addCoreISelPasses(AddMachinePass &) const; |
383 | |
384 | /// Add the complete, standard set of LLVM CodeGen passes. |
385 | /// Fully developed targets will not generally override this. |
386 | Error addMachinePasses(AddMachinePass &) const; |
387 | |
388 | /// Add passes to lower exception handling for the code generator. |
389 | void addPassesToHandleExceptions(AddIRPass &) const; |
390 | |
391 | /// Add common target configurable passes that perform LLVM IR to IR |
392 | /// transforms following machine independent optimization. |
393 | void addIRPasses(AddIRPass &) const; |
394 | |
395 | /// Add pass to prepare the LLVM IR for code generation. This should be done |
396 | /// before exception handling preparation passes. |
397 | void addCodeGenPrepare(AddIRPass &) const; |
398 | |
399 | /// Add common passes that perform LLVM IR to IR transforms in preparation for |
400 | /// instruction selection. |
401 | void addISelPrepare(AddIRPass &) const; |
402 | |
403 | /// Methods with trivial inline returns are convenient points in the common |
404 | /// codegen pass pipeline where targets may insert passes. Methods with |
405 | /// out-of-line standard implementations are major CodeGen stages called by |
406 | /// addMachinePasses. Some targets may override major stages when inserting |
407 | /// passes is insufficient, but maintaining overriden stages is more work. |
408 | /// |
409 | |
410 | /// addMachineSSAOptimization - Add standard passes that optimize machine |
411 | /// instructions in SSA form. |
412 | void addMachineSSAOptimization(AddMachinePass &) const; |
413 | |
414 | /// addFastRegAlloc - Add the minimum set of target-independent passes that |
415 | /// are required for fast register allocation. |
416 | Error addFastRegAlloc(AddMachinePass &) const; |
417 | |
418 | /// addOptimizedRegAlloc - Add passes related to register allocation. |
419 | /// LLVMTargetMachine provides standard regalloc passes for most targets. |
420 | void addOptimizedRegAlloc(AddMachinePass &) const; |
421 | |
422 | /// Add passes that optimize machine instructions after register allocation. |
423 | void addMachineLateOptimization(AddMachinePass &) const; |
424 | |
425 | /// addGCPasses - Add late codegen passes that analyze code for garbage |
426 | /// collection. This should return true if GC info should be printed after |
427 | /// these passes. |
428 | void addGCPasses(AddMachinePass &) const {} |
429 | |
430 | /// Add standard basic block placement passes. |
431 | void addBlockPlacement(AddMachinePass &) const; |
432 | |
433 | using CreateMCStreamer = |
434 | std::function<Expected<std::unique_ptr<MCStreamer>>(MCContext &)>; |
435 | void addAsmPrinter(AddMachinePass &, CreateMCStreamer) const { |
436 | llvm_unreachable("addAsmPrinter is not overridden" ); |
437 | } |
438 | |
439 | /// Utilities for targets to add passes to the pass manager. |
440 | /// |
441 | |
442 | /// createTargetRegisterAllocator - Create the register allocator pass for |
443 | /// this target at the current optimization level. |
444 | void addTargetRegisterAllocator(AddMachinePass &, bool Optimized) const; |
445 | |
446 | /// addMachinePasses helper to create the target-selected or overriden |
447 | /// regalloc pass. |
448 | void addRegAllocPass(AddMachinePass &, bool Optimized) const; |
449 | |
450 | /// Add core register alloator passes which do the actual register assignment |
451 | /// and rewriting. \returns true if any passes were added. |
452 | Error addRegAssignmentFast(AddMachinePass &) const; |
453 | Error addRegAssignmentOptimized(AddMachinePass &) const; |
454 | |
455 | /// Allow the target to disable a specific pass by default. |
456 | /// Backend can declare unwanted passes in constructor. |
457 | template <typename... PassTs> void disablePass() { |
458 | BeforeCallbacks.emplace_back( |
459 | [](StringRef Name) { return ((Name != PassTs::name()) && ...); }); |
460 | } |
461 | |
462 | /// Insert InsertedPass pass after TargetPass pass. |
463 | /// Only machine function passes are supported. |
464 | template <typename TargetPassT, typename InsertedPassT> |
465 | void insertPass(InsertedPassT &&Pass) { |
466 | AfterCallbacks.emplace_back( |
467 | [&](StringRef Name, MachineFunctionPassManager &MFPM) mutable { |
468 | if (Name == TargetPassT::name()) |
469 | MFPM.addPass(std::forward<InsertedPassT>(Pass)); |
470 | }); |
471 | } |
472 | |
473 | private: |
474 | DerivedT &derived() { return static_cast<DerivedT &>(*this); } |
475 | const DerivedT &derived() const { |
476 | return static_cast<const DerivedT &>(*this); |
477 | } |
478 | |
479 | bool runBeforeAdding(StringRef Name) const { |
480 | bool ShouldAdd = true; |
481 | for (auto &C : BeforeCallbacks) |
482 | ShouldAdd &= C(Name); |
483 | return ShouldAdd; |
484 | } |
485 | |
486 | void setStartStopPasses(const TargetPassConfig::StartStopInfo &Info) const; |
487 | |
488 | Error verifyStartStop(const TargetPassConfig::StartStopInfo &Info) const; |
489 | |
490 | mutable SmallVector<llvm::unique_function<bool(StringRef)>, 4> |
491 | BeforeCallbacks; |
492 | mutable SmallVector< |
493 | llvm::unique_function<void(StringRef, MachineFunctionPassManager &)>, 4> |
494 | AfterCallbacks; |
495 | |
496 | /// Helper variable for `-start-before/-start-after/-stop-before/-stop-after` |
497 | mutable bool Started = true; |
498 | mutable bool Stopped = true; |
499 | }; |
500 | |
501 | template <typename Derived, typename TargetMachineT> |
502 | Error CodeGenPassBuilder<Derived, TargetMachineT>::buildPipeline( |
503 | ModulePassManager &MPM, raw_pwrite_stream &Out, raw_pwrite_stream *DwoOut, |
504 | CodeGenFileType FileType) const { |
505 | auto StartStopInfo = TargetPassConfig::getStartStopInfo(PIC&: *PIC); |
506 | if (!StartStopInfo) |
507 | return StartStopInfo.takeError(); |
508 | setStartStopPasses(*StartStopInfo); |
509 | |
510 | bool PrintAsm = TargetPassConfig::willCompleteCodeGenPipeline(); |
511 | bool PrintMIR = !PrintAsm && FileType != CodeGenFileType::Null; |
512 | |
513 | { |
514 | AddIRPass addIRPass(MPM, derived()); |
515 | addIRPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>()); |
516 | addIRPass(RequireAnalysisPass<CollectorMetadataAnalysis, Module>()); |
517 | addISelPasses(addIRPass); |
518 | } |
519 | |
520 | AddMachinePass addPass(MPM, derived()); |
521 | |
522 | if (PrintMIR) |
523 | addPass(PrintMIRPreparePass(Out), /*Force=*/true); |
524 | |
525 | if (auto Err = addCoreISelPasses(addPass)) |
526 | return std::move(Err); |
527 | |
528 | if (auto Err = derived().addMachinePasses(addPass)) |
529 | return std::move(Err); |
530 | |
531 | if (PrintAsm) { |
532 | derived().addAsmPrinter( |
533 | addPass, [this, &Out, DwoOut, FileType](MCContext &Ctx) { |
534 | return this->TM.createMCStreamer(Out, DwoOut, FileType, Ctx); |
535 | }); |
536 | } |
537 | |
538 | if (PrintMIR) |
539 | addPass(PrintMIRPass(Out), /*Force=*/true); |
540 | |
541 | addPass(FreeMachineFunctionPass()); |
542 | return verifyStartStop(Info: *StartStopInfo); |
543 | } |
544 | |
545 | template <typename Derived, typename TargetMachineT> |
546 | void CodeGenPassBuilder<Derived, TargetMachineT>::setStartStopPasses( |
547 | const TargetPassConfig::StartStopInfo &Info) const { |
548 | if (!Info.StartPass.empty()) { |
549 | Started = false; |
550 | BeforeCallbacks.emplace_back([this, &Info, AfterFlag = Info.StartAfter, |
551 | Count = 0u](StringRef ClassName) mutable { |
552 | if (Count == Info.StartInstanceNum) { |
553 | if (AfterFlag) { |
554 | AfterFlag = false; |
555 | Started = true; |
556 | } |
557 | return Started; |
558 | } |
559 | |
560 | auto PassName = PIC->getPassNameForClassName(ClassName); |
561 | if (Info.StartPass == PassName && ++Count == Info.StartInstanceNum) |
562 | Started = !Info.StartAfter; |
563 | |
564 | return Started; |
565 | }); |
566 | } |
567 | |
568 | if (!Info.StopPass.empty()) { |
569 | Stopped = false; |
570 | BeforeCallbacks.emplace_back([this, &Info, AfterFlag = Info.StopAfter, |
571 | Count = 0u](StringRef ClassName) mutable { |
572 | if (Count == Info.StopInstanceNum) { |
573 | if (AfterFlag) { |
574 | AfterFlag = false; |
575 | Stopped = true; |
576 | } |
577 | return !Stopped; |
578 | } |
579 | |
580 | auto PassName = PIC->getPassNameForClassName(ClassName); |
581 | if (Info.StopPass == PassName && ++Count == Info.StopInstanceNum) |
582 | Stopped = !Info.StopAfter; |
583 | return !Stopped; |
584 | }); |
585 | } |
586 | } |
587 | |
588 | template <typename Derived, typename TargetMachineT> |
589 | Error CodeGenPassBuilder<Derived, TargetMachineT>::verifyStartStop( |
590 | const TargetPassConfig::StartStopInfo &Info) const { |
591 | if (Started && Stopped) |
592 | return Error::success(); |
593 | |
594 | if (!Started) |
595 | return make_error<StringError>( |
596 | Args: "Can't find start pass \"" + |
597 | PIC->getPassNameForClassName(ClassName: Info.StartPass) + "\"." , |
598 | Args: std::make_error_code(e: std::errc::invalid_argument)); |
599 | if (!Stopped) |
600 | return make_error<StringError>( |
601 | Args: "Can't find stop pass \"" + |
602 | PIC->getPassNameForClassName(ClassName: Info.StopPass) + "\"." , |
603 | Args: std::make_error_code(e: std::errc::invalid_argument)); |
604 | return Error::success(); |
605 | } |
606 | |
607 | template <typename Derived, typename TargetMachineT> |
608 | void CodeGenPassBuilder<Derived, TargetMachineT>::addISelPasses( |
609 | AddIRPass &addPass) const { |
610 | derived().addGlobalMergePass(addPass); |
611 | if (TM.useEmulatedTLS()) |
612 | addPass(LowerEmuTLSPass()); |
613 | |
614 | addPass(PreISelIntrinsicLoweringPass(TM)); |
615 | |
616 | derived().addIRPasses(addPass); |
617 | derived().addCodeGenPrepare(addPass); |
618 | addPassesToHandleExceptions(addPass); |
619 | derived().addISelPrepare(addPass); |
620 | } |
621 | |
622 | /// Add common target configurable passes that perform LLVM IR to IR transforms |
623 | /// following machine independent optimization. |
624 | template <typename Derived, typename TargetMachineT> |
625 | void CodeGenPassBuilder<Derived, TargetMachineT>::addIRPasses( |
626 | AddIRPass &addPass) const { |
627 | // Before running any passes, run the verifier to determine if the input |
628 | // coming from the front-end and/or optimizer is valid. |
629 | if (!Opt.DisableVerify) |
630 | addPass(VerifierPass()); |
631 | |
632 | // Run loop strength reduction before anything else. |
633 | if (getOptLevel() != CodeGenOptLevel::None && !Opt.DisableLSR) { |
634 | addPass(createFunctionToLoopPassAdaptor(Pass: LoopStrengthReducePass(), |
635 | /*UseMemorySSA=*/UseMemorySSA: true)); |
636 | // FIXME: use -stop-after so we could remove PrintLSR |
637 | if (Opt.PrintLSR) |
638 | addPass(PrintFunctionPass(dbgs(), "\n\n*** Code after LSR ***\n" )); |
639 | } |
640 | |
641 | if (getOptLevel() != CodeGenOptLevel::None) { |
642 | // The MergeICmpsPass tries to create memcmp calls by grouping sequences of |
643 | // loads and compares. ExpandMemCmpPass then tries to expand those calls |
644 | // into optimally-sized loads and compares. The transforms are enabled by a |
645 | // target lowering hook. |
646 | if (!Opt.DisableMergeICmps) |
647 | addPass(MergeICmpsPass()); |
648 | addPass(ExpandMemCmpPass(&TM)); |
649 | } |
650 | |
651 | // Run GC lowering passes for builtin collectors |
652 | // TODO: add a pass insertion point here |
653 | addPass(GCLoweringPass()); |
654 | addPass(ShadowStackGCLoweringPass()); |
655 | addPass(LowerConstantIntrinsicsPass()); |
656 | |
657 | // Make sure that no unreachable blocks are instruction selected. |
658 | addPass(UnreachableBlockElimPass()); |
659 | |
660 | // Prepare expensive constants for SelectionDAG. |
661 | if (getOptLevel() != CodeGenOptLevel::None && !Opt.DisableConstantHoisting) |
662 | addPass(ConstantHoistingPass()); |
663 | |
664 | // Replace calls to LLVM intrinsics (e.g., exp, log) operating on vector |
665 | // operands with calls to the corresponding functions in a vector library. |
666 | if (getOptLevel() != CodeGenOptLevel::None) |
667 | addPass(ReplaceWithVeclib()); |
668 | |
669 | if (getOptLevel() != CodeGenOptLevel::None && |
670 | !Opt.DisablePartialLibcallInlining) |
671 | addPass(PartiallyInlineLibCallsPass()); |
672 | |
673 | // Instrument function entry and exit, e.g. with calls to mcount(). |
674 | addPass(EntryExitInstrumenterPass(/*PostInlining=*/true)); |
675 | |
676 | // Add scalarization of target's unsupported masked memory intrinsics pass. |
677 | // the unsupported intrinsic will be replaced with a chain of basic blocks, |
678 | // that stores/loads element one-by-one if the appropriate mask bit is set. |
679 | addPass(ScalarizeMaskedMemIntrinPass()); |
680 | |
681 | // Expand reduction intrinsics into shuffle sequences if the target wants to. |
682 | addPass(ExpandReductionsPass()); |
683 | |
684 | // Convert conditional moves to conditional jumps when profitable. |
685 | if (getOptLevel() != CodeGenOptLevel::None && !Opt.DisableSelectOptimize) |
686 | addPass(SelectOptimizePass(&TM)); |
687 | } |
688 | |
689 | /// Turn exception handling constructs into something the code generators can |
690 | /// handle. |
691 | template <typename Derived, typename TargetMachineT> |
692 | void CodeGenPassBuilder<Derived, TargetMachineT>::addPassesToHandleExceptions( |
693 | AddIRPass &addPass) const { |
694 | const MCAsmInfo *MCAI = TM.getMCAsmInfo(); |
695 | assert(MCAI && "No MCAsmInfo" ); |
696 | switch (MCAI->getExceptionHandlingType()) { |
697 | case ExceptionHandling::SjLj: |
698 | // SjLj piggy-backs on dwarf for this bit. The cleanups done apply to both |
699 | // Dwarf EH prepare needs to be run after SjLj prepare. Otherwise, |
700 | // catch info can get misplaced when a selector ends up more than one block |
701 | // removed from the parent invoke(s). This could happen when a landing |
702 | // pad is shared by multiple invokes and is also a target of a normal |
703 | // edge from elsewhere. |
704 | addPass(SjLjEHPreparePass(&TM)); |
705 | [[fallthrough]]; |
706 | case ExceptionHandling::DwarfCFI: |
707 | case ExceptionHandling::ARM: |
708 | case ExceptionHandling::AIX: |
709 | case ExceptionHandling::ZOS: |
710 | addPass(DwarfEHPreparePass(&TM)); |
711 | break; |
712 | case ExceptionHandling::WinEH: |
713 | // We support using both GCC-style and MSVC-style exceptions on Windows, so |
714 | // add both preparation passes. Each pass will only actually run if it |
715 | // recognizes the personality function. |
716 | addPass(WinEHPreparePass()); |
717 | addPass(DwarfEHPreparePass(&TM)); |
718 | break; |
719 | case ExceptionHandling::Wasm: |
720 | // Wasm EH uses Windows EH instructions, but it does not need to demote PHIs |
721 | // on catchpads and cleanuppads because it does not outline them into |
722 | // funclets. Catchswitch blocks are not lowered in SelectionDAG, so we |
723 | // should remove PHIs there. |
724 | addPass(WinEHPreparePass(/*DemoteCatchSwitchPHIOnly=*/false)); |
725 | addPass(WasmEHPreparePass()); |
726 | break; |
727 | case ExceptionHandling::None: |
728 | addPass(LowerInvokePass()); |
729 | |
730 | // The lower invoke pass may create unreachable code. Remove it. |
731 | addPass(UnreachableBlockElimPass()); |
732 | break; |
733 | } |
734 | } |
735 | |
736 | /// Add pass to prepare the LLVM IR for code generation. This should be done |
737 | /// before exception handling preparation passes. |
738 | template <typename Derived, typename TargetMachineT> |
739 | void CodeGenPassBuilder<Derived, TargetMachineT>::addCodeGenPrepare( |
740 | AddIRPass &addPass) const { |
741 | if (getOptLevel() != CodeGenOptLevel::None && !Opt.DisableCGP) |
742 | addPass(CodeGenPreparePass(&TM)); |
743 | // TODO: Default ctor'd RewriteSymbolPass is no-op. |
744 | // addPass(RewriteSymbolPass()); |
745 | } |
746 | |
747 | /// Add common passes that perform LLVM IR to IR transforms in preparation for |
748 | /// instruction selection. |
749 | template <typename Derived, typename TargetMachineT> |
750 | void CodeGenPassBuilder<Derived, TargetMachineT>::addISelPrepare( |
751 | AddIRPass &addPass) const { |
752 | derived().addPreISel(addPass); |
753 | |
754 | addPass(CallBrPreparePass()); |
755 | // Add both the safe stack and the stack protection passes: each of them will |
756 | // only protect functions that have corresponding attributes. |
757 | addPass(SafeStackPass(&TM)); |
758 | addPass(StackProtectorPass(&TM)); |
759 | |
760 | if (Opt.PrintISelInput) |
761 | addPass(PrintFunctionPass(dbgs(), |
762 | "\n\n*** Final LLVM Code input to ISel ***\n" )); |
763 | |
764 | // All passes which modify the LLVM IR are now complete; run the verifier |
765 | // to ensure that the IR is valid. |
766 | if (!Opt.DisableVerify) |
767 | addPass(VerifierPass()); |
768 | } |
769 | |
770 | template <typename Derived, typename TargetMachineT> |
771 | Error CodeGenPassBuilder<Derived, TargetMachineT>::addCoreISelPasses( |
772 | AddMachinePass &addPass) const { |
773 | // Enable FastISel with -fast-isel, but allow that to be overridden. |
774 | TM.setO0WantsFastISel(Opt.EnableFastISelOption.value_or(u: true)); |
775 | |
776 | // Determine an instruction selector. |
777 | enum class SelectorType { SelectionDAG, FastISel, GlobalISel }; |
778 | SelectorType Selector; |
779 | |
780 | if (Opt.EnableFastISelOption && *Opt.EnableFastISelOption == true) |
781 | Selector = SelectorType::FastISel; |
782 | else if ((Opt.EnableGlobalISelOption && |
783 | *Opt.EnableGlobalISelOption == true) || |
784 | (TM.Options.EnableGlobalISel && |
785 | (!Opt.EnableGlobalISelOption || |
786 | *Opt.EnableGlobalISelOption == false))) |
787 | Selector = SelectorType::GlobalISel; |
788 | else if (TM.getOptLevel() == CodeGenOptLevel::None && TM.getO0WantsFastISel()) |
789 | Selector = SelectorType::FastISel; |
790 | else |
791 | Selector = SelectorType::SelectionDAG; |
792 | |
793 | // Set consistently TM.Options.EnableFastISel and EnableGlobalISel. |
794 | if (Selector == SelectorType::FastISel) { |
795 | TM.setFastISel(true); |
796 | TM.setGlobalISel(false); |
797 | } else if (Selector == SelectorType::GlobalISel) { |
798 | TM.setFastISel(false); |
799 | TM.setGlobalISel(true); |
800 | } |
801 | |
802 | // Add instruction selector passes. |
803 | if (Selector == SelectorType::GlobalISel) { |
804 | if (auto Err = derived().addIRTranslator(addPass)) |
805 | return std::move(Err); |
806 | |
807 | derived().addPreLegalizeMachineIR(addPass); |
808 | |
809 | if (auto Err = derived().addLegalizeMachineIR(addPass)) |
810 | return std::move(Err); |
811 | |
812 | // Before running the register bank selector, ask the target if it |
813 | // wants to run some passes. |
814 | derived().addPreRegBankSelect(addPass); |
815 | |
816 | if (auto Err = derived().addRegBankSelect(addPass)) |
817 | return std::move(Err); |
818 | |
819 | derived().addPreGlobalInstructionSelect(addPass); |
820 | |
821 | if (auto Err = derived().addGlobalInstructionSelect(addPass)) |
822 | return std::move(Err); |
823 | |
824 | // Pass to reset the MachineFunction if the ISel failed. |
825 | addPass(ResetMachineFunctionPass(reportDiagnosticWhenGlobalISelFallback(), |
826 | isGlobalISelAbortEnabled())); |
827 | |
828 | // Provide a fallback path when we do not want to abort on |
829 | // not-yet-supported input. |
830 | if (!isGlobalISelAbortEnabled()) |
831 | if (auto Err = derived().addInstSelector(addPass)) |
832 | return std::move(Err); |
833 | |
834 | } else if (auto Err = derived().addInstSelector(addPass)) |
835 | return std::move(Err); |
836 | |
837 | // Expand pseudo-instructions emitted by ISel. Don't run the verifier before |
838 | // FinalizeISel. |
839 | addPass(FinalizeISelPass()); |
840 | |
841 | // // Print the instruction selected machine code... |
842 | // printAndVerify("After Instruction Selection"); |
843 | |
844 | return Error::success(); |
845 | } |
846 | |
847 | /// Add the complete set of target-independent postISel code generator passes. |
848 | /// |
849 | /// This can be read as the standard order of major LLVM CodeGen stages. Stages |
850 | /// with nontrivial configuration or multiple passes are broken out below in |
851 | /// add%Stage routines. |
852 | /// |
853 | /// Any CodeGenPassBuilder<Derived, TargetMachine>::addXX routine may be |
854 | /// overriden by the Target. The addPre/Post methods with empty header |
855 | /// implementations allow injecting target-specific fixups just before or after |
856 | /// major stages. Additionally, targets have the flexibility to change pass |
857 | /// order within a stage by overriding default implementation of add%Stage |
858 | /// routines below. Each technique has maintainability tradeoffs because |
859 | /// alternate pass orders are not well supported. addPre/Post works better if |
860 | /// the target pass is easily tied to a common pass. But if it has subtle |
861 | /// dependencies on multiple passes, the target should override the stage |
862 | /// instead. |
863 | template <typename Derived, typename TargetMachineT> |
864 | Error CodeGenPassBuilder<Derived, TargetMachineT>::addMachinePasses( |
865 | AddMachinePass &addPass) const { |
866 | // Add passes that optimize machine instructions in SSA form. |
867 | if (getOptLevel() != CodeGenOptLevel::None) { |
868 | derived().addMachineSSAOptimization(addPass); |
869 | } else { |
870 | // If the target requests it, assign local variables to stack slots relative |
871 | // to one another and simplify frame index references where possible. |
872 | addPass(LocalStackSlotPass()); |
873 | } |
874 | |
875 | if (TM.Options.EnableIPRA) |
876 | addPass(RegUsageInfoPropagationPass()); |
877 | |
878 | // Run pre-ra passes. |
879 | derived().addPreRegAlloc(addPass); |
880 | |
881 | // Run register allocation and passes that are tightly coupled with it, |
882 | // including phi elimination and scheduling. |
883 | if (*Opt.OptimizeRegAlloc) { |
884 | derived().addOptimizedRegAlloc(addPass); |
885 | } else { |
886 | if (auto Err = derived().addFastRegAlloc(addPass)) |
887 | return Err; |
888 | } |
889 | |
890 | // Run post-ra passes. |
891 | derived().addPostRegAlloc(addPass); |
892 | |
893 | addPass(RemoveRedundantDebugValuesPass()); |
894 | |
895 | // Insert prolog/epilog code. Eliminate abstract frame index references... |
896 | if (getOptLevel() != CodeGenOptLevel::None) { |
897 | addPass(PostRAMachineSinkingPass()); |
898 | addPass(ShrinkWrapPass()); |
899 | } |
900 | |
901 | addPass(PrologEpilogInserterPass()); |
902 | |
903 | /// Add passes that optimize machine instructions after register allocation. |
904 | if (getOptLevel() != CodeGenOptLevel::None) |
905 | derived().addMachineLateOptimization(addPass); |
906 | |
907 | // Expand pseudo instructions before second scheduling pass. |
908 | addPass(ExpandPostRAPseudosPass()); |
909 | |
910 | // Run pre-sched2 passes. |
911 | derived().addPreSched2(addPass); |
912 | |
913 | if (Opt.EnableImplicitNullChecks) |
914 | addPass(ImplicitNullChecksPass()); |
915 | |
916 | // Second pass scheduler. |
917 | // Let Target optionally insert this pass by itself at some other |
918 | // point. |
919 | if (getOptLevel() != CodeGenOptLevel::None && |
920 | !TM.targetSchedulesPostRAScheduling()) { |
921 | if (Opt.MISchedPostRA) |
922 | addPass(PostMachineSchedulerPass()); |
923 | else |
924 | addPass(PostRASchedulerPass()); |
925 | } |
926 | |
927 | // GC |
928 | derived().addGCPasses(addPass); |
929 | |
930 | // Basic block placement. |
931 | if (getOptLevel() != CodeGenOptLevel::None) |
932 | derived().addBlockPlacement(addPass); |
933 | |
934 | // Insert before XRay Instrumentation. |
935 | addPass(FEntryInserterPass()); |
936 | |
937 | addPass(XRayInstrumentationPass()); |
938 | addPass(PatchableFunctionPass()); |
939 | |
940 | derived().addPreEmitPass(addPass); |
941 | |
942 | if (TM.Options.EnableIPRA) |
943 | // Collect register usage information and produce a register mask of |
944 | // clobbered registers, to be used to optimize call sites. |
945 | addPass(RegUsageInfoCollectorPass()); |
946 | |
947 | addPass(FuncletLayoutPass()); |
948 | |
949 | addPass(StackMapLivenessPass()); |
950 | addPass(LiveDebugValuesPass()); |
951 | addPass(MachineSanitizerBinaryMetadata()); |
952 | |
953 | if (TM.Options.EnableMachineOutliner && |
954 | getOptLevel() != CodeGenOptLevel::None && |
955 | Opt.EnableMachineOutliner != RunOutliner::NeverOutline) { |
956 | bool RunOnAllFunctions = |
957 | (Opt.EnableMachineOutliner == RunOutliner::AlwaysOutline); |
958 | bool AddOutliner = RunOnAllFunctions || TM.Options.SupportsDefaultOutlining; |
959 | if (AddOutliner) |
960 | addPass(MachineOutlinerPass(RunOnAllFunctions)); |
961 | } |
962 | |
963 | // Add passes that directly emit MI after all other MI passes. |
964 | derived().addPreEmitPass2(addPass); |
965 | |
966 | return Error::success(); |
967 | } |
968 | |
969 | /// Add passes that optimize machine instructions in SSA form. |
970 | template <typename Derived, typename TargetMachineT> |
971 | void CodeGenPassBuilder<Derived, TargetMachineT>::addMachineSSAOptimization( |
972 | AddMachinePass &addPass) const { |
973 | // Pre-ra tail duplication. |
974 | addPass(EarlyTailDuplicatePass()); |
975 | |
976 | // Optimize PHIs before DCE: removing dead PHI cycles may make more |
977 | // instructions dead. |
978 | addPass(OptimizePHIsPass()); |
979 | |
980 | // This pass merges large allocas. StackSlotColoring is a different pass |
981 | // which merges spill slots. |
982 | addPass(StackColoringPass()); |
983 | |
984 | // If the target requests it, assign local variables to stack slots relative |
985 | // to one another and simplify frame index references where possible. |
986 | addPass(LocalStackSlotPass()); |
987 | |
988 | // With optimization, dead code should already be eliminated. However |
989 | // there is one known exception: lowered code for arguments that are only |
990 | // used by tail calls, where the tail calls reuse the incoming stack |
991 | // arguments directly (see t11 in test/CodeGen/X86/sibcall.ll). |
992 | addPass(DeadMachineInstructionElimPass()); |
993 | |
994 | // Allow targets to insert passes that improve instruction level parallelism, |
995 | // like if-conversion. Such passes will typically need dominator trees and |
996 | // loop info, just like LICM and CSE below. |
997 | derived().addILPOpts(addPass); |
998 | |
999 | addPass(EarlyMachineLICMPass()); |
1000 | addPass(MachineCSEPass()); |
1001 | |
1002 | addPass(MachineSinkingPass()); |
1003 | |
1004 | addPass(PeepholeOptimizerPass()); |
1005 | // Clean-up the dead code that may have been generated by peephole |
1006 | // rewriting. |
1007 | addPass(DeadMachineInstructionElimPass()); |
1008 | } |
1009 | |
1010 | //===---------------------------------------------------------------------===// |
1011 | /// Register Allocation Pass Configuration |
1012 | //===---------------------------------------------------------------------===// |
1013 | |
1014 | /// Instantiate the default register allocator pass for this target for either |
1015 | /// the optimized or unoptimized allocation path. This will be added to the pass |
1016 | /// manager by addFastRegAlloc in the unoptimized case or addOptimizedRegAlloc |
1017 | /// in the optimized case. |
1018 | /// |
1019 | /// A target that uses the standard regalloc pass order for fast or optimized |
1020 | /// allocation may still override this for per-target regalloc |
1021 | /// selection. But -regalloc=... always takes precedence. |
1022 | template <typename Derived, typename TargetMachineT> |
1023 | void CodeGenPassBuilder<Derived, TargetMachineT>::addTargetRegisterAllocator( |
1024 | AddMachinePass &addPass, bool Optimized) const { |
1025 | if (Optimized) |
1026 | addPass(RAGreedyPass()); |
1027 | else |
1028 | addPass(RAFastPass()); |
1029 | } |
1030 | |
1031 | /// Find and instantiate the register allocation pass requested by this target |
1032 | /// at the current optimization level. Different register allocators are |
1033 | /// defined as separate passes because they may require different analysis. |
1034 | template <typename Derived, typename TargetMachineT> |
1035 | void CodeGenPassBuilder<Derived, TargetMachineT>::addRegAllocPass( |
1036 | AddMachinePass &addPass, bool Optimized) const { |
1037 | // TODO: Parse Opt.RegAlloc to add register allocator. |
1038 | } |
1039 | |
1040 | template <typename Derived, typename TargetMachineT> |
1041 | Error CodeGenPassBuilder<Derived, TargetMachineT>::addRegAssignmentFast( |
1042 | AddMachinePass &addPass) const { |
1043 | // TODO: Ensure allocator is default or fast. |
1044 | addRegAllocPass(addPass, Optimized: false); |
1045 | return Error::success(); |
1046 | } |
1047 | |
1048 | template <typename Derived, typename TargetMachineT> |
1049 | Error CodeGenPassBuilder<Derived, TargetMachineT>::addRegAssignmentOptimized( |
1050 | AddMachinePass &addPass) const { |
1051 | // Add the selected register allocation pass. |
1052 | addRegAllocPass(addPass, Optimized: true); |
1053 | |
1054 | // Allow targets to change the register assignments before rewriting. |
1055 | derived().addPreRewrite(addPass); |
1056 | |
1057 | // Finally rewrite virtual registers. |
1058 | addPass(VirtRegRewriterPass()); |
1059 | // Perform stack slot coloring and post-ra machine LICM. |
1060 | // |
1061 | // FIXME: Re-enable coloring with register when it's capable of adding |
1062 | // kill markers. |
1063 | addPass(StackSlotColoringPass()); |
1064 | |
1065 | return Error::success(); |
1066 | } |
1067 | |
1068 | /// Add the minimum set of target-independent passes that are required for |
1069 | /// register allocation. No coalescing or scheduling. |
1070 | template <typename Derived, typename TargetMachineT> |
1071 | Error CodeGenPassBuilder<Derived, TargetMachineT>::addFastRegAlloc( |
1072 | AddMachinePass &addPass) const { |
1073 | addPass(PHIEliminationPass()); |
1074 | addPass(TwoAddressInstructionPass()); |
1075 | return derived().addRegAssignmentFast(addPass); |
1076 | } |
1077 | |
1078 | /// Add standard target-independent passes that are tightly coupled with |
1079 | /// optimized register allocation, including coalescing, machine instruction |
1080 | /// scheduling, and register allocation itself. |
1081 | template <typename Derived, typename TargetMachineT> |
1082 | void CodeGenPassBuilder<Derived, TargetMachineT>::addOptimizedRegAlloc( |
1083 | AddMachinePass &addPass) const { |
1084 | addPass(DetectDeadLanesPass()); |
1085 | |
1086 | addPass(InitUndefPass()); |
1087 | |
1088 | addPass(ProcessImplicitDefsPass()); |
1089 | |
1090 | // Edge splitting is smarter with machine loop info. |
1091 | addPass(PHIEliminationPass()); |
1092 | |
1093 | // Eventually, we want to run LiveIntervals before PHI elimination. |
1094 | if (Opt.EarlyLiveIntervals) |
1095 | addPass(LiveIntervalsPass()); |
1096 | |
1097 | addPass(TwoAddressInstructionPass()); |
1098 | addPass(RegisterCoalescerPass()); |
1099 | |
1100 | // The machine scheduler may accidentally create disconnected components |
1101 | // when moving subregister definitions around, avoid this by splitting them to |
1102 | // separate vregs before. Splitting can also improve reg. allocation quality. |
1103 | addPass(RenameIndependentSubregsPass()); |
1104 | |
1105 | // PreRA instruction scheduling. |
1106 | addPass(MachineSchedulerPass()); |
1107 | |
1108 | if (derived().addRegAssignmentOptimized(addPass)) { |
1109 | // Allow targets to expand pseudo instructions depending on the choice of |
1110 | // registers before MachineCopyPropagation. |
1111 | derived().addPostRewrite(addPass); |
1112 | |
1113 | // Copy propagate to forward register uses and try to eliminate COPYs that |
1114 | // were not coalesced. |
1115 | addPass(MachineCopyPropagationPass()); |
1116 | |
1117 | // Run post-ra machine LICM to hoist reloads / remats. |
1118 | // |
1119 | // FIXME: can this move into MachineLateOptimization? |
1120 | addPass(MachineLICMPass()); |
1121 | } |
1122 | } |
1123 | |
1124 | //===---------------------------------------------------------------------===// |
1125 | /// Post RegAlloc Pass Configuration |
1126 | //===---------------------------------------------------------------------===// |
1127 | |
1128 | /// Add passes that optimize machine instructions after register allocation. |
1129 | template <typename Derived, typename TargetMachineT> |
1130 | void CodeGenPassBuilder<Derived, TargetMachineT>::addMachineLateOptimization( |
1131 | AddMachinePass &addPass) const { |
1132 | // Branch folding must be run after regalloc and prolog/epilog insertion. |
1133 | addPass(BranchFolderPass()); |
1134 | |
1135 | // Tail duplication. |
1136 | // Note that duplicating tail just increases code size and degrades |
1137 | // performance for targets that require Structured Control Flow. |
1138 | // In addition it can also make CFG irreducible. Thus we disable it. |
1139 | if (!TM.requiresStructuredCFG()) |
1140 | addPass(TailDuplicatePass()); |
1141 | |
1142 | // Cleanup of redundant (identical) address/immediate loads. |
1143 | addPass(MachineLateInstrsCleanupPass()); |
1144 | |
1145 | // Copy propagation. |
1146 | addPass(MachineCopyPropagationPass()); |
1147 | } |
1148 | |
1149 | /// Add standard basic block placement passes. |
1150 | template <typename Derived, typename TargetMachineT> |
1151 | void CodeGenPassBuilder<Derived, TargetMachineT>::addBlockPlacement( |
1152 | AddMachinePass &addPass) const { |
1153 | addPass(MachineBlockPlacementPass()); |
1154 | // Run a separate pass to collect block placement statistics. |
1155 | if (Opt.EnableBlockPlacementStats) |
1156 | addPass(MachineBlockPlacementStatsPass()); |
1157 | } |
1158 | |
1159 | } // namespace llvm |
1160 | |
1161 | #endif // LLVM_PASSES_CODEGENPASSBUILDER_H |
1162 | |