1 | //===- AlwaysInliner.cpp - Code to inline always_inline functions ----------===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | // |
9 | // This file implements a custom inliner that handles only functions that |
10 | // are marked as "always inline". |
11 | // |
12 | //===----------------------------------------------------------------------===// |
13 | |
14 | #include "llvm/Transforms/IPO/AlwaysInliner.h" |
15 | #include "llvm/ADT/SetVector.h" |
16 | #include "llvm/Analysis/AliasAnalysis.h" |
17 | #include "llvm/Analysis/AssumptionCache.h" |
18 | #include "llvm/Analysis/InlineCost.h" |
19 | #include "llvm/Analysis/OptimizationRemarkEmitter.h" |
20 | #include "llvm/Analysis/ProfileSummaryInfo.h" |
21 | #include "llvm/IR/Module.h" |
22 | #include "llvm/InitializePasses.h" |
23 | #include "llvm/Transforms/IPO/Inliner.h" |
24 | #include "llvm/Transforms/Utils/Cloning.h" |
25 | #include "llvm/Transforms/Utils/ModuleUtils.h" |
26 | |
27 | using namespace llvm; |
28 | |
29 | #define DEBUG_TYPE "inline" |
30 | |
31 | namespace { |
32 | |
33 | bool AlwaysInlineImpl( |
34 | Module &M, bool InsertLifetime, ProfileSummaryInfo &PSI, |
35 | function_ref<AssumptionCache &(Function &)> GetAssumptionCache, |
36 | function_ref<AAResults &(Function &)> GetAAR, |
37 | function_ref<BlockFrequencyInfo &(Function &)> GetBFI) { |
38 | SmallSetVector<CallBase *, 16> Calls; |
39 | bool Changed = false; |
40 | SmallVector<Function *, 16> InlinedFunctions; |
41 | for (Function &F : M) { |
42 | // When callee coroutine function is inlined into caller coroutine function |
43 | // before coro-split pass, |
44 | // coro-early pass can not handle this quiet well. |
45 | // So we won't inline the coroutine function if it have not been unsplited |
46 | if (F.isPresplitCoroutine()) |
47 | continue; |
48 | |
49 | if (!F.isDeclaration() && isInlineViable(Callee&: F).isSuccess()) { |
50 | Calls.clear(); |
51 | |
52 | for (User *U : F.users()) |
53 | if (auto *CB = dyn_cast<CallBase>(Val: U)) |
54 | if (CB->getCalledFunction() == &F && |
55 | CB->hasFnAttr(Attribute::AlwaysInline) && |
56 | !CB->getAttributes().hasFnAttr(Attribute::NoInline)) |
57 | Calls.insert(X: CB); |
58 | |
59 | for (CallBase *CB : Calls) { |
60 | Function *Caller = CB->getCaller(); |
61 | OptimizationRemarkEmitter ORE(Caller); |
62 | DebugLoc DLoc = CB->getDebugLoc(); |
63 | BasicBlock *Block = CB->getParent(); |
64 | |
65 | InlineFunctionInfo IFI(GetAssumptionCache, &PSI, |
66 | GetBFI ? &GetBFI(*Caller) : nullptr, |
67 | GetBFI ? &GetBFI(F) : nullptr); |
68 | |
69 | InlineResult Res = InlineFunction(CB&: *CB, IFI, /*MergeAttributes=*/true, |
70 | CalleeAAR: &GetAAR(F), InsertLifetime); |
71 | if (!Res.isSuccess()) { |
72 | ORE.emit(RemarkBuilder: [&]() { |
73 | return OptimizationRemarkMissed(DEBUG_TYPE, "NotInlined" , DLoc, |
74 | Block) |
75 | << "'" << ore::NV("Callee" , &F) << "' is not inlined into '" |
76 | << ore::NV("Caller" , Caller) |
77 | << "': " << ore::NV("Reason" , Res.getFailureReason()); |
78 | }); |
79 | continue; |
80 | } |
81 | |
82 | emitInlinedIntoBasedOnCost( |
83 | ORE, DLoc, Block, Callee: F, Caller: *Caller, |
84 | IC: InlineCost::getAlways(Reason: "always inline attribute" ), |
85 | /*ForProfileContext=*/false, DEBUG_TYPE); |
86 | |
87 | Changed = true; |
88 | } |
89 | |
90 | if (F.hasFnAttribute(Attribute::AlwaysInline)) { |
91 | // Remember to try and delete this function afterward. This both avoids |
92 | // re-walking the rest of the module and avoids dealing with any |
93 | // iterator invalidation issues while deleting functions. |
94 | InlinedFunctions.push_back(Elt: &F); |
95 | } |
96 | } |
97 | } |
98 | |
99 | // Remove any live functions. |
100 | erase_if(C&: InlinedFunctions, P: [&](Function *F) { |
101 | F->removeDeadConstantUsers(); |
102 | return !F->isDefTriviallyDead(); |
103 | }); |
104 | |
105 | // Delete the non-comdat ones from the module and also from our vector. |
106 | auto NonComdatBegin = partition( |
107 | Range&: InlinedFunctions, P: [&](Function *F) { return F->hasComdat(); }); |
108 | for (Function *F : make_range(x: NonComdatBegin, y: InlinedFunctions.end())) { |
109 | M.getFunctionList().erase(IT: F); |
110 | Changed = true; |
111 | } |
112 | InlinedFunctions.erase(CS: NonComdatBegin, CE: InlinedFunctions.end()); |
113 | |
114 | if (!InlinedFunctions.empty()) { |
115 | // Now we just have the comdat functions. Filter out the ones whose comdats |
116 | // are not actually dead. |
117 | filterDeadComdatFunctions(DeadComdatFunctions&: InlinedFunctions); |
118 | // The remaining functions are actually dead. |
119 | for (Function *F : InlinedFunctions) { |
120 | M.getFunctionList().erase(IT: F); |
121 | Changed = true; |
122 | } |
123 | } |
124 | |
125 | return Changed; |
126 | } |
127 | |
128 | struct AlwaysInlinerLegacyPass : public ModulePass { |
129 | bool InsertLifetime; |
130 | |
131 | AlwaysInlinerLegacyPass() |
132 | : AlwaysInlinerLegacyPass(/*InsertLifetime*/ true) {} |
133 | |
134 | AlwaysInlinerLegacyPass(bool InsertLifetime) |
135 | : ModulePass(ID), InsertLifetime(InsertLifetime) { |
136 | initializeAlwaysInlinerLegacyPassPass(*PassRegistry::getPassRegistry()); |
137 | } |
138 | |
139 | /// Main run interface method. We override here to avoid calling skipSCC(). |
140 | bool runOnModule(Module &M) override { |
141 | |
142 | auto &PSI = getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI(); |
143 | auto GetAAR = [&](Function &F) -> AAResults & { |
144 | return getAnalysis<AAResultsWrapperPass>(F).getAAResults(); |
145 | }; |
146 | auto GetAssumptionCache = [&](Function &F) -> AssumptionCache & { |
147 | return getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F); |
148 | }; |
149 | |
150 | return AlwaysInlineImpl(M, InsertLifetime, PSI, GetAssumptionCache, GetAAR, |
151 | /*GetBFI*/ nullptr); |
152 | } |
153 | |
154 | static char ID; // Pass identification, replacement for typeid |
155 | |
156 | void getAnalysisUsage(AnalysisUsage &AU) const override { |
157 | AU.addRequired<AssumptionCacheTracker>(); |
158 | AU.addRequired<AAResultsWrapperPass>(); |
159 | AU.addRequired<ProfileSummaryInfoWrapperPass>(); |
160 | } |
161 | }; |
162 | |
163 | } // namespace |
164 | |
165 | char AlwaysInlinerLegacyPass::ID = 0; |
166 | INITIALIZE_PASS_BEGIN(AlwaysInlinerLegacyPass, "always-inline" , |
167 | "Inliner for always_inline functions" , false, false) |
168 | INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass) |
169 | INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker) |
170 | INITIALIZE_PASS_DEPENDENCY(ProfileSummaryInfoWrapperPass) |
171 | INITIALIZE_PASS_END(AlwaysInlinerLegacyPass, "always-inline" , |
172 | "Inliner for always_inline functions" , false, false) |
173 | |
174 | Pass *llvm::createAlwaysInlinerLegacyPass(bool InsertLifetime) { |
175 | return new AlwaysInlinerLegacyPass(InsertLifetime); |
176 | } |
177 | |
178 | PreservedAnalyses AlwaysInlinerPass::run(Module &M, |
179 | ModuleAnalysisManager &MAM) { |
180 | FunctionAnalysisManager &FAM = |
181 | MAM.getResult<FunctionAnalysisManagerModuleProxy>(IR&: M).getManager(); |
182 | auto GetAssumptionCache = [&](Function &F) -> AssumptionCache & { |
183 | return FAM.getResult<AssumptionAnalysis>(IR&: F); |
184 | }; |
185 | auto GetBFI = [&](Function &F) -> BlockFrequencyInfo & { |
186 | return FAM.getResult<BlockFrequencyAnalysis>(IR&: F); |
187 | }; |
188 | auto GetAAR = [&](Function &F) -> AAResults & { |
189 | return FAM.getResult<AAManager>(IR&: F); |
190 | }; |
191 | auto &PSI = MAM.getResult<ProfileSummaryAnalysis>(IR&: M); |
192 | |
193 | bool Changed = AlwaysInlineImpl(M, InsertLifetime, PSI, GetAssumptionCache, |
194 | GetAAR, GetBFI); |
195 | |
196 | return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all(); |
197 | } |
198 | |