1 | //===-LTO.cpp - LLVM Link Time Optimizer ----------------------------------===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | // |
9 | // This file implements functions and classes used to support LTO. |
10 | // |
11 | //===----------------------------------------------------------------------===// |
12 | |
13 | #include "llvm/LTO/LTO.h" |
14 | #include "llvm/ADT/ScopeExit.h" |
15 | #include "llvm/ADT/SmallSet.h" |
16 | #include "llvm/ADT/Statistic.h" |
17 | #include "llvm/ADT/StringExtras.h" |
18 | #include "llvm/Analysis/OptimizationRemarkEmitter.h" |
19 | #include "llvm/Analysis/StackSafetyAnalysis.h" |
20 | #include "llvm/Analysis/TargetLibraryInfo.h" |
21 | #include "llvm/Analysis/TargetTransformInfo.h" |
22 | #include "llvm/Bitcode/BitcodeReader.h" |
23 | #include "llvm/Bitcode/BitcodeWriter.h" |
24 | #include "llvm/CodeGen/Analysis.h" |
25 | #include "llvm/Config/llvm-config.h" |
26 | #include "llvm/IR/AutoUpgrade.h" |
27 | #include "llvm/IR/DiagnosticPrinter.h" |
28 | #include "llvm/IR/Intrinsics.h" |
29 | #include "llvm/IR/LLVMRemarkStreamer.h" |
30 | #include "llvm/IR/LegacyPassManager.h" |
31 | #include "llvm/IR/Mangler.h" |
32 | #include "llvm/IR/Metadata.h" |
33 | #include "llvm/LTO/LTOBackend.h" |
34 | #include "llvm/LTO/SummaryBasedOptimizations.h" |
35 | #include "llvm/Linker/IRMover.h" |
36 | #include "llvm/MC/TargetRegistry.h" |
37 | #include "llvm/Object/IRObjectFile.h" |
38 | #include "llvm/Support/CommandLine.h" |
39 | #include "llvm/Support/Error.h" |
40 | #include "llvm/Support/FileSystem.h" |
41 | #include "llvm/Support/ManagedStatic.h" |
42 | #include "llvm/Support/MemoryBuffer.h" |
43 | #include "llvm/Support/Path.h" |
44 | #include "llvm/Support/SHA1.h" |
45 | #include "llvm/Support/SourceMgr.h" |
46 | #include "llvm/Support/ThreadPool.h" |
47 | #include "llvm/Support/Threading.h" |
48 | #include "llvm/Support/TimeProfiler.h" |
49 | #include "llvm/Support/ToolOutputFile.h" |
50 | #include "llvm/Support/VCSRevision.h" |
51 | #include "llvm/Support/raw_ostream.h" |
52 | #include "llvm/Target/TargetOptions.h" |
53 | #include "llvm/Transforms/IPO.h" |
54 | #include "llvm/Transforms/IPO/MemProfContextDisambiguation.h" |
55 | #include "llvm/Transforms/IPO/WholeProgramDevirt.h" |
56 | #include "llvm/Transforms/Utils/FunctionImportUtils.h" |
57 | #include "llvm/Transforms/Utils/SplitModule.h" |
58 | |
59 | #include <optional> |
60 | #include <set> |
61 | |
62 | using namespace llvm; |
63 | using namespace lto; |
64 | using namespace object; |
65 | |
66 | #define DEBUG_TYPE "lto" |
67 | |
68 | static cl::opt<bool> |
69 | DumpThinCGSCCs("dump-thin-cg-sccs" , cl::init(Val: false), cl::Hidden, |
70 | cl::desc("Dump the SCCs in the ThinLTO index's callgraph" )); |
71 | |
72 | namespace llvm { |
73 | /// Enable global value internalization in LTO. |
74 | cl::opt<bool> EnableLTOInternalization( |
75 | "enable-lto-internalization" , cl::init(Val: true), cl::Hidden, |
76 | cl::desc("Enable global value internalization in LTO" )); |
77 | |
78 | /// Indicate we are linking with an allocator that supports hot/cold operator |
79 | /// new interfaces. |
80 | extern cl::opt<bool> SupportsHotColdNew; |
81 | |
82 | /// Enable MemProf context disambiguation for thin link. |
83 | extern cl::opt<bool> EnableMemProfContextDisambiguation; |
84 | } // namespace llvm |
85 | |
86 | // Computes a unique hash for the Module considering the current list of |
87 | // export/import and other global analysis results. |
88 | // The hash is produced in \p Key. |
89 | void llvm::computeLTOCacheKey( |
90 | SmallString<40> &Key, const Config &Conf, const ModuleSummaryIndex &Index, |
91 | StringRef ModuleID, const FunctionImporter::ImportMapTy &ImportList, |
92 | const FunctionImporter::ExportSetTy &ExportList, |
93 | const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR, |
94 | const GVSummaryMapTy &DefinedGlobals, |
95 | const std::set<GlobalValue::GUID> &CfiFunctionDefs, |
96 | const std::set<GlobalValue::GUID> &CfiFunctionDecls) { |
97 | // Compute the unique hash for this entry. |
98 | // This is based on the current compiler version, the module itself, the |
99 | // export list, the hash for every single module in the import list, the |
100 | // list of ResolvedODR for the module, and the list of preserved symbols. |
101 | SHA1 Hasher; |
102 | |
103 | // Start with the compiler revision |
104 | Hasher.update(LLVM_VERSION_STRING); |
105 | #ifdef LLVM_REVISION |
106 | Hasher.update(LLVM_REVISION); |
107 | #endif |
108 | |
109 | // Include the parts of the LTO configuration that affect code generation. |
110 | auto AddString = [&](StringRef Str) { |
111 | Hasher.update(Str); |
112 | Hasher.update(Data: ArrayRef<uint8_t>{0}); |
113 | }; |
114 | auto AddUnsigned = [&](unsigned I) { |
115 | uint8_t Data[4]; |
116 | support::endian::write32le(P: Data, V: I); |
117 | Hasher.update(Data: ArrayRef<uint8_t>{Data, 4}); |
118 | }; |
119 | auto AddUint64 = [&](uint64_t I) { |
120 | uint8_t Data[8]; |
121 | support::endian::write64le(P: Data, V: I); |
122 | Hasher.update(Data: ArrayRef<uint8_t>{Data, 8}); |
123 | }; |
124 | AddString(Conf.CPU); |
125 | // FIXME: Hash more of Options. For now all clients initialize Options from |
126 | // command-line flags (which is unsupported in production), but may set |
127 | // RelaxELFRelocations. The clang driver can also pass FunctionSections, |
128 | // DataSections and DebuggerTuning via command line flags. |
129 | AddUnsigned(Conf.Options.RelaxELFRelocations); |
130 | AddUnsigned(Conf.Options.FunctionSections); |
131 | AddUnsigned(Conf.Options.DataSections); |
132 | AddUnsigned((unsigned)Conf.Options.DebuggerTuning); |
133 | for (auto &A : Conf.MAttrs) |
134 | AddString(A); |
135 | if (Conf.RelocModel) |
136 | AddUnsigned(*Conf.RelocModel); |
137 | else |
138 | AddUnsigned(-1); |
139 | if (Conf.CodeModel) |
140 | AddUnsigned(*Conf.CodeModel); |
141 | else |
142 | AddUnsigned(-1); |
143 | for (const auto &S : Conf.MllvmArgs) |
144 | AddString(S); |
145 | AddUnsigned(static_cast<int>(Conf.CGOptLevel)); |
146 | AddUnsigned(static_cast<int>(Conf.CGFileType)); |
147 | AddUnsigned(Conf.OptLevel); |
148 | AddUnsigned(Conf.Freestanding); |
149 | AddString(Conf.OptPipeline); |
150 | AddString(Conf.AAPipeline); |
151 | AddString(Conf.OverrideTriple); |
152 | AddString(Conf.DefaultTriple); |
153 | AddString(Conf.DwoDir); |
154 | |
155 | // Include the hash for the current module |
156 | auto ModHash = Index.getModuleHash(ModPath: ModuleID); |
157 | Hasher.update(Data: ArrayRef<uint8_t>((uint8_t *)&ModHash[0], sizeof(ModHash))); |
158 | |
159 | std::vector<uint64_t> ExportsGUID; |
160 | ExportsGUID.reserve(n: ExportList.size()); |
161 | for (const auto &VI : ExportList) { |
162 | auto GUID = VI.getGUID(); |
163 | ExportsGUID.push_back(x: GUID); |
164 | } |
165 | |
166 | // Sort the export list elements GUIDs. |
167 | llvm::sort(C&: ExportsGUID); |
168 | for (uint64_t GUID : ExportsGUID) { |
169 | // The export list can impact the internalization, be conservative here |
170 | Hasher.update(Data: ArrayRef<uint8_t>((uint8_t *)&GUID, sizeof(GUID))); |
171 | } |
172 | |
173 | // Include the hash for every module we import functions from. The set of |
174 | // imported symbols for each module may affect code generation and is |
175 | // sensitive to link order, so include that as well. |
176 | using ImportMapIteratorTy = FunctionImporter::ImportMapTy::const_iterator; |
177 | struct ImportModule { |
178 | ImportMapIteratorTy ModIt; |
179 | const ModuleSummaryIndex::ModuleInfo *ModInfo; |
180 | |
181 | StringRef getIdentifier() const { return ModIt->getFirst(); } |
182 | const FunctionImporter::FunctionsToImportTy &getFunctions() const { |
183 | return ModIt->second; |
184 | } |
185 | |
186 | const ModuleHash &getHash() const { return ModInfo->second; } |
187 | }; |
188 | |
189 | std::vector<ImportModule> ImportModulesVector; |
190 | ImportModulesVector.reserve(n: ImportList.size()); |
191 | |
192 | for (ImportMapIteratorTy It = ImportList.begin(); It != ImportList.end(); |
193 | ++It) { |
194 | ImportModulesVector.push_back(x: {.ModIt: It, .ModInfo: Index.getModule(ModPath: It->getFirst())}); |
195 | } |
196 | // Order using module hash, to be both independent of module name and |
197 | // module order. |
198 | llvm::sort(C&: ImportModulesVector, |
199 | Comp: [](const ImportModule &Lhs, const ImportModule &Rhs) -> bool { |
200 | return Lhs.getHash() < Rhs.getHash(); |
201 | }); |
202 | for (const ImportModule &Entry : ImportModulesVector) { |
203 | auto ModHash = Entry.getHash(); |
204 | Hasher.update(Data: ArrayRef<uint8_t>((uint8_t *)&ModHash[0], sizeof(ModHash))); |
205 | |
206 | AddUint64(Entry.getFunctions().size()); |
207 | for (auto &Fn : Entry.getFunctions()) |
208 | AddUint64(Fn); |
209 | } |
210 | |
211 | // Include the hash for the resolved ODR. |
212 | for (auto &Entry : ResolvedODR) { |
213 | Hasher.update(Data: ArrayRef<uint8_t>((const uint8_t *)&Entry.first, |
214 | sizeof(GlobalValue::GUID))); |
215 | Hasher.update(Data: ArrayRef<uint8_t>((const uint8_t *)&Entry.second, |
216 | sizeof(GlobalValue::LinkageTypes))); |
217 | } |
218 | |
219 | // Members of CfiFunctionDefs and CfiFunctionDecls that are referenced or |
220 | // defined in this module. |
221 | std::set<GlobalValue::GUID> UsedCfiDefs; |
222 | std::set<GlobalValue::GUID> UsedCfiDecls; |
223 | |
224 | // Typeids used in this module. |
225 | std::set<GlobalValue::GUID> UsedTypeIds; |
226 | |
227 | auto AddUsedCfiGlobal = [&](GlobalValue::GUID ValueGUID) { |
228 | if (CfiFunctionDefs.count(x: ValueGUID)) |
229 | UsedCfiDefs.insert(x: ValueGUID); |
230 | if (CfiFunctionDecls.count(x: ValueGUID)) |
231 | UsedCfiDecls.insert(x: ValueGUID); |
232 | }; |
233 | |
234 | auto AddUsedThings = [&](GlobalValueSummary *GS) { |
235 | if (!GS) return; |
236 | AddUnsigned(GS->getVisibility()); |
237 | AddUnsigned(GS->isLive()); |
238 | AddUnsigned(GS->canAutoHide()); |
239 | for (const ValueInfo &VI : GS->refs()) { |
240 | AddUnsigned(VI.isDSOLocal(WithDSOLocalPropagation: Index.withDSOLocalPropagation())); |
241 | AddUsedCfiGlobal(VI.getGUID()); |
242 | } |
243 | if (auto *GVS = dyn_cast<GlobalVarSummary>(Val: GS)) { |
244 | AddUnsigned(GVS->maybeReadOnly()); |
245 | AddUnsigned(GVS->maybeWriteOnly()); |
246 | } |
247 | if (auto *FS = dyn_cast<FunctionSummary>(Val: GS)) { |
248 | for (auto &TT : FS->type_tests()) |
249 | UsedTypeIds.insert(x: TT); |
250 | for (auto &TT : FS->type_test_assume_vcalls()) |
251 | UsedTypeIds.insert(x: TT.GUID); |
252 | for (auto &TT : FS->type_checked_load_vcalls()) |
253 | UsedTypeIds.insert(x: TT.GUID); |
254 | for (auto &TT : FS->type_test_assume_const_vcalls()) |
255 | UsedTypeIds.insert(x: TT.VFunc.GUID); |
256 | for (auto &TT : FS->type_checked_load_const_vcalls()) |
257 | UsedTypeIds.insert(x: TT.VFunc.GUID); |
258 | for (auto &ET : FS->calls()) { |
259 | AddUnsigned(ET.first.isDSOLocal(WithDSOLocalPropagation: Index.withDSOLocalPropagation())); |
260 | AddUsedCfiGlobal(ET.first.getGUID()); |
261 | } |
262 | } |
263 | }; |
264 | |
265 | // Include the hash for the linkage type to reflect internalization and weak |
266 | // resolution, and collect any used type identifier resolutions. |
267 | for (auto &GS : DefinedGlobals) { |
268 | GlobalValue::LinkageTypes Linkage = GS.second->linkage(); |
269 | Hasher.update( |
270 | Data: ArrayRef<uint8_t>((const uint8_t *)&Linkage, sizeof(Linkage))); |
271 | AddUsedCfiGlobal(GS.first); |
272 | AddUsedThings(GS.second); |
273 | } |
274 | |
275 | // Imported functions may introduce new uses of type identifier resolutions, |
276 | // so we need to collect their used resolutions as well. |
277 | for (const ImportModule &ImpM : ImportModulesVector) |
278 | for (auto &ImpF : ImpM.getFunctions()) { |
279 | GlobalValueSummary *S = |
280 | Index.findSummaryInModule(ValueGUID: ImpF, ModuleId: ImpM.getIdentifier()); |
281 | AddUsedThings(S); |
282 | // If this is an alias, we also care about any types/etc. that the aliasee |
283 | // may reference. |
284 | if (auto *AS = dyn_cast_or_null<AliasSummary>(Val: S)) |
285 | AddUsedThings(AS->getBaseObject()); |
286 | } |
287 | |
288 | auto AddTypeIdSummary = [&](StringRef TId, const TypeIdSummary &S) { |
289 | AddString(TId); |
290 | |
291 | AddUnsigned(S.TTRes.TheKind); |
292 | AddUnsigned(S.TTRes.SizeM1BitWidth); |
293 | |
294 | AddUint64(S.TTRes.AlignLog2); |
295 | AddUint64(S.TTRes.SizeM1); |
296 | AddUint64(S.TTRes.BitMask); |
297 | AddUint64(S.TTRes.InlineBits); |
298 | |
299 | AddUint64(S.WPDRes.size()); |
300 | for (auto &WPD : S.WPDRes) { |
301 | AddUnsigned(WPD.first); |
302 | AddUnsigned(WPD.second.TheKind); |
303 | AddString(WPD.second.SingleImplName); |
304 | |
305 | AddUint64(WPD.second.ResByArg.size()); |
306 | for (auto &ByArg : WPD.second.ResByArg) { |
307 | AddUint64(ByArg.first.size()); |
308 | for (uint64_t Arg : ByArg.first) |
309 | AddUint64(Arg); |
310 | AddUnsigned(ByArg.second.TheKind); |
311 | AddUint64(ByArg.second.Info); |
312 | AddUnsigned(ByArg.second.Byte); |
313 | AddUnsigned(ByArg.second.Bit); |
314 | } |
315 | } |
316 | }; |
317 | |
318 | // Include the hash for all type identifiers used by this module. |
319 | for (GlobalValue::GUID TId : UsedTypeIds) { |
320 | auto TidIter = Index.typeIds().equal_range(x: TId); |
321 | for (auto It = TidIter.first; It != TidIter.second; ++It) |
322 | AddTypeIdSummary(It->second.first, It->second.second); |
323 | } |
324 | |
325 | AddUnsigned(UsedCfiDefs.size()); |
326 | for (auto &V : UsedCfiDefs) |
327 | AddUint64(V); |
328 | |
329 | AddUnsigned(UsedCfiDecls.size()); |
330 | for (auto &V : UsedCfiDecls) |
331 | AddUint64(V); |
332 | |
333 | if (!Conf.SampleProfile.empty()) { |
334 | auto FileOrErr = MemoryBuffer::getFile(Filename: Conf.SampleProfile); |
335 | if (FileOrErr) { |
336 | Hasher.update(Str: FileOrErr.get()->getBuffer()); |
337 | |
338 | if (!Conf.ProfileRemapping.empty()) { |
339 | FileOrErr = MemoryBuffer::getFile(Filename: Conf.ProfileRemapping); |
340 | if (FileOrErr) |
341 | Hasher.update(Str: FileOrErr.get()->getBuffer()); |
342 | } |
343 | } |
344 | } |
345 | |
346 | Key = toHex(Input: Hasher.result()); |
347 | } |
348 | |
349 | static void thinLTOResolvePrevailingGUID( |
350 | const Config &C, ValueInfo VI, |
351 | DenseSet<GlobalValueSummary *> &GlobalInvolvedWithAlias, |
352 | function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)> |
353 | isPrevailing, |
354 | function_ref<void(StringRef, GlobalValue::GUID, GlobalValue::LinkageTypes)> |
355 | recordNewLinkage, |
356 | const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols) { |
357 | GlobalValue::VisibilityTypes Visibility = |
358 | C.VisibilityScheme == Config::ELF ? VI.getELFVisibility() |
359 | : GlobalValue::DefaultVisibility; |
360 | for (auto &S : VI.getSummaryList()) { |
361 | GlobalValue::LinkageTypes OriginalLinkage = S->linkage(); |
362 | // Ignore local and appending linkage values since the linker |
363 | // doesn't resolve them. |
364 | if (GlobalValue::isLocalLinkage(Linkage: OriginalLinkage) || |
365 | GlobalValue::isAppendingLinkage(Linkage: S->linkage())) |
366 | continue; |
367 | // We need to emit only one of these. The prevailing module will keep it, |
368 | // but turned into a weak, while the others will drop it when possible. |
369 | // This is both a compile-time optimization and a correctness |
370 | // transformation. This is necessary for correctness when we have exported |
371 | // a reference - we need to convert the linkonce to weak to |
372 | // ensure a copy is kept to satisfy the exported reference. |
373 | // FIXME: We may want to split the compile time and correctness |
374 | // aspects into separate routines. |
375 | if (isPrevailing(VI.getGUID(), S.get())) { |
376 | if (GlobalValue::isLinkOnceLinkage(Linkage: OriginalLinkage)) { |
377 | S->setLinkage(GlobalValue::getWeakLinkage( |
378 | ODR: GlobalValue::isLinkOnceODRLinkage(Linkage: OriginalLinkage))); |
379 | // The kept copy is eligible for auto-hiding (hidden visibility) if all |
380 | // copies were (i.e. they were all linkonce_odr global unnamed addr). |
381 | // If any copy is not (e.g. it was originally weak_odr), then the symbol |
382 | // must remain externally available (e.g. a weak_odr from an explicitly |
383 | // instantiated template). Additionally, if it is in the |
384 | // GUIDPreservedSymbols set, that means that it is visibile outside |
385 | // the summary (e.g. in a native object or a bitcode file without |
386 | // summary), and in that case we cannot hide it as it isn't possible to |
387 | // check all copies. |
388 | S->setCanAutoHide(VI.canAutoHide() && |
389 | !GUIDPreservedSymbols.count(V: VI.getGUID())); |
390 | } |
391 | if (C.VisibilityScheme == Config::FromPrevailing) |
392 | Visibility = S->getVisibility(); |
393 | } |
394 | // Alias and aliasee can't be turned into available_externally. |
395 | else if (!isa<AliasSummary>(Val: S.get()) && |
396 | !GlobalInvolvedWithAlias.count(V: S.get())) |
397 | S->setLinkage(GlobalValue::AvailableExternallyLinkage); |
398 | |
399 | // For ELF, set visibility to the computed visibility from summaries. We |
400 | // don't track visibility from declarations so this may be more relaxed than |
401 | // the most constraining one. |
402 | if (C.VisibilityScheme == Config::ELF) |
403 | S->setVisibility(Visibility); |
404 | |
405 | if (S->linkage() != OriginalLinkage) |
406 | recordNewLinkage(S->modulePath(), VI.getGUID(), S->linkage()); |
407 | } |
408 | |
409 | if (C.VisibilityScheme == Config::FromPrevailing) { |
410 | for (auto &S : VI.getSummaryList()) { |
411 | GlobalValue::LinkageTypes OriginalLinkage = S->linkage(); |
412 | if (GlobalValue::isLocalLinkage(Linkage: OriginalLinkage) || |
413 | GlobalValue::isAppendingLinkage(Linkage: S->linkage())) |
414 | continue; |
415 | S->setVisibility(Visibility); |
416 | } |
417 | } |
418 | } |
419 | |
420 | /// Resolve linkage for prevailing symbols in the \p Index. |
421 | // |
422 | // We'd like to drop these functions if they are no longer referenced in the |
423 | // current module. However there is a chance that another module is still |
424 | // referencing them because of the import. We make sure we always emit at least |
425 | // one copy. |
426 | void llvm::thinLTOResolvePrevailingInIndex( |
427 | const Config &C, ModuleSummaryIndex &Index, |
428 | function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)> |
429 | isPrevailing, |
430 | function_ref<void(StringRef, GlobalValue::GUID, GlobalValue::LinkageTypes)> |
431 | recordNewLinkage, |
432 | const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols) { |
433 | // We won't optimize the globals that are referenced by an alias for now |
434 | // Ideally we should turn the alias into a global and duplicate the definition |
435 | // when needed. |
436 | DenseSet<GlobalValueSummary *> GlobalInvolvedWithAlias; |
437 | for (auto &I : Index) |
438 | for (auto &S : I.second.SummaryList) |
439 | if (auto AS = dyn_cast<AliasSummary>(Val: S.get())) |
440 | GlobalInvolvedWithAlias.insert(V: &AS->getAliasee()); |
441 | |
442 | for (auto &I : Index) |
443 | thinLTOResolvePrevailingGUID(C, VI: Index.getValueInfo(R: I), |
444 | GlobalInvolvedWithAlias, isPrevailing, |
445 | recordNewLinkage, GUIDPreservedSymbols); |
446 | } |
447 | |
448 | static void thinLTOInternalizeAndPromoteGUID( |
449 | ValueInfo VI, function_ref<bool(StringRef, ValueInfo)> isExported, |
450 | function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)> |
451 | isPrevailing) { |
452 | auto ExternallyVisibleCopies = |
453 | llvm::count_if(Range: VI.getSummaryList(), |
454 | P: [](const std::unique_ptr<GlobalValueSummary> &Summary) { |
455 | return !GlobalValue::isLocalLinkage(Linkage: Summary->linkage()); |
456 | }); |
457 | |
458 | for (auto &S : VI.getSummaryList()) { |
459 | // First see if we need to promote an internal value because it is not |
460 | // exported. |
461 | if (isExported(S->modulePath(), VI)) { |
462 | if (GlobalValue::isLocalLinkage(Linkage: S->linkage())) |
463 | S->setLinkage(GlobalValue::ExternalLinkage); |
464 | continue; |
465 | } |
466 | |
467 | // Otherwise, see if we can internalize. |
468 | if (!EnableLTOInternalization) |
469 | continue; |
470 | |
471 | // Non-exported values with external linkage can be internalized. |
472 | if (GlobalValue::isExternalLinkage(Linkage: S->linkage())) { |
473 | S->setLinkage(GlobalValue::InternalLinkage); |
474 | continue; |
475 | } |
476 | |
477 | // Non-exported function and variable definitions with a weak-for-linker |
478 | // linkage can be internalized in certain cases. The minimum legality |
479 | // requirements would be that they are not address taken to ensure that we |
480 | // don't break pointer equality checks, and that variables are either read- |
481 | // or write-only. For functions, this is the case if either all copies are |
482 | // [local_]unnamed_addr, or we can propagate reference edge attributes |
483 | // (which is how this is guaranteed for variables, when analyzing whether |
484 | // they are read or write-only). |
485 | // |
486 | // However, we only get to this code for weak-for-linkage values in one of |
487 | // two cases: |
488 | // 1) The prevailing copy is not in IR (it is in native code). |
489 | // 2) The prevailing copy in IR is not exported from its module. |
490 | // Additionally, at least for the new LTO API, case 2 will only happen if |
491 | // there is exactly one definition of the value (i.e. in exactly one |
492 | // module), as duplicate defs are result in the value being marked exported. |
493 | // Likely, users of the legacy LTO API are similar, however, currently there |
494 | // are llvm-lto based tests of the legacy LTO API that do not mark |
495 | // duplicate linkonce_odr copies as exported via the tool, so we need |
496 | // to handle that case below by checking the number of copies. |
497 | // |
498 | // Generally, we only want to internalize a weak-for-linker value in case |
499 | // 2, because in case 1 we cannot see how the value is used to know if it |
500 | // is read or write-only. We also don't want to bloat the binary with |
501 | // multiple internalized copies of non-prevailing linkonce/weak functions. |
502 | // Note if we don't internalize, we will convert non-prevailing copies to |
503 | // available_externally anyway, so that we drop them after inlining. The |
504 | // only reason to internalize such a function is if we indeed have a single |
505 | // copy, because internalizing it won't increase binary size, and enables |
506 | // use of inliner heuristics that are more aggressive in the face of a |
507 | // single call to a static (local). For variables, internalizing a read or |
508 | // write only variable can enable more aggressive optimization. However, we |
509 | // already perform this elsewhere in the ThinLTO backend handling for |
510 | // read or write-only variables (processGlobalForThinLTO). |
511 | // |
512 | // Therefore, only internalize linkonce/weak if there is a single copy, that |
513 | // is prevailing in this IR module. We can do so aggressively, without |
514 | // requiring the address to be insignificant, or that a variable be read or |
515 | // write-only. |
516 | if (!GlobalValue::isWeakForLinker(Linkage: S->linkage()) || |
517 | GlobalValue::isExternalWeakLinkage(Linkage: S->linkage())) |
518 | continue; |
519 | |
520 | if (isPrevailing(VI.getGUID(), S.get()) && ExternallyVisibleCopies == 1) |
521 | S->setLinkage(GlobalValue::InternalLinkage); |
522 | } |
523 | } |
524 | |
525 | // Update the linkages in the given \p Index to mark exported values |
526 | // as external and non-exported values as internal. |
527 | void llvm::thinLTOInternalizeAndPromoteInIndex( |
528 | ModuleSummaryIndex &Index, |
529 | function_ref<bool(StringRef, ValueInfo)> isExported, |
530 | function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)> |
531 | isPrevailing) { |
532 | for (auto &I : Index) |
533 | thinLTOInternalizeAndPromoteGUID(VI: Index.getValueInfo(R: I), isExported, |
534 | isPrevailing); |
535 | } |
536 | |
537 | // Requires a destructor for std::vector<InputModule>. |
538 | InputFile::~InputFile() = default; |
539 | |
540 | Expected<std::unique_ptr<InputFile>> InputFile::create(MemoryBufferRef Object) { |
541 | std::unique_ptr<InputFile> File(new InputFile); |
542 | |
543 | Expected<IRSymtabFile> FOrErr = readIRSymtab(MBRef: Object); |
544 | if (!FOrErr) |
545 | return FOrErr.takeError(); |
546 | |
547 | File->TargetTriple = FOrErr->TheReader.getTargetTriple(); |
548 | File->SourceFileName = FOrErr->TheReader.getSourceFileName(); |
549 | File->COFFLinkerOpts = FOrErr->TheReader.getCOFFLinkerOpts(); |
550 | File->DependentLibraries = FOrErr->TheReader.getDependentLibraries(); |
551 | File->ComdatTable = FOrErr->TheReader.getComdatTable(); |
552 | |
553 | for (unsigned I = 0; I != FOrErr->Mods.size(); ++I) { |
554 | size_t Begin = File->Symbols.size(); |
555 | for (const irsymtab::Reader::SymbolRef &Sym : |
556 | FOrErr->TheReader.module_symbols(I)) |
557 | // Skip symbols that are irrelevant to LTO. Note that this condition needs |
558 | // to match the one in Skip() in LTO::addRegularLTO(). |
559 | if (Sym.isGlobal() && !Sym.isFormatSpecific()) |
560 | File->Symbols.push_back(x: Sym); |
561 | File->ModuleSymIndices.push_back(x: {Begin, File->Symbols.size()}); |
562 | } |
563 | |
564 | File->Mods = FOrErr->Mods; |
565 | File->Strtab = std::move(FOrErr->Strtab); |
566 | return std::move(File); |
567 | } |
568 | |
569 | StringRef InputFile::getName() const { |
570 | return Mods[0].getModuleIdentifier(); |
571 | } |
572 | |
573 | BitcodeModule &InputFile::getSingleBitcodeModule() { |
574 | assert(Mods.size() == 1 && "Expect only one bitcode module" ); |
575 | return Mods[0]; |
576 | } |
577 | |
578 | LTO::RegularLTOState::RegularLTOState(unsigned ParallelCodeGenParallelismLevel, |
579 | const Config &Conf) |
580 | : ParallelCodeGenParallelismLevel(ParallelCodeGenParallelismLevel), |
581 | Ctx(Conf), CombinedModule(std::make_unique<Module>(args: "ld-temp.o" , args&: Ctx)), |
582 | Mover(std::make_unique<IRMover>(args&: *CombinedModule)) {} |
583 | |
584 | LTO::ThinLTOState::ThinLTOState(ThinBackend Backend) |
585 | : Backend(Backend), CombinedIndex(/*HaveGVs*/ false) { |
586 | if (!Backend) |
587 | this->Backend = |
588 | createInProcessThinBackend(Parallelism: llvm::heavyweight_hardware_concurrency()); |
589 | } |
590 | |
591 | LTO::LTO(Config Conf, ThinBackend Backend, |
592 | unsigned ParallelCodeGenParallelismLevel, LTOKind LTOMode) |
593 | : Conf(std::move(Conf)), |
594 | RegularLTO(ParallelCodeGenParallelismLevel, this->Conf), |
595 | ThinLTO(std::move(Backend)), |
596 | GlobalResolutions(std::make_optional<StringMap<GlobalResolution>>()), |
597 | LTOMode(LTOMode) {} |
598 | |
599 | // Requires a destructor for MapVector<BitcodeModule>. |
600 | LTO::~LTO() = default; |
601 | |
602 | // Add the symbols in the given module to the GlobalResolutions map, and resolve |
603 | // their partitions. |
604 | void LTO::addModuleToGlobalRes(ArrayRef<InputFile::Symbol> Syms, |
605 | ArrayRef<SymbolResolution> Res, |
606 | unsigned Partition, bool InSummary) { |
607 | auto *ResI = Res.begin(); |
608 | auto *ResE = Res.end(); |
609 | (void)ResE; |
610 | const Triple TT(RegularLTO.CombinedModule->getTargetTriple()); |
611 | for (const InputFile::Symbol &Sym : Syms) { |
612 | assert(ResI != ResE); |
613 | SymbolResolution Res = *ResI++; |
614 | |
615 | auto &GlobalRes = (*GlobalResolutions)[Sym.getName()]; |
616 | GlobalRes.UnnamedAddr &= Sym.isUnnamedAddr(); |
617 | if (Res.Prevailing) { |
618 | assert(!GlobalRes.Prevailing && |
619 | "Multiple prevailing defs are not allowed" ); |
620 | GlobalRes.Prevailing = true; |
621 | GlobalRes.IRName = std::string(Sym.getIRName()); |
622 | } else if (!GlobalRes.Prevailing && GlobalRes.IRName.empty()) { |
623 | // Sometimes it can be two copies of symbol in a module and prevailing |
624 | // symbol can have no IR name. That might happen if symbol is defined in |
625 | // module level inline asm block. In case we have multiple modules with |
626 | // the same symbol we want to use IR name of the prevailing symbol. |
627 | // Otherwise, if we haven't seen a prevailing symbol, set the name so that |
628 | // we can later use it to check if there is any prevailing copy in IR. |
629 | GlobalRes.IRName = std::string(Sym.getIRName()); |
630 | } |
631 | |
632 | // In rare occasion, the symbol used to initialize GlobalRes has a different |
633 | // IRName from the inspected Symbol. This can happen on macOS + iOS, when a |
634 | // symbol is referenced through its mangled name, say @"\01_symbol" while |
635 | // the IRName is @symbol (the prefix underscore comes from MachO mangling). |
636 | // In that case, we have the same actual Symbol that can get two different |
637 | // GUID, leading to some invalid internalization. Workaround this by marking |
638 | // the GlobalRes external. |
639 | |
640 | // FIXME: instead of this check, it would be desirable to compute GUIDs |
641 | // based on mangled name, but this requires an access to the Target Triple |
642 | // and would be relatively invasive on the codebase. |
643 | if (GlobalRes.IRName != Sym.getIRName()) { |
644 | GlobalRes.Partition = GlobalResolution::External; |
645 | GlobalRes.VisibleOutsideSummary = true; |
646 | } |
647 | |
648 | // Set the partition to external if we know it is re-defined by the linker |
649 | // with -defsym or -wrap options, used elsewhere, e.g. it is visible to a |
650 | // regular object, is referenced from llvm.compiler.used/llvm.used, or was |
651 | // already recorded as being referenced from a different partition. |
652 | if (Res.LinkerRedefined || Res.VisibleToRegularObj || Sym.isUsed() || |
653 | (GlobalRes.Partition != GlobalResolution::Unknown && |
654 | GlobalRes.Partition != Partition)) { |
655 | GlobalRes.Partition = GlobalResolution::External; |
656 | } else |
657 | // First recorded reference, save the current partition. |
658 | GlobalRes.Partition = Partition; |
659 | |
660 | // Flag as visible outside of summary if visible from a regular object or |
661 | // from a module that does not have a summary. |
662 | GlobalRes.VisibleOutsideSummary |= |
663 | (Res.VisibleToRegularObj || Sym.isUsed() || !InSummary); |
664 | |
665 | GlobalRes.ExportDynamic |= Res.ExportDynamic; |
666 | } |
667 | } |
668 | |
669 | static void writeToResolutionFile(raw_ostream &OS, InputFile *Input, |
670 | ArrayRef<SymbolResolution> Res) { |
671 | StringRef Path = Input->getName(); |
672 | OS << Path << '\n'; |
673 | auto ResI = Res.begin(); |
674 | for (const InputFile::Symbol &Sym : Input->symbols()) { |
675 | assert(ResI != Res.end()); |
676 | SymbolResolution Res = *ResI++; |
677 | |
678 | OS << "-r=" << Path << ',' << Sym.getName() << ','; |
679 | if (Res.Prevailing) |
680 | OS << 'p'; |
681 | if (Res.FinalDefinitionInLinkageUnit) |
682 | OS << 'l'; |
683 | if (Res.VisibleToRegularObj) |
684 | OS << 'x'; |
685 | if (Res.LinkerRedefined) |
686 | OS << 'r'; |
687 | OS << '\n'; |
688 | } |
689 | OS.flush(); |
690 | assert(ResI == Res.end()); |
691 | } |
692 | |
693 | Error LTO::add(std::unique_ptr<InputFile> Input, |
694 | ArrayRef<SymbolResolution> Res) { |
695 | assert(!CalledGetMaxTasks); |
696 | |
697 | if (Conf.ResolutionFile) |
698 | writeToResolutionFile(OS&: *Conf.ResolutionFile, Input: Input.get(), Res); |
699 | |
700 | if (RegularLTO.CombinedModule->getTargetTriple().empty()) { |
701 | RegularLTO.CombinedModule->setTargetTriple(Input->getTargetTriple()); |
702 | if (Triple(Input->getTargetTriple()).isOSBinFormatELF()) |
703 | Conf.VisibilityScheme = Config::ELF; |
704 | } |
705 | |
706 | const SymbolResolution *ResI = Res.begin(); |
707 | for (unsigned I = 0; I != Input->Mods.size(); ++I) |
708 | if (Error Err = addModule(Input&: *Input, ModI: I, ResI, ResE: Res.end())) |
709 | return Err; |
710 | |
711 | assert(ResI == Res.end()); |
712 | return Error::success(); |
713 | } |
714 | |
715 | Error LTO::addModule(InputFile &Input, unsigned ModI, |
716 | const SymbolResolution *&ResI, |
717 | const SymbolResolution *ResE) { |
718 | Expected<BitcodeLTOInfo> LTOInfo = Input.Mods[ModI].getLTOInfo(); |
719 | if (!LTOInfo) |
720 | return LTOInfo.takeError(); |
721 | |
722 | if (EnableSplitLTOUnit) { |
723 | // If only some modules were split, flag this in the index so that |
724 | // we can skip or error on optimizations that need consistently split |
725 | // modules (whole program devirt and lower type tests). |
726 | if (*EnableSplitLTOUnit != LTOInfo->EnableSplitLTOUnit) |
727 | ThinLTO.CombinedIndex.setPartiallySplitLTOUnits(); |
728 | } else |
729 | EnableSplitLTOUnit = LTOInfo->EnableSplitLTOUnit; |
730 | |
731 | BitcodeModule BM = Input.Mods[ModI]; |
732 | |
733 | if ((LTOMode == LTOK_UnifiedRegular || LTOMode == LTOK_UnifiedThin) && |
734 | !LTOInfo->UnifiedLTO) |
735 | return make_error<StringError>( |
736 | Args: "unified LTO compilation must use " |
737 | "compatible bitcode modules (use -funified-lto)" , |
738 | Args: inconvertibleErrorCode()); |
739 | |
740 | if (LTOInfo->UnifiedLTO && LTOMode == LTOK_Default) |
741 | LTOMode = LTOK_UnifiedThin; |
742 | |
743 | bool IsThinLTO = LTOInfo->IsThinLTO && (LTOMode != LTOK_UnifiedRegular); |
744 | |
745 | auto ModSyms = Input.module_symbols(I: ModI); |
746 | addModuleToGlobalRes(Syms: ModSyms, Res: {ResI, ResE}, |
747 | Partition: IsThinLTO ? ThinLTO.ModuleMap.size() + 1 : 0, |
748 | InSummary: LTOInfo->HasSummary); |
749 | |
750 | if (IsThinLTO) |
751 | return addThinLTO(BM, Syms: ModSyms, ResI, ResE); |
752 | |
753 | RegularLTO.EmptyCombinedModule = false; |
754 | Expected<RegularLTOState::AddedModule> ModOrErr = |
755 | addRegularLTO(BM, Syms: ModSyms, ResI, ResE); |
756 | if (!ModOrErr) |
757 | return ModOrErr.takeError(); |
758 | |
759 | if (!LTOInfo->HasSummary) |
760 | return linkRegularLTO(Mod: std::move(*ModOrErr), /*LivenessFromIndex=*/false); |
761 | |
762 | // Regular LTO module summaries are added to a dummy module that represents |
763 | // the combined regular LTO module. |
764 | if (Error Err = BM.readSummary(CombinedIndex&: ThinLTO.CombinedIndex, ModulePath: "" )) |
765 | return Err; |
766 | RegularLTO.ModsWithSummaries.push_back(x: std::move(*ModOrErr)); |
767 | return Error::success(); |
768 | } |
769 | |
770 | // Checks whether the given global value is in a non-prevailing comdat |
771 | // (comdat containing values the linker indicated were not prevailing, |
772 | // which we then dropped to available_externally), and if so, removes |
773 | // it from the comdat. This is called for all global values to ensure the |
774 | // comdat is empty rather than leaving an incomplete comdat. It is needed for |
775 | // regular LTO modules, in case we are in a mixed-LTO mode (both regular |
776 | // and thin LTO modules) compilation. Since the regular LTO module will be |
777 | // linked first in the final native link, we want to make sure the linker |
778 | // doesn't select any of these incomplete comdats that would be left |
779 | // in the regular LTO module without this cleanup. |
780 | static void |
781 | handleNonPrevailingComdat(GlobalValue &GV, |
782 | std::set<const Comdat *> &NonPrevailingComdats) { |
783 | Comdat *C = GV.getComdat(); |
784 | if (!C) |
785 | return; |
786 | |
787 | if (!NonPrevailingComdats.count(x: C)) |
788 | return; |
789 | |
790 | // Additionally need to drop all global values from the comdat to |
791 | // available_externally, to satisfy the COMDAT requirement that all members |
792 | // are discarded as a unit. The non-local linkage global values avoid |
793 | // duplicate definition linker errors. |
794 | GV.setLinkage(GlobalValue::AvailableExternallyLinkage); |
795 | |
796 | if (auto GO = dyn_cast<GlobalObject>(Val: &GV)) |
797 | GO->setComdat(nullptr); |
798 | } |
799 | |
800 | // Add a regular LTO object to the link. |
801 | // The resulting module needs to be linked into the combined LTO module with |
802 | // linkRegularLTO. |
803 | Expected<LTO::RegularLTOState::AddedModule> |
804 | LTO::addRegularLTO(BitcodeModule BM, ArrayRef<InputFile::Symbol> Syms, |
805 | const SymbolResolution *&ResI, |
806 | const SymbolResolution *ResE) { |
807 | RegularLTOState::AddedModule Mod; |
808 | Expected<std::unique_ptr<Module>> MOrErr = |
809 | BM.getLazyModule(Context&: RegularLTO.Ctx, /*ShouldLazyLoadMetadata*/ true, |
810 | /*IsImporting*/ false); |
811 | if (!MOrErr) |
812 | return MOrErr.takeError(); |
813 | Module &M = **MOrErr; |
814 | Mod.M = std::move(*MOrErr); |
815 | |
816 | if (Error Err = M.materializeMetadata()) |
817 | return std::move(Err); |
818 | |
819 | // If cfi.functions is present and we are in regular LTO mode, LowerTypeTests |
820 | // will rename local functions in the merged module as "<function name>.1". |
821 | // This causes linking errors, since other parts of the module expect the |
822 | // original function name. |
823 | if (LTOMode == LTOK_UnifiedRegular) |
824 | if (NamedMDNode *CfiFunctionsMD = M.getNamedMetadata(Name: "cfi.functions" )) |
825 | M.eraseNamedMetadata(NMD: CfiFunctionsMD); |
826 | |
827 | UpgradeDebugInfo(M); |
828 | |
829 | ModuleSymbolTable SymTab; |
830 | SymTab.addModule(M: &M); |
831 | |
832 | for (GlobalVariable &GV : M.globals()) |
833 | if (GV.hasAppendingLinkage()) |
834 | Mod.Keep.push_back(x: &GV); |
835 | |
836 | DenseSet<GlobalObject *> AliasedGlobals; |
837 | for (auto &GA : M.aliases()) |
838 | if (GlobalObject *GO = GA.getAliaseeObject()) |
839 | AliasedGlobals.insert(V: GO); |
840 | |
841 | // In this function we need IR GlobalValues matching the symbols in Syms |
842 | // (which is not backed by a module), so we need to enumerate them in the same |
843 | // order. The symbol enumeration order of a ModuleSymbolTable intentionally |
844 | // matches the order of an irsymtab, but when we read the irsymtab in |
845 | // InputFile::create we omit some symbols that are irrelevant to LTO. The |
846 | // Skip() function skips the same symbols from the module as InputFile does |
847 | // from the symbol table. |
848 | auto MsymI = SymTab.symbols().begin(), MsymE = SymTab.symbols().end(); |
849 | auto Skip = [&]() { |
850 | while (MsymI != MsymE) { |
851 | auto Flags = SymTab.getSymbolFlags(S: *MsymI); |
852 | if ((Flags & object::BasicSymbolRef::SF_Global) && |
853 | !(Flags & object::BasicSymbolRef::SF_FormatSpecific)) |
854 | return; |
855 | ++MsymI; |
856 | } |
857 | }; |
858 | Skip(); |
859 | |
860 | std::set<const Comdat *> NonPrevailingComdats; |
861 | SmallSet<StringRef, 2> NonPrevailingAsmSymbols; |
862 | for (const InputFile::Symbol &Sym : Syms) { |
863 | assert(ResI != ResE); |
864 | SymbolResolution Res = *ResI++; |
865 | |
866 | assert(MsymI != MsymE); |
867 | ModuleSymbolTable::Symbol Msym = *MsymI++; |
868 | Skip(); |
869 | |
870 | if (GlobalValue *GV = dyn_cast_if_present<GlobalValue *>(Val&: Msym)) { |
871 | if (Res.Prevailing) { |
872 | if (Sym.isUndefined()) |
873 | continue; |
874 | Mod.Keep.push_back(x: GV); |
875 | // For symbols re-defined with linker -wrap and -defsym options, |
876 | // set the linkage to weak to inhibit IPO. The linkage will be |
877 | // restored by the linker. |
878 | if (Res.LinkerRedefined) |
879 | GV->setLinkage(GlobalValue::WeakAnyLinkage); |
880 | |
881 | GlobalValue::LinkageTypes OriginalLinkage = GV->getLinkage(); |
882 | if (GlobalValue::isLinkOnceLinkage(Linkage: OriginalLinkage)) |
883 | GV->setLinkage(GlobalValue::getWeakLinkage( |
884 | ODR: GlobalValue::isLinkOnceODRLinkage(Linkage: OriginalLinkage))); |
885 | } else if (isa<GlobalObject>(Val: GV) && |
886 | (GV->hasLinkOnceODRLinkage() || GV->hasWeakODRLinkage() || |
887 | GV->hasAvailableExternallyLinkage()) && |
888 | !AliasedGlobals.count(V: cast<GlobalObject>(Val: GV))) { |
889 | // Any of the above three types of linkage indicates that the |
890 | // chosen prevailing symbol will have the same semantics as this copy of |
891 | // the symbol, so we may be able to link it with available_externally |
892 | // linkage. We will decide later whether to do that when we link this |
893 | // module (in linkRegularLTO), based on whether it is undefined. |
894 | Mod.Keep.push_back(x: GV); |
895 | GV->setLinkage(GlobalValue::AvailableExternallyLinkage); |
896 | if (GV->hasComdat()) |
897 | NonPrevailingComdats.insert(x: GV->getComdat()); |
898 | cast<GlobalObject>(Val: GV)->setComdat(nullptr); |
899 | } |
900 | |
901 | // Set the 'local' flag based on the linker resolution for this symbol. |
902 | if (Res.FinalDefinitionInLinkageUnit) { |
903 | GV->setDSOLocal(true); |
904 | if (GV->hasDLLImportStorageClass()) |
905 | GV->setDLLStorageClass(GlobalValue::DLLStorageClassTypes:: |
906 | DefaultStorageClass); |
907 | } |
908 | } else if (auto *AS = |
909 | dyn_cast_if_present<ModuleSymbolTable::AsmSymbol *>(Val&: Msym)) { |
910 | // Collect non-prevailing symbols. |
911 | if (!Res.Prevailing) |
912 | NonPrevailingAsmSymbols.insert(V: AS->first); |
913 | } else { |
914 | llvm_unreachable("unknown symbol type" ); |
915 | } |
916 | |
917 | // Common resolution: collect the maximum size/alignment over all commons. |
918 | // We also record if we see an instance of a common as prevailing, so that |
919 | // if none is prevailing we can ignore it later. |
920 | if (Sym.isCommon()) { |
921 | // FIXME: We should figure out what to do about commons defined by asm. |
922 | // For now they aren't reported correctly by ModuleSymbolTable. |
923 | auto &CommonRes = RegularLTO.Commons[std::string(Sym.getIRName())]; |
924 | CommonRes.Size = std::max(a: CommonRes.Size, b: Sym.getCommonSize()); |
925 | if (uint32_t SymAlignValue = Sym.getCommonAlignment()) { |
926 | CommonRes.Alignment = |
927 | std::max(a: Align(SymAlignValue), b: CommonRes.Alignment); |
928 | } |
929 | CommonRes.Prevailing |= Res.Prevailing; |
930 | } |
931 | } |
932 | |
933 | if (!M.getComdatSymbolTable().empty()) |
934 | for (GlobalValue &GV : M.global_values()) |
935 | handleNonPrevailingComdat(GV, NonPrevailingComdats); |
936 | |
937 | // Prepend ".lto_discard <sym>, <sym>*" directive to each module inline asm |
938 | // block. |
939 | if (!M.getModuleInlineAsm().empty()) { |
940 | std::string NewIA = ".lto_discard" ; |
941 | if (!NonPrevailingAsmSymbols.empty()) { |
942 | // Don't dicard a symbol if there is a live .symver for it. |
943 | ModuleSymbolTable::CollectAsmSymvers( |
944 | M, AsmSymver: [&](StringRef Name, StringRef Alias) { |
945 | if (!NonPrevailingAsmSymbols.count(V: Alias)) |
946 | NonPrevailingAsmSymbols.erase(V: Name); |
947 | }); |
948 | NewIA += " " + llvm::join(R&: NonPrevailingAsmSymbols, Separator: ", " ); |
949 | } |
950 | NewIA += "\n" ; |
951 | M.setModuleInlineAsm(NewIA + M.getModuleInlineAsm()); |
952 | } |
953 | |
954 | assert(MsymI == MsymE); |
955 | return std::move(Mod); |
956 | } |
957 | |
958 | Error LTO::linkRegularLTO(RegularLTOState::AddedModule Mod, |
959 | bool LivenessFromIndex) { |
960 | std::vector<GlobalValue *> Keep; |
961 | for (GlobalValue *GV : Mod.Keep) { |
962 | if (LivenessFromIndex && !ThinLTO.CombinedIndex.isGUIDLive(GUID: GV->getGUID())) { |
963 | if (Function *F = dyn_cast<Function>(Val: GV)) { |
964 | if (DiagnosticOutputFile) { |
965 | if (Error Err = F->materialize()) |
966 | return Err; |
967 | OptimizationRemarkEmitter ORE(F, nullptr); |
968 | ORE.emit(OptDiag&: OptimizationRemark(DEBUG_TYPE, "deadfunction" , F) |
969 | << ore::NV("Function" , F) |
970 | << " not added to the combined module " ); |
971 | } |
972 | } |
973 | continue; |
974 | } |
975 | |
976 | if (!GV->hasAvailableExternallyLinkage()) { |
977 | Keep.push_back(x: GV); |
978 | continue; |
979 | } |
980 | |
981 | // Only link available_externally definitions if we don't already have a |
982 | // definition. |
983 | GlobalValue *CombinedGV = |
984 | RegularLTO.CombinedModule->getNamedValue(Name: GV->getName()); |
985 | if (CombinedGV && !CombinedGV->isDeclaration()) |
986 | continue; |
987 | |
988 | Keep.push_back(x: GV); |
989 | } |
990 | |
991 | return RegularLTO.Mover->move(Src: std::move(Mod.M), ValuesToLink: Keep, AddLazyFor: nullptr, |
992 | /* IsPerformingImport */ false); |
993 | } |
994 | |
995 | // Add a ThinLTO module to the link. |
996 | Error LTO::addThinLTO(BitcodeModule BM, ArrayRef<InputFile::Symbol> Syms, |
997 | const SymbolResolution *&ResI, |
998 | const SymbolResolution *ResE) { |
999 | const SymbolResolution *ResITmp = ResI; |
1000 | for (const InputFile::Symbol &Sym : Syms) { |
1001 | assert(ResITmp != ResE); |
1002 | SymbolResolution Res = *ResITmp++; |
1003 | |
1004 | if (!Sym.getIRName().empty()) { |
1005 | auto GUID = GlobalValue::getGUID(GlobalName: GlobalValue::getGlobalIdentifier( |
1006 | Name: Sym.getIRName(), Linkage: GlobalValue::ExternalLinkage, FileName: "" )); |
1007 | if (Res.Prevailing) |
1008 | ThinLTO.PrevailingModuleForGUID[GUID] = BM.getModuleIdentifier(); |
1009 | } |
1010 | } |
1011 | |
1012 | if (Error Err = |
1013 | BM.readSummary(CombinedIndex&: ThinLTO.CombinedIndex, ModulePath: BM.getModuleIdentifier(), |
1014 | IsPrevailing: [&](GlobalValue::GUID GUID) { |
1015 | return ThinLTO.PrevailingModuleForGUID[GUID] == |
1016 | BM.getModuleIdentifier(); |
1017 | })) |
1018 | return Err; |
1019 | LLVM_DEBUG(dbgs() << "Module " << BM.getModuleIdentifier() << "\n" ); |
1020 | |
1021 | for (const InputFile::Symbol &Sym : Syms) { |
1022 | assert(ResI != ResE); |
1023 | SymbolResolution Res = *ResI++; |
1024 | |
1025 | if (!Sym.getIRName().empty()) { |
1026 | auto GUID = GlobalValue::getGUID(GlobalName: GlobalValue::getGlobalIdentifier( |
1027 | Name: Sym.getIRName(), Linkage: GlobalValue::ExternalLinkage, FileName: "" )); |
1028 | if (Res.Prevailing) { |
1029 | assert(ThinLTO.PrevailingModuleForGUID[GUID] == |
1030 | BM.getModuleIdentifier()); |
1031 | |
1032 | // For linker redefined symbols (via --wrap or --defsym) we want to |
1033 | // switch the linkage to `weak` to prevent IPOs from happening. |
1034 | // Find the summary in the module for this very GV and record the new |
1035 | // linkage so that we can switch it when we import the GV. |
1036 | if (Res.LinkerRedefined) |
1037 | if (auto S = ThinLTO.CombinedIndex.findSummaryInModule( |
1038 | ValueGUID: GUID, ModuleId: BM.getModuleIdentifier())) |
1039 | S->setLinkage(GlobalValue::WeakAnyLinkage); |
1040 | } |
1041 | |
1042 | // If the linker resolved the symbol to a local definition then mark it |
1043 | // as local in the summary for the module we are adding. |
1044 | if (Res.FinalDefinitionInLinkageUnit) { |
1045 | if (auto S = ThinLTO.CombinedIndex.findSummaryInModule( |
1046 | ValueGUID: GUID, ModuleId: BM.getModuleIdentifier())) { |
1047 | S->setDSOLocal(true); |
1048 | } |
1049 | } |
1050 | } |
1051 | } |
1052 | |
1053 | if (!ThinLTO.ModuleMap.insert(KV: {BM.getModuleIdentifier(), BM}).second) |
1054 | return make_error<StringError>( |
1055 | Args: "Expected at most one ThinLTO module per bitcode file" , |
1056 | Args: inconvertibleErrorCode()); |
1057 | |
1058 | if (!Conf.ThinLTOModulesToCompile.empty()) { |
1059 | if (!ThinLTO.ModulesToCompile) |
1060 | ThinLTO.ModulesToCompile = ModuleMapType(); |
1061 | // This is a fuzzy name matching where only modules with name containing the |
1062 | // specified switch values are going to be compiled. |
1063 | for (const std::string &Name : Conf.ThinLTOModulesToCompile) { |
1064 | if (BM.getModuleIdentifier().contains(Other: Name)) { |
1065 | ThinLTO.ModulesToCompile->insert(KV: {BM.getModuleIdentifier(), BM}); |
1066 | llvm::errs() << "[ThinLTO] Selecting " << BM.getModuleIdentifier() |
1067 | << " to compile\n" ; |
1068 | } |
1069 | } |
1070 | } |
1071 | |
1072 | return Error::success(); |
1073 | } |
1074 | |
1075 | unsigned LTO::getMaxTasks() const { |
1076 | CalledGetMaxTasks = true; |
1077 | auto ModuleCount = ThinLTO.ModulesToCompile ? ThinLTO.ModulesToCompile->size() |
1078 | : ThinLTO.ModuleMap.size(); |
1079 | return RegularLTO.ParallelCodeGenParallelismLevel + ModuleCount; |
1080 | } |
1081 | |
1082 | // If only some of the modules were split, we cannot correctly handle |
1083 | // code that contains type tests or type checked loads. |
1084 | Error LTO::checkPartiallySplit() { |
1085 | if (!ThinLTO.CombinedIndex.partiallySplitLTOUnits()) |
1086 | return Error::success(); |
1087 | |
1088 | Function *TypeTestFunc = RegularLTO.CombinedModule->getFunction( |
1089 | Name: Intrinsic::getName(Intrinsic::type_test)); |
1090 | Function *TypeCheckedLoadFunc = RegularLTO.CombinedModule->getFunction( |
1091 | Name: Intrinsic::getName(Intrinsic::type_checked_load)); |
1092 | Function *TypeCheckedLoadRelativeFunc = |
1093 | RegularLTO.CombinedModule->getFunction( |
1094 | Name: Intrinsic::getName(Intrinsic::type_checked_load_relative)); |
1095 | |
1096 | // First check if there are type tests / type checked loads in the |
1097 | // merged regular LTO module IR. |
1098 | if ((TypeTestFunc && !TypeTestFunc->use_empty()) || |
1099 | (TypeCheckedLoadFunc && !TypeCheckedLoadFunc->use_empty()) || |
1100 | (TypeCheckedLoadRelativeFunc && |
1101 | !TypeCheckedLoadRelativeFunc->use_empty())) |
1102 | return make_error<StringError>( |
1103 | Args: "inconsistent LTO Unit splitting (recompile with -fsplit-lto-unit)" , |
1104 | Args: inconvertibleErrorCode()); |
1105 | |
1106 | // Otherwise check if there are any recorded in the combined summary from the |
1107 | // ThinLTO modules. |
1108 | for (auto &P : ThinLTO.CombinedIndex) { |
1109 | for (auto &S : P.second.SummaryList) { |
1110 | auto *FS = dyn_cast<FunctionSummary>(Val: S.get()); |
1111 | if (!FS) |
1112 | continue; |
1113 | if (!FS->type_test_assume_vcalls().empty() || |
1114 | !FS->type_checked_load_vcalls().empty() || |
1115 | !FS->type_test_assume_const_vcalls().empty() || |
1116 | !FS->type_checked_load_const_vcalls().empty() || |
1117 | !FS->type_tests().empty()) |
1118 | return make_error<StringError>( |
1119 | Args: "inconsistent LTO Unit splitting (recompile with -fsplit-lto-unit)" , |
1120 | Args: inconvertibleErrorCode()); |
1121 | } |
1122 | } |
1123 | return Error::success(); |
1124 | } |
1125 | |
1126 | Error LTO::run(AddStreamFn AddStream, FileCache Cache) { |
1127 | // Compute "dead" symbols, we don't want to import/export these! |
1128 | DenseSet<GlobalValue::GUID> GUIDPreservedSymbols; |
1129 | DenseMap<GlobalValue::GUID, PrevailingType> GUIDPrevailingResolutions; |
1130 | for (auto &Res : *GlobalResolutions) { |
1131 | // Normally resolution have IR name of symbol. We can do nothing here |
1132 | // otherwise. See comments in GlobalResolution struct for more details. |
1133 | if (Res.second.IRName.empty()) |
1134 | continue; |
1135 | |
1136 | GlobalValue::GUID GUID = GlobalValue::getGUID( |
1137 | GlobalName: GlobalValue::dropLLVMManglingEscape(Name: Res.second.IRName)); |
1138 | |
1139 | if (Res.second.VisibleOutsideSummary && Res.second.Prevailing) |
1140 | GUIDPreservedSymbols.insert(V: GUID); |
1141 | |
1142 | if (Res.second.ExportDynamic) |
1143 | DynamicExportSymbols.insert(V: GUID); |
1144 | |
1145 | GUIDPrevailingResolutions[GUID] = |
1146 | Res.second.Prevailing ? PrevailingType::Yes : PrevailingType::No; |
1147 | } |
1148 | |
1149 | auto isPrevailing = [&](GlobalValue::GUID G) { |
1150 | auto It = GUIDPrevailingResolutions.find(Val: G); |
1151 | if (It == GUIDPrevailingResolutions.end()) |
1152 | return PrevailingType::Unknown; |
1153 | return It->second; |
1154 | }; |
1155 | computeDeadSymbolsWithConstProp(Index&: ThinLTO.CombinedIndex, GUIDPreservedSymbols, |
1156 | isPrevailing, ImportEnabled: Conf.OptLevel > 0); |
1157 | |
1158 | // Setup output file to emit statistics. |
1159 | auto StatsFileOrErr = setupStatsFile(Conf.StatsFile); |
1160 | if (!StatsFileOrErr) |
1161 | return StatsFileOrErr.takeError(); |
1162 | std::unique_ptr<ToolOutputFile> StatsFile = std::move(StatsFileOrErr.get()); |
1163 | |
1164 | // TODO: Ideally this would be controlled automatically by detecting that we |
1165 | // are linking with an allocator that supports these interfaces, rather than |
1166 | // an internal option (which would still be needed for tests, however). For |
1167 | // example, if the library exported a symbol like __malloc_hot_cold the linker |
1168 | // could recognize that and set a flag in the lto::Config. |
1169 | if (SupportsHotColdNew) |
1170 | ThinLTO.CombinedIndex.setWithSupportsHotColdNew(); |
1171 | |
1172 | Error Result = runRegularLTO(AddStream); |
1173 | if (!Result) |
1174 | // This will reset the GlobalResolutions optional once done with it to |
1175 | // reduce peak memory before importing. |
1176 | Result = runThinLTO(AddStream, Cache, GUIDPreservedSymbols); |
1177 | |
1178 | if (StatsFile) |
1179 | PrintStatisticsJSON(OS&: StatsFile->os()); |
1180 | |
1181 | return Result; |
1182 | } |
1183 | |
1184 | void lto::updateMemProfAttributes(Module &Mod, |
1185 | const ModuleSummaryIndex &Index) { |
1186 | if (Index.withSupportsHotColdNew()) |
1187 | return; |
1188 | |
1189 | // The profile matcher applies hotness attributes directly for allocations, |
1190 | // and those will cause us to generate calls to the hot/cold interfaces |
1191 | // unconditionally. If supports-hot-cold-new was not enabled in the LTO |
1192 | // link then assume we don't want these calls (e.g. not linking with |
1193 | // the appropriate library, or otherwise trying to disable this behavior). |
1194 | for (auto &F : Mod) { |
1195 | for (auto &BB : F) { |
1196 | for (auto &I : BB) { |
1197 | auto *CI = dyn_cast<CallBase>(Val: &I); |
1198 | if (!CI) |
1199 | continue; |
1200 | if (CI->hasFnAttr(Kind: "memprof" )) |
1201 | CI->removeFnAttr(Kind: "memprof" ); |
1202 | // Strip off all memprof metadata as it is no longer needed. |
1203 | // Importantly, this avoids the addition of new memprof attributes |
1204 | // after inlining propagation. |
1205 | // TODO: If we support additional types of MemProf metadata beyond hot |
1206 | // and cold, we will need to update the metadata based on the allocator |
1207 | // APIs supported instead of completely stripping all. |
1208 | CI->setMetadata(KindID: LLVMContext::MD_memprof, Node: nullptr); |
1209 | CI->setMetadata(KindID: LLVMContext::MD_callsite, Node: nullptr); |
1210 | } |
1211 | } |
1212 | } |
1213 | } |
1214 | |
1215 | Error LTO::runRegularLTO(AddStreamFn AddStream) { |
1216 | // Setup optimization remarks. |
1217 | auto DiagFileOrErr = lto::setupLLVMOptimizationRemarks( |
1218 | Context&: RegularLTO.CombinedModule->getContext(), RemarksFilename: Conf.RemarksFilename, |
1219 | RemarksPasses: Conf.RemarksPasses, RemarksFormat: Conf.RemarksFormat, RemarksWithHotness: Conf.RemarksWithHotness, |
1220 | RemarksHotnessThreshold: Conf.RemarksHotnessThreshold); |
1221 | LLVM_DEBUG(dbgs() << "Running regular LTO\n" ); |
1222 | if (!DiagFileOrErr) |
1223 | return DiagFileOrErr.takeError(); |
1224 | DiagnosticOutputFile = std::move(*DiagFileOrErr); |
1225 | |
1226 | // Finalize linking of regular LTO modules containing summaries now that |
1227 | // we have computed liveness information. |
1228 | for (auto &M : RegularLTO.ModsWithSummaries) |
1229 | if (Error Err = linkRegularLTO(Mod: std::move(M), |
1230 | /*LivenessFromIndex=*/true)) |
1231 | return Err; |
1232 | |
1233 | // Ensure we don't have inconsistently split LTO units with type tests. |
1234 | // FIXME: this checks both LTO and ThinLTO. It happens to work as we take |
1235 | // this path both cases but eventually this should be split into two and |
1236 | // do the ThinLTO checks in `runThinLTO`. |
1237 | if (Error Err = checkPartiallySplit()) |
1238 | return Err; |
1239 | |
1240 | // Make sure commons have the right size/alignment: we kept the largest from |
1241 | // all the prevailing when adding the inputs, and we apply it here. |
1242 | const DataLayout &DL = RegularLTO.CombinedModule->getDataLayout(); |
1243 | for (auto &I : RegularLTO.Commons) { |
1244 | if (!I.second.Prevailing) |
1245 | // Don't do anything if no instance of this common was prevailing. |
1246 | continue; |
1247 | GlobalVariable *OldGV = RegularLTO.CombinedModule->getNamedGlobal(Name: I.first); |
1248 | if (OldGV && DL.getTypeAllocSize(Ty: OldGV->getValueType()) == I.second.Size) { |
1249 | // Don't create a new global if the type is already correct, just make |
1250 | // sure the alignment is correct. |
1251 | OldGV->setAlignment(I.second.Alignment); |
1252 | continue; |
1253 | } |
1254 | ArrayType *Ty = |
1255 | ArrayType::get(ElementType: Type::getInt8Ty(C&: RegularLTO.Ctx), NumElements: I.second.Size); |
1256 | auto *GV = new GlobalVariable(*RegularLTO.CombinedModule, Ty, false, |
1257 | GlobalValue::CommonLinkage, |
1258 | ConstantAggregateZero::get(Ty), "" ); |
1259 | GV->setAlignment(I.second.Alignment); |
1260 | if (OldGV) { |
1261 | OldGV->replaceAllUsesWith(V: GV); |
1262 | GV->takeName(V: OldGV); |
1263 | OldGV->eraseFromParent(); |
1264 | } else { |
1265 | GV->setName(I.first); |
1266 | } |
1267 | } |
1268 | |
1269 | updateMemProfAttributes(Mod&: *RegularLTO.CombinedModule, Index: ThinLTO.CombinedIndex); |
1270 | |
1271 | bool WholeProgramVisibilityEnabledInLTO = |
1272 | Conf.HasWholeProgramVisibility && |
1273 | // If validation is enabled, upgrade visibility only when all vtables |
1274 | // have typeinfos. |
1275 | (!Conf.ValidateAllVtablesHaveTypeInfos || Conf.AllVtablesHaveTypeInfos); |
1276 | |
1277 | // This returns true when the name is local or not defined. Locals are |
1278 | // expected to be handled separately. |
1279 | auto IsVisibleToRegularObj = [&](StringRef name) { |
1280 | auto It = GlobalResolutions->find(Key: name); |
1281 | return (It == GlobalResolutions->end() || It->second.VisibleOutsideSummary); |
1282 | }; |
1283 | |
1284 | // If allowed, upgrade public vcall visibility metadata to linkage unit |
1285 | // visibility before whole program devirtualization in the optimizer. |
1286 | updateVCallVisibilityInModule( |
1287 | M&: *RegularLTO.CombinedModule, WholeProgramVisibilityEnabledInLTO, |
1288 | DynamicExportSymbols, ValidateAllVtablesHaveTypeInfos: Conf.ValidateAllVtablesHaveTypeInfos, |
1289 | IsVisibleToRegularObj); |
1290 | updatePublicTypeTestCalls(M&: *RegularLTO.CombinedModule, |
1291 | WholeProgramVisibilityEnabledInLTO); |
1292 | |
1293 | if (Conf.PreOptModuleHook && |
1294 | !Conf.PreOptModuleHook(0, *RegularLTO.CombinedModule)) |
1295 | return finalizeOptimizationRemarks(DiagOutputFile: std::move(DiagnosticOutputFile)); |
1296 | |
1297 | if (!Conf.CodeGenOnly) { |
1298 | for (const auto &R : *GlobalResolutions) { |
1299 | GlobalValue *GV = |
1300 | RegularLTO.CombinedModule->getNamedValue(Name: R.second.IRName); |
1301 | if (!R.second.isPrevailingIRSymbol()) |
1302 | continue; |
1303 | if (R.second.Partition != 0 && |
1304 | R.second.Partition != GlobalResolution::External) |
1305 | continue; |
1306 | |
1307 | // Ignore symbols defined in other partitions. |
1308 | // Also skip declarations, which are not allowed to have internal linkage. |
1309 | if (!GV || GV->hasLocalLinkage() || GV->isDeclaration()) |
1310 | continue; |
1311 | |
1312 | // Symbols that are marked DLLImport or DLLExport should not be |
1313 | // internalized, as they are either externally visible or referencing |
1314 | // external symbols. Symbols that have AvailableExternally or Appending |
1315 | // linkage might be used by future passes and should be kept as is. |
1316 | // These linkages are seen in Unified regular LTO, because the process |
1317 | // of creating split LTO units introduces symbols with that linkage into |
1318 | // one of the created modules. Normally, only the ThinLTO backend would |
1319 | // compile this module, but Unified Regular LTO processes both |
1320 | // modules created by the splitting process as regular LTO modules. |
1321 | if ((LTOMode == LTOKind::LTOK_UnifiedRegular) && |
1322 | ((GV->getDLLStorageClass() != GlobalValue::DefaultStorageClass) || |
1323 | GV->hasAvailableExternallyLinkage() || GV->hasAppendingLinkage())) |
1324 | continue; |
1325 | |
1326 | GV->setUnnamedAddr(R.second.UnnamedAddr ? GlobalValue::UnnamedAddr::Global |
1327 | : GlobalValue::UnnamedAddr::None); |
1328 | if (EnableLTOInternalization && R.second.Partition == 0) |
1329 | GV->setLinkage(GlobalValue::InternalLinkage); |
1330 | } |
1331 | |
1332 | if (Conf.PostInternalizeModuleHook && |
1333 | !Conf.PostInternalizeModuleHook(0, *RegularLTO.CombinedModule)) |
1334 | return finalizeOptimizationRemarks(DiagOutputFile: std::move(DiagnosticOutputFile)); |
1335 | } |
1336 | |
1337 | if (!RegularLTO.EmptyCombinedModule || Conf.AlwaysEmitRegularLTOObj) { |
1338 | if (Error Err = |
1339 | backend(C: Conf, AddStream, ParallelCodeGenParallelismLevel: RegularLTO.ParallelCodeGenParallelismLevel, |
1340 | M&: *RegularLTO.CombinedModule, CombinedIndex&: ThinLTO.CombinedIndex)) |
1341 | return Err; |
1342 | } |
1343 | |
1344 | return finalizeOptimizationRemarks(DiagOutputFile: std::move(DiagnosticOutputFile)); |
1345 | } |
1346 | |
1347 | static const char *libcallRoutineNames[] = { |
1348 | #define HANDLE_LIBCALL(code, name) name, |
1349 | #include "llvm/IR/RuntimeLibcalls.def" |
1350 | #undef HANDLE_LIBCALL |
1351 | }; |
1352 | |
1353 | ArrayRef<const char*> LTO::getRuntimeLibcallSymbols() { |
1354 | return ArrayRef(libcallRoutineNames); |
1355 | } |
1356 | |
1357 | /// This class defines the interface to the ThinLTO backend. |
1358 | class lto::ThinBackendProc { |
1359 | protected: |
1360 | const Config &Conf; |
1361 | ModuleSummaryIndex &CombinedIndex; |
1362 | const DenseMap<StringRef, GVSummaryMapTy> &ModuleToDefinedGVSummaries; |
1363 | lto::IndexWriteCallback OnWrite; |
1364 | bool ShouldEmitImportsFiles; |
1365 | |
1366 | public: |
1367 | ThinBackendProc( |
1368 | const Config &Conf, ModuleSummaryIndex &CombinedIndex, |
1369 | const DenseMap<StringRef, GVSummaryMapTy> &ModuleToDefinedGVSummaries, |
1370 | lto::IndexWriteCallback OnWrite, bool ShouldEmitImportsFiles) |
1371 | : Conf(Conf), CombinedIndex(CombinedIndex), |
1372 | ModuleToDefinedGVSummaries(ModuleToDefinedGVSummaries), |
1373 | OnWrite(OnWrite), ShouldEmitImportsFiles(ShouldEmitImportsFiles) {} |
1374 | |
1375 | virtual ~ThinBackendProc() = default; |
1376 | virtual Error start( |
1377 | unsigned Task, BitcodeModule BM, |
1378 | const FunctionImporter::ImportMapTy &ImportList, |
1379 | const FunctionImporter::ExportSetTy &ExportList, |
1380 | const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR, |
1381 | MapVector<StringRef, BitcodeModule> &ModuleMap) = 0; |
1382 | virtual Error wait() = 0; |
1383 | virtual unsigned getThreadCount() = 0; |
1384 | |
1385 | // Write sharded indices and (optionally) imports to disk |
1386 | Error emitFiles(const FunctionImporter::ImportMapTy &ImportList, |
1387 | llvm::StringRef ModulePath, |
1388 | const std::string &NewModulePath) { |
1389 | std::map<std::string, GVSummaryMapTy> ModuleToSummariesForIndex; |
1390 | std::error_code EC; |
1391 | gatherImportedSummariesForModule(ModulePath, ModuleToDefinedGVSummaries, |
1392 | ImportList, ModuleToSummariesForIndex); |
1393 | |
1394 | raw_fd_ostream OS(NewModulePath + ".thinlto.bc" , EC, |
1395 | sys::fs::OpenFlags::OF_None); |
1396 | if (EC) |
1397 | return errorCodeToError(EC); |
1398 | writeIndexToFile(Index: CombinedIndex, Out&: OS, ModuleToSummariesForIndex: &ModuleToSummariesForIndex); |
1399 | |
1400 | if (ShouldEmitImportsFiles) { |
1401 | EC = EmitImportsFiles(ModulePath, OutputFilename: NewModulePath + ".imports" , |
1402 | ModuleToSummariesForIndex); |
1403 | if (EC) |
1404 | return errorCodeToError(EC); |
1405 | } |
1406 | return Error::success(); |
1407 | } |
1408 | }; |
1409 | |
1410 | namespace { |
1411 | class InProcessThinBackend : public ThinBackendProc { |
1412 | ThreadPool BackendThreadPool; |
1413 | AddStreamFn AddStream; |
1414 | FileCache Cache; |
1415 | std::set<GlobalValue::GUID> CfiFunctionDefs; |
1416 | std::set<GlobalValue::GUID> CfiFunctionDecls; |
1417 | |
1418 | std::optional<Error> Err; |
1419 | std::mutex ErrMu; |
1420 | |
1421 | bool ShouldEmitIndexFiles; |
1422 | |
1423 | public: |
1424 | InProcessThinBackend( |
1425 | const Config &Conf, ModuleSummaryIndex &CombinedIndex, |
1426 | ThreadPoolStrategy ThinLTOParallelism, |
1427 | const DenseMap<StringRef, GVSummaryMapTy> &ModuleToDefinedGVSummaries, |
1428 | AddStreamFn AddStream, FileCache Cache, lto::IndexWriteCallback OnWrite, |
1429 | bool ShouldEmitIndexFiles, bool ShouldEmitImportsFiles) |
1430 | : ThinBackendProc(Conf, CombinedIndex, ModuleToDefinedGVSummaries, |
1431 | OnWrite, ShouldEmitImportsFiles), |
1432 | BackendThreadPool(ThinLTOParallelism), AddStream(std::move(AddStream)), |
1433 | Cache(std::move(Cache)), ShouldEmitIndexFiles(ShouldEmitIndexFiles) { |
1434 | for (auto &Name : CombinedIndex.cfiFunctionDefs()) |
1435 | CfiFunctionDefs.insert( |
1436 | x: GlobalValue::getGUID(GlobalName: GlobalValue::dropLLVMManglingEscape(Name))); |
1437 | for (auto &Name : CombinedIndex.cfiFunctionDecls()) |
1438 | CfiFunctionDecls.insert( |
1439 | x: GlobalValue::getGUID(GlobalName: GlobalValue::dropLLVMManglingEscape(Name))); |
1440 | } |
1441 | |
1442 | Error runThinLTOBackendThread( |
1443 | AddStreamFn AddStream, FileCache Cache, unsigned Task, BitcodeModule BM, |
1444 | ModuleSummaryIndex &CombinedIndex, |
1445 | const FunctionImporter::ImportMapTy &ImportList, |
1446 | const FunctionImporter::ExportSetTy &ExportList, |
1447 | const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR, |
1448 | const GVSummaryMapTy &DefinedGlobals, |
1449 | MapVector<StringRef, BitcodeModule> &ModuleMap) { |
1450 | auto RunThinBackend = [&](AddStreamFn AddStream) { |
1451 | LTOLLVMContext BackendContext(Conf); |
1452 | Expected<std::unique_ptr<Module>> MOrErr = BM.parseModule(Context&: BackendContext); |
1453 | if (!MOrErr) |
1454 | return MOrErr.takeError(); |
1455 | |
1456 | return thinBackend(C: Conf, Task, AddStream, M&: **MOrErr, CombinedIndex, |
1457 | ImportList, DefinedGlobals, ModuleMap: &ModuleMap); |
1458 | }; |
1459 | |
1460 | auto ModuleID = BM.getModuleIdentifier(); |
1461 | |
1462 | if (ShouldEmitIndexFiles) { |
1463 | if (auto E = emitFiles(ImportList, ModulePath: ModuleID, NewModulePath: ModuleID.str())) |
1464 | return E; |
1465 | } |
1466 | |
1467 | if (!Cache || !CombinedIndex.modulePaths().count(Key: ModuleID) || |
1468 | all_of(Range: CombinedIndex.getModuleHash(ModPath: ModuleID), |
1469 | P: [](uint32_t V) { return V == 0; })) |
1470 | // Cache disabled or no entry for this module in the combined index or |
1471 | // no module hash. |
1472 | return RunThinBackend(AddStream); |
1473 | |
1474 | SmallString<40> Key; |
1475 | // The module may be cached, this helps handling it. |
1476 | computeLTOCacheKey(Key, Conf, Index: CombinedIndex, ModuleID, ImportList, |
1477 | ExportList, ResolvedODR, DefinedGlobals, CfiFunctionDefs, |
1478 | CfiFunctionDecls); |
1479 | Expected<AddStreamFn> CacheAddStreamOrErr = Cache(Task, Key, ModuleID); |
1480 | if (Error Err = CacheAddStreamOrErr.takeError()) |
1481 | return Err; |
1482 | AddStreamFn &CacheAddStream = *CacheAddStreamOrErr; |
1483 | if (CacheAddStream) |
1484 | return RunThinBackend(CacheAddStream); |
1485 | |
1486 | return Error::success(); |
1487 | } |
1488 | |
1489 | Error start( |
1490 | unsigned Task, BitcodeModule BM, |
1491 | const FunctionImporter::ImportMapTy &ImportList, |
1492 | const FunctionImporter::ExportSetTy &ExportList, |
1493 | const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR, |
1494 | MapVector<StringRef, BitcodeModule> &ModuleMap) override { |
1495 | StringRef ModulePath = BM.getModuleIdentifier(); |
1496 | assert(ModuleToDefinedGVSummaries.count(ModulePath)); |
1497 | const GVSummaryMapTy &DefinedGlobals = |
1498 | ModuleToDefinedGVSummaries.find(Val: ModulePath)->second; |
1499 | BackendThreadPool.async( |
1500 | F: [=](BitcodeModule BM, ModuleSummaryIndex &CombinedIndex, |
1501 | const FunctionImporter::ImportMapTy &ImportList, |
1502 | const FunctionImporter::ExportSetTy &ExportList, |
1503 | const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> |
1504 | &ResolvedODR, |
1505 | const GVSummaryMapTy &DefinedGlobals, |
1506 | MapVector<StringRef, BitcodeModule> &ModuleMap) { |
1507 | if (LLVM_ENABLE_THREADS && Conf.TimeTraceEnabled) |
1508 | timeTraceProfilerInitialize(TimeTraceGranularity: Conf.TimeTraceGranularity, |
1509 | ProcName: "thin backend" ); |
1510 | Error E = runThinLTOBackendThread( |
1511 | AddStream, Cache, Task, BM, CombinedIndex, ImportList, ExportList, |
1512 | ResolvedODR, DefinedGlobals, ModuleMap); |
1513 | if (E) { |
1514 | std::unique_lock<std::mutex> L(ErrMu); |
1515 | if (Err) |
1516 | Err = joinErrors(E1: std::move(*Err), E2: std::move(E)); |
1517 | else |
1518 | Err = std::move(E); |
1519 | } |
1520 | if (LLVM_ENABLE_THREADS && Conf.TimeTraceEnabled) |
1521 | timeTraceProfilerFinishThread(); |
1522 | }, |
1523 | ArgList&: BM, ArgList: std::ref(t&: CombinedIndex), ArgList: std::ref(t: ImportList), ArgList: std::ref(t: ExportList), |
1524 | ArgList: std::ref(t: ResolvedODR), ArgList: std::ref(t: DefinedGlobals), ArgList: std::ref(t&: ModuleMap)); |
1525 | |
1526 | if (OnWrite) |
1527 | OnWrite(std::string(ModulePath)); |
1528 | return Error::success(); |
1529 | } |
1530 | |
1531 | Error wait() override { |
1532 | BackendThreadPool.wait(); |
1533 | if (Err) |
1534 | return std::move(*Err); |
1535 | else |
1536 | return Error::success(); |
1537 | } |
1538 | |
1539 | unsigned getThreadCount() override { |
1540 | return BackendThreadPool.getThreadCount(); |
1541 | } |
1542 | }; |
1543 | } // end anonymous namespace |
1544 | |
1545 | ThinBackend lto::createInProcessThinBackend(ThreadPoolStrategy Parallelism, |
1546 | lto::IndexWriteCallback OnWrite, |
1547 | bool ShouldEmitIndexFiles, |
1548 | bool ShouldEmitImportsFiles) { |
1549 | return |
1550 | [=](const Config &Conf, ModuleSummaryIndex &CombinedIndex, |
1551 | const DenseMap<StringRef, GVSummaryMapTy> &ModuleToDefinedGVSummaries, |
1552 | AddStreamFn AddStream, FileCache Cache) { |
1553 | return std::make_unique<InProcessThinBackend>( |
1554 | args: Conf, args&: CombinedIndex, args: Parallelism, args: ModuleToDefinedGVSummaries, |
1555 | args&: AddStream, args&: Cache, args: OnWrite, args: ShouldEmitIndexFiles, |
1556 | args: ShouldEmitImportsFiles); |
1557 | }; |
1558 | } |
1559 | |
1560 | // Given the original \p Path to an output file, replace any path |
1561 | // prefix matching \p OldPrefix with \p NewPrefix. Also, create the |
1562 | // resulting directory if it does not yet exist. |
1563 | std::string lto::getThinLTOOutputFile(StringRef Path, StringRef OldPrefix, |
1564 | StringRef NewPrefix) { |
1565 | if (OldPrefix.empty() && NewPrefix.empty()) |
1566 | return std::string(Path); |
1567 | SmallString<128> NewPath(Path); |
1568 | llvm::sys::path::replace_path_prefix(Path&: NewPath, OldPrefix, NewPrefix); |
1569 | StringRef ParentPath = llvm::sys::path::parent_path(path: NewPath.str()); |
1570 | if (!ParentPath.empty()) { |
1571 | // Make sure the new directory exists, creating it if necessary. |
1572 | if (std::error_code EC = llvm::sys::fs::create_directories(path: ParentPath)) |
1573 | llvm::errs() << "warning: could not create directory '" << ParentPath |
1574 | << "': " << EC.message() << '\n'; |
1575 | } |
1576 | return std::string(NewPath); |
1577 | } |
1578 | |
1579 | namespace { |
1580 | class WriteIndexesThinBackend : public ThinBackendProc { |
1581 | std::string OldPrefix, NewPrefix, NativeObjectPrefix; |
1582 | raw_fd_ostream *LinkedObjectsFile; |
1583 | |
1584 | public: |
1585 | WriteIndexesThinBackend( |
1586 | const Config &Conf, ModuleSummaryIndex &CombinedIndex, |
1587 | const DenseMap<StringRef, GVSummaryMapTy> &ModuleToDefinedGVSummaries, |
1588 | std::string OldPrefix, std::string NewPrefix, |
1589 | std::string NativeObjectPrefix, bool ShouldEmitImportsFiles, |
1590 | raw_fd_ostream *LinkedObjectsFile, lto::IndexWriteCallback OnWrite) |
1591 | : ThinBackendProc(Conf, CombinedIndex, ModuleToDefinedGVSummaries, |
1592 | OnWrite, ShouldEmitImportsFiles), |
1593 | OldPrefix(OldPrefix), NewPrefix(NewPrefix), |
1594 | NativeObjectPrefix(NativeObjectPrefix), |
1595 | LinkedObjectsFile(LinkedObjectsFile) {} |
1596 | |
1597 | Error start( |
1598 | unsigned Task, BitcodeModule BM, |
1599 | const FunctionImporter::ImportMapTy &ImportList, |
1600 | const FunctionImporter::ExportSetTy &ExportList, |
1601 | const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR, |
1602 | MapVector<StringRef, BitcodeModule> &ModuleMap) override { |
1603 | StringRef ModulePath = BM.getModuleIdentifier(); |
1604 | std::string NewModulePath = |
1605 | getThinLTOOutputFile(Path: ModulePath, OldPrefix, NewPrefix); |
1606 | |
1607 | if (LinkedObjectsFile) { |
1608 | std::string ObjectPrefix = |
1609 | NativeObjectPrefix.empty() ? NewPrefix : NativeObjectPrefix; |
1610 | std::string LinkedObjectsFilePath = |
1611 | getThinLTOOutputFile(Path: ModulePath, OldPrefix, NewPrefix: ObjectPrefix); |
1612 | *LinkedObjectsFile << LinkedObjectsFilePath << '\n'; |
1613 | } |
1614 | |
1615 | if (auto E = emitFiles(ImportList, ModulePath, NewModulePath)) |
1616 | return E; |
1617 | |
1618 | if (OnWrite) |
1619 | OnWrite(std::string(ModulePath)); |
1620 | return Error::success(); |
1621 | } |
1622 | |
1623 | Error wait() override { return Error::success(); } |
1624 | |
1625 | // WriteIndexesThinBackend should always return 1 to prevent module |
1626 | // re-ordering and avoid non-determinism in the final link. |
1627 | unsigned getThreadCount() override { return 1; } |
1628 | }; |
1629 | } // end anonymous namespace |
1630 | |
1631 | ThinBackend lto::createWriteIndexesThinBackend( |
1632 | std::string OldPrefix, std::string NewPrefix, |
1633 | std::string NativeObjectPrefix, bool ShouldEmitImportsFiles, |
1634 | raw_fd_ostream *LinkedObjectsFile, IndexWriteCallback OnWrite) { |
1635 | return |
1636 | [=](const Config &Conf, ModuleSummaryIndex &CombinedIndex, |
1637 | const DenseMap<StringRef, GVSummaryMapTy> &ModuleToDefinedGVSummaries, |
1638 | AddStreamFn AddStream, FileCache Cache) { |
1639 | return std::make_unique<WriteIndexesThinBackend>( |
1640 | args: Conf, args&: CombinedIndex, args: ModuleToDefinedGVSummaries, args: OldPrefix, |
1641 | args: NewPrefix, args: NativeObjectPrefix, args: ShouldEmitImportsFiles, |
1642 | args: LinkedObjectsFile, args: OnWrite); |
1643 | }; |
1644 | } |
1645 | |
1646 | Error LTO::runThinLTO(AddStreamFn AddStream, FileCache Cache, |
1647 | const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols) { |
1648 | LLVM_DEBUG(dbgs() << "Running ThinLTO\n" ); |
1649 | ThinLTO.CombinedIndex.releaseTemporaryMemory(); |
1650 | timeTraceProfilerBegin(Name: "ThinLink" , Detail: StringRef("" )); |
1651 | auto TimeTraceScopeExit = llvm::make_scope_exit(F: []() { |
1652 | if (llvm::timeTraceProfilerEnabled()) |
1653 | llvm::timeTraceProfilerEnd(); |
1654 | }); |
1655 | if (ThinLTO.ModuleMap.empty()) |
1656 | return Error::success(); |
1657 | |
1658 | if (ThinLTO.ModulesToCompile && ThinLTO.ModulesToCompile->empty()) { |
1659 | llvm::errs() << "warning: [ThinLTO] No module compiled\n" ; |
1660 | return Error::success(); |
1661 | } |
1662 | |
1663 | if (Conf.CombinedIndexHook && |
1664 | !Conf.CombinedIndexHook(ThinLTO.CombinedIndex, GUIDPreservedSymbols)) |
1665 | return Error::success(); |
1666 | |
1667 | // Collect for each module the list of function it defines (GUID -> |
1668 | // Summary). |
1669 | DenseMap<StringRef, GVSummaryMapTy> ModuleToDefinedGVSummaries( |
1670 | ThinLTO.ModuleMap.size()); |
1671 | ThinLTO.CombinedIndex.collectDefinedGVSummariesPerModule( |
1672 | ModuleToDefinedGVSummaries); |
1673 | // Create entries for any modules that didn't have any GV summaries |
1674 | // (either they didn't have any GVs to start with, or we suppressed |
1675 | // generation of the summaries because they e.g. had inline assembly |
1676 | // uses that couldn't be promoted/renamed on export). This is so |
1677 | // InProcessThinBackend::start can still launch a backend thread, which |
1678 | // is passed the map of summaries for the module, without any special |
1679 | // handling for this case. |
1680 | for (auto &Mod : ThinLTO.ModuleMap) |
1681 | if (!ModuleToDefinedGVSummaries.count(Val: Mod.first)) |
1682 | ModuleToDefinedGVSummaries.try_emplace(Key: Mod.first); |
1683 | |
1684 | // Synthesize entry counts for functions in the CombinedIndex. |
1685 | computeSyntheticCounts(Index&: ThinLTO.CombinedIndex); |
1686 | |
1687 | DenseMap<StringRef, FunctionImporter::ImportMapTy> ImportLists( |
1688 | ThinLTO.ModuleMap.size()); |
1689 | DenseMap<StringRef, FunctionImporter::ExportSetTy> ExportLists( |
1690 | ThinLTO.ModuleMap.size()); |
1691 | StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR; |
1692 | |
1693 | if (DumpThinCGSCCs) |
1694 | ThinLTO.CombinedIndex.dumpSCCs(OS&: outs()); |
1695 | |
1696 | std::set<GlobalValue::GUID> ExportedGUIDs; |
1697 | |
1698 | bool WholeProgramVisibilityEnabledInLTO = |
1699 | Conf.HasWholeProgramVisibility && |
1700 | // If validation is enabled, upgrade visibility only when all vtables |
1701 | // have typeinfos. |
1702 | (!Conf.ValidateAllVtablesHaveTypeInfos || Conf.AllVtablesHaveTypeInfos); |
1703 | if (hasWholeProgramVisibility(WholeProgramVisibilityEnabledInLTO)) |
1704 | ThinLTO.CombinedIndex.setWithWholeProgramVisibility(); |
1705 | |
1706 | // If we're validating, get the vtable symbols that should not be |
1707 | // upgraded because they correspond to typeIDs outside of index-based |
1708 | // WPD info. |
1709 | DenseSet<GlobalValue::GUID> VisibleToRegularObjSymbols; |
1710 | if (WholeProgramVisibilityEnabledInLTO && |
1711 | Conf.ValidateAllVtablesHaveTypeInfos) { |
1712 | // This returns true when the name is local or not defined. Locals are |
1713 | // expected to be handled separately. |
1714 | auto IsVisibleToRegularObj = [&](StringRef name) { |
1715 | auto It = GlobalResolutions->find(Key: name); |
1716 | return (It == GlobalResolutions->end() || |
1717 | It->second.VisibleOutsideSummary); |
1718 | }; |
1719 | |
1720 | getVisibleToRegularObjVtableGUIDs(Index&: ThinLTO.CombinedIndex, |
1721 | VisibleToRegularObjSymbols, |
1722 | IsVisibleToRegularObj); |
1723 | } |
1724 | |
1725 | // If allowed, upgrade public vcall visibility to linkage unit visibility in |
1726 | // the summaries before whole program devirtualization below. |
1727 | updateVCallVisibilityInIndex( |
1728 | Index&: ThinLTO.CombinedIndex, WholeProgramVisibilityEnabledInLTO, |
1729 | DynamicExportSymbols, VisibleToRegularObjSymbols); |
1730 | |
1731 | // Perform index-based WPD. This will return immediately if there are |
1732 | // no index entries in the typeIdMetadata map (e.g. if we are instead |
1733 | // performing IR-based WPD in hybrid regular/thin LTO mode). |
1734 | std::map<ValueInfo, std::vector<VTableSlotSummary>> LocalWPDTargetsMap; |
1735 | runWholeProgramDevirtOnIndex(Summary&: ThinLTO.CombinedIndex, ExportedGUIDs, |
1736 | LocalWPDTargetsMap); |
1737 | |
1738 | auto isPrevailing = [&](GlobalValue::GUID GUID, const GlobalValueSummary *S) { |
1739 | return ThinLTO.PrevailingModuleForGUID[GUID] == S->modulePath(); |
1740 | }; |
1741 | if (EnableMemProfContextDisambiguation) { |
1742 | MemProfContextDisambiguation ContextDisambiguation; |
1743 | ContextDisambiguation.run(Index&: ThinLTO.CombinedIndex, isPrevailing); |
1744 | } |
1745 | |
1746 | // Figure out which symbols need to be internalized. This also needs to happen |
1747 | // at -O0 because summary-based DCE is implemented using internalization, and |
1748 | // we must apply DCE consistently with the full LTO module in order to avoid |
1749 | // undefined references during the final link. |
1750 | for (auto &Res : *GlobalResolutions) { |
1751 | // If the symbol does not have external references or it is not prevailing, |
1752 | // then not need to mark it as exported from a ThinLTO partition. |
1753 | if (Res.second.Partition != GlobalResolution::External || |
1754 | !Res.second.isPrevailingIRSymbol()) |
1755 | continue; |
1756 | auto GUID = GlobalValue::getGUID( |
1757 | GlobalName: GlobalValue::dropLLVMManglingEscape(Name: Res.second.IRName)); |
1758 | // Mark exported unless index-based analysis determined it to be dead. |
1759 | if (ThinLTO.CombinedIndex.isGUIDLive(GUID)) |
1760 | ExportedGUIDs.insert(x: GUID); |
1761 | } |
1762 | |
1763 | // Reset the GlobalResolutions to deallocate the associated memory, as there |
1764 | // are no further accesses. We specifically want to do this before computing |
1765 | // cross module importing, which adds to peak memory via the computed import |
1766 | // and export lists. |
1767 | GlobalResolutions.reset(); |
1768 | |
1769 | if (Conf.OptLevel > 0) |
1770 | ComputeCrossModuleImport(Index: ThinLTO.CombinedIndex, ModuleToDefinedGVSummaries, |
1771 | isPrevailing, ImportLists, ExportLists); |
1772 | |
1773 | // Any functions referenced by the jump table in the regular LTO object must |
1774 | // be exported. |
1775 | for (auto &Def : ThinLTO.CombinedIndex.cfiFunctionDefs()) |
1776 | ExportedGUIDs.insert( |
1777 | x: GlobalValue::getGUID(GlobalName: GlobalValue::dropLLVMManglingEscape(Name: Def))); |
1778 | for (auto &Decl : ThinLTO.CombinedIndex.cfiFunctionDecls()) |
1779 | ExportedGUIDs.insert( |
1780 | x: GlobalValue::getGUID(GlobalName: GlobalValue::dropLLVMManglingEscape(Name: Decl))); |
1781 | |
1782 | auto isExported = [&](StringRef ModuleIdentifier, ValueInfo VI) { |
1783 | const auto &ExportList = ExportLists.find(Val: ModuleIdentifier); |
1784 | return (ExportList != ExportLists.end() && ExportList->second.count(V: VI)) || |
1785 | ExportedGUIDs.count(x: VI.getGUID()); |
1786 | }; |
1787 | |
1788 | // Update local devirtualized targets that were exported by cross-module |
1789 | // importing or by other devirtualizations marked in the ExportedGUIDs set. |
1790 | updateIndexWPDForExports(Summary&: ThinLTO.CombinedIndex, isExported, |
1791 | LocalWPDTargetsMap); |
1792 | |
1793 | thinLTOInternalizeAndPromoteInIndex(Index&: ThinLTO.CombinedIndex, isExported, |
1794 | isPrevailing); |
1795 | |
1796 | auto recordNewLinkage = [&](StringRef ModuleIdentifier, |
1797 | GlobalValue::GUID GUID, |
1798 | GlobalValue::LinkageTypes NewLinkage) { |
1799 | ResolvedODR[ModuleIdentifier][GUID] = NewLinkage; |
1800 | }; |
1801 | thinLTOResolvePrevailingInIndex(C: Conf, Index&: ThinLTO.CombinedIndex, isPrevailing, |
1802 | recordNewLinkage, GUIDPreservedSymbols); |
1803 | |
1804 | thinLTOPropagateFunctionAttrs(Index&: ThinLTO.CombinedIndex, isPrevailing); |
1805 | |
1806 | generateParamAccessSummary(Index&: ThinLTO.CombinedIndex); |
1807 | |
1808 | if (llvm::timeTraceProfilerEnabled()) |
1809 | llvm::timeTraceProfilerEnd(); |
1810 | |
1811 | TimeTraceScopeExit.release(); |
1812 | |
1813 | std::unique_ptr<ThinBackendProc> BackendProc = |
1814 | ThinLTO.Backend(Conf, ThinLTO.CombinedIndex, ModuleToDefinedGVSummaries, |
1815 | AddStream, Cache); |
1816 | |
1817 | auto &ModuleMap = |
1818 | ThinLTO.ModulesToCompile ? *ThinLTO.ModulesToCompile : ThinLTO.ModuleMap; |
1819 | |
1820 | auto ProcessOneModule = [&](int I) -> Error { |
1821 | auto &Mod = *(ModuleMap.begin() + I); |
1822 | // Tasks 0 through ParallelCodeGenParallelismLevel-1 are reserved for |
1823 | // combined module and parallel code generation partitions. |
1824 | return BackendProc->start(Task: RegularLTO.ParallelCodeGenParallelismLevel + I, |
1825 | BM: Mod.second, ImportList: ImportLists[Mod.first], |
1826 | ExportList: ExportLists[Mod.first], ResolvedODR: ResolvedODR[Mod.first], |
1827 | ModuleMap&: ThinLTO.ModuleMap); |
1828 | }; |
1829 | |
1830 | if (BackendProc->getThreadCount() == 1) { |
1831 | // Process the modules in the order they were provided on the command-line. |
1832 | // It is important for this codepath to be used for WriteIndexesThinBackend, |
1833 | // to ensure the emitted LinkedObjectsFile lists ThinLTO objects in the same |
1834 | // order as the inputs, which otherwise would affect the final link order. |
1835 | for (int I = 0, E = ModuleMap.size(); I != E; ++I) |
1836 | if (Error E = ProcessOneModule(I)) |
1837 | return E; |
1838 | } else { |
1839 | // When executing in parallel, process largest bitsize modules first to |
1840 | // improve parallelism, and avoid starving the thread pool near the end. |
1841 | // This saves about 15 sec on a 36-core machine while link `clang.exe` (out |
1842 | // of 100 sec). |
1843 | std::vector<BitcodeModule *> ModulesVec; |
1844 | ModulesVec.reserve(n: ModuleMap.size()); |
1845 | for (auto &Mod : ModuleMap) |
1846 | ModulesVec.push_back(x: &Mod.second); |
1847 | for (int I : generateModulesOrdering(R: ModulesVec)) |
1848 | if (Error E = ProcessOneModule(I)) |
1849 | return E; |
1850 | } |
1851 | return BackendProc->wait(); |
1852 | } |
1853 | |
1854 | Expected<std::unique_ptr<ToolOutputFile>> lto::( |
1855 | LLVMContext &Context, StringRef , StringRef , |
1856 | StringRef , bool , |
1857 | std::optional<uint64_t> , int Count) { |
1858 | std::string Filename = std::string(RemarksFilename); |
1859 | // For ThinLTO, file.opt.<format> becomes |
1860 | // file.opt.<format>.thin.<num>.<format>. |
1861 | if (!Filename.empty() && Count != -1) |
1862 | Filename = |
1863 | (Twine(Filename) + ".thin." + llvm::utostr(X: Count) + "." + RemarksFormat) |
1864 | .str(); |
1865 | |
1866 | auto ResultOrErr = llvm::setupLLVMOptimizationRemarks( |
1867 | Context, RemarksFilename: Filename, RemarksPasses, RemarksFormat, RemarksWithHotness, |
1868 | RemarksHotnessThreshold); |
1869 | if (Error E = ResultOrErr.takeError()) |
1870 | return std::move(E); |
1871 | |
1872 | if (*ResultOrErr) |
1873 | (*ResultOrErr)->keep(); |
1874 | |
1875 | return ResultOrErr; |
1876 | } |
1877 | |
1878 | Expected<std::unique_ptr<ToolOutputFile>> |
1879 | lto::setupStatsFile(StringRef StatsFilename) { |
1880 | // Setup output file to emit statistics. |
1881 | if (StatsFilename.empty()) |
1882 | return nullptr; |
1883 | |
1884 | llvm::EnableStatistics(DoPrintOnExit: false); |
1885 | std::error_code EC; |
1886 | auto StatsFile = |
1887 | std::make_unique<ToolOutputFile>(args&: StatsFilename, args&: EC, args: sys::fs::OF_None); |
1888 | if (EC) |
1889 | return errorCodeToError(EC); |
1890 | |
1891 | StatsFile->keep(); |
1892 | return std::move(StatsFile); |
1893 | } |
1894 | |
1895 | // Compute the ordering we will process the inputs: the rough heuristic here |
1896 | // is to sort them per size so that the largest module get schedule as soon as |
1897 | // possible. This is purely a compile-time optimization. |
1898 | std::vector<int> lto::generateModulesOrdering(ArrayRef<BitcodeModule *> R) { |
1899 | auto Seq = llvm::seq<int>(Begin: 0, End: R.size()); |
1900 | std::vector<int> ModulesOrdering(Seq.begin(), Seq.end()); |
1901 | llvm::sort(C&: ModulesOrdering, Comp: [&](int LeftIndex, int RightIndex) { |
1902 | auto LSize = R[LeftIndex]->getBuffer().size(); |
1903 | auto RSize = R[RightIndex]->getBuffer().size(); |
1904 | return LSize > RSize; |
1905 | }); |
1906 | return ModulesOrdering; |
1907 | } |
1908 | |