1 | //===-ThinLTOCodeGenerator.cpp - LLVM Link Time Optimizer -----------------===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | // |
9 | // This file implements the Thin Link Time Optimization library. This library is |
10 | // intended to be used by linker to optimize code at link time. |
11 | // |
12 | //===----------------------------------------------------------------------===// |
13 | |
14 | #include "llvm/LTO/legacy/ThinLTOCodeGenerator.h" |
15 | #include "llvm/Support/CommandLine.h" |
16 | |
17 | #include "llvm/ADT/ScopeExit.h" |
18 | #include "llvm/ADT/Statistic.h" |
19 | #include "llvm/ADT/StringExtras.h" |
20 | #include "llvm/Analysis/AliasAnalysis.h" |
21 | #include "llvm/Analysis/ModuleSummaryAnalysis.h" |
22 | #include "llvm/Analysis/ProfileSummaryInfo.h" |
23 | #include "llvm/Analysis/TargetLibraryInfo.h" |
24 | #include "llvm/Bitcode/BitcodeReader.h" |
25 | #include "llvm/Bitcode/BitcodeWriter.h" |
26 | #include "llvm/Bitcode/BitcodeWriterPass.h" |
27 | #include "llvm/Config/llvm-config.h" |
28 | #include "llvm/IR/DebugInfo.h" |
29 | #include "llvm/IR/DiagnosticPrinter.h" |
30 | #include "llvm/IR/LegacyPassManager.h" |
31 | #include "llvm/IR/LLVMContext.h" |
32 | #include "llvm/IR/LLVMRemarkStreamer.h" |
33 | #include "llvm/IR/Mangler.h" |
34 | #include "llvm/IR/PassTimingInfo.h" |
35 | #include "llvm/IR/Verifier.h" |
36 | #include "llvm/IRReader/IRReader.h" |
37 | #include "llvm/LTO/LTO.h" |
38 | #include "llvm/LTO/SummaryBasedOptimizations.h" |
39 | #include "llvm/MC/TargetRegistry.h" |
40 | #include "llvm/Object/IRObjectFile.h" |
41 | #include "llvm/Passes/PassBuilder.h" |
42 | #include "llvm/Passes/StandardInstrumentations.h" |
43 | #include "llvm/Remarks/HotnessThresholdParser.h" |
44 | #include "llvm/Support/CachePruning.h" |
45 | #include "llvm/Support/Debug.h" |
46 | #include "llvm/Support/Error.h" |
47 | #include "llvm/Support/FileSystem.h" |
48 | #include "llvm/Support/Path.h" |
49 | #include "llvm/Support/SHA1.h" |
50 | #include "llvm/Support/SmallVectorMemoryBuffer.h" |
51 | #include "llvm/Support/ThreadPool.h" |
52 | #include "llvm/Support/Threading.h" |
53 | #include "llvm/Support/ToolOutputFile.h" |
54 | #include "llvm/Support/raw_ostream.h" |
55 | #include "llvm/Target/TargetMachine.h" |
56 | #include "llvm/TargetParser/SubtargetFeature.h" |
57 | #include "llvm/Transforms/IPO/FunctionAttrs.h" |
58 | #include "llvm/Transforms/IPO/FunctionImport.h" |
59 | #include "llvm/Transforms/IPO/Internalize.h" |
60 | #include "llvm/Transforms/IPO/WholeProgramDevirt.h" |
61 | #include "llvm/Transforms/ObjCARC.h" |
62 | #include "llvm/Transforms/Utils/FunctionImportUtils.h" |
63 | |
64 | #include <numeric> |
65 | |
66 | #if !defined(_MSC_VER) && !defined(__MINGW32__) |
67 | #include <unistd.h> |
68 | #else |
69 | #include <io.h> |
70 | #endif |
71 | |
72 | using namespace llvm; |
73 | |
74 | #define DEBUG_TYPE "thinlto" |
75 | |
76 | namespace llvm { |
77 | // Flags -discard-value-names, defined in LTOCodeGenerator.cpp |
78 | extern cl::opt<bool> LTODiscardValueNames; |
79 | extern cl::opt<std::string> ; |
80 | extern cl::opt<std::string> ; |
81 | extern cl::opt<bool> ; |
82 | extern cl::opt<std::optional<uint64_t>, false, remarks::HotnessThresholdParser> |
83 | ; |
84 | extern cl::opt<std::string> ; |
85 | } |
86 | |
87 | namespace { |
88 | |
89 | // Default to using all available threads in the system, but using only one |
90 | // thred per core, as indicated by the usage of |
91 | // heavyweight_hardware_concurrency() below. |
92 | static cl::opt<int> ThreadCount("threads" , cl::init(Val: 0)); |
93 | |
94 | // Simple helper to save temporary files for debug. |
95 | static void saveTempBitcode(const Module &TheModule, StringRef TempDir, |
96 | unsigned count, StringRef Suffix) { |
97 | if (TempDir.empty()) |
98 | return; |
99 | // User asked to save temps, let dump the bitcode file after import. |
100 | std::string SaveTempPath = (TempDir + llvm::Twine(count) + Suffix).str(); |
101 | std::error_code EC; |
102 | raw_fd_ostream OS(SaveTempPath, EC, sys::fs::OF_None); |
103 | if (EC) |
104 | report_fatal_error(reason: Twine("Failed to open " ) + SaveTempPath + |
105 | " to save optimized bitcode\n" ); |
106 | WriteBitcodeToFile(M: TheModule, Out&: OS, /* ShouldPreserveUseListOrder */ true); |
107 | } |
108 | |
109 | static const GlobalValueSummary * |
110 | getFirstDefinitionForLinker(const GlobalValueSummaryList &GVSummaryList) { |
111 | // If there is any strong definition anywhere, get it. |
112 | auto StrongDefForLinker = llvm::find_if( |
113 | Range: GVSummaryList, P: [](const std::unique_ptr<GlobalValueSummary> &Summary) { |
114 | auto Linkage = Summary->linkage(); |
115 | return !GlobalValue::isAvailableExternallyLinkage(Linkage) && |
116 | !GlobalValue::isWeakForLinker(Linkage); |
117 | }); |
118 | if (StrongDefForLinker != GVSummaryList.end()) |
119 | return StrongDefForLinker->get(); |
120 | // Get the first *linker visible* definition for this global in the summary |
121 | // list. |
122 | auto FirstDefForLinker = llvm::find_if( |
123 | Range: GVSummaryList, P: [](const std::unique_ptr<GlobalValueSummary> &Summary) { |
124 | auto Linkage = Summary->linkage(); |
125 | return !GlobalValue::isAvailableExternallyLinkage(Linkage); |
126 | }); |
127 | // Extern templates can be emitted as available_externally. |
128 | if (FirstDefForLinker == GVSummaryList.end()) |
129 | return nullptr; |
130 | return FirstDefForLinker->get(); |
131 | } |
132 | |
133 | // Populate map of GUID to the prevailing copy for any multiply defined |
134 | // symbols. Currently assume first copy is prevailing, or any strong |
135 | // definition. Can be refined with Linker information in the future. |
136 | static void computePrevailingCopies( |
137 | const ModuleSummaryIndex &Index, |
138 | DenseMap<GlobalValue::GUID, const GlobalValueSummary *> &PrevailingCopy) { |
139 | auto HasMultipleCopies = [&](const GlobalValueSummaryList &GVSummaryList) { |
140 | return GVSummaryList.size() > 1; |
141 | }; |
142 | |
143 | for (auto &I : Index) { |
144 | if (HasMultipleCopies(I.second.SummaryList)) |
145 | PrevailingCopy[I.first] = |
146 | getFirstDefinitionForLinker(GVSummaryList: I.second.SummaryList); |
147 | } |
148 | } |
149 | |
150 | static StringMap<lto::InputFile *> |
151 | generateModuleMap(std::vector<std::unique_ptr<lto::InputFile>> &Modules) { |
152 | StringMap<lto::InputFile *> ModuleMap; |
153 | for (auto &M : Modules) { |
154 | LLVM_DEBUG(dbgs() << "Adding module " << M->getName() << " to ModuleMap\n" ); |
155 | assert(!ModuleMap.contains(M->getName()) && |
156 | "Expect unique Buffer Identifier" ); |
157 | ModuleMap[M->getName()] = M.get(); |
158 | } |
159 | return ModuleMap; |
160 | } |
161 | |
162 | static void promoteModule(Module &TheModule, const ModuleSummaryIndex &Index, |
163 | bool ClearDSOLocalOnDeclarations) { |
164 | if (renameModuleForThinLTO(M&: TheModule, Index, ClearDSOLocalOnDeclarations)) |
165 | report_fatal_error(reason: "renameModuleForThinLTO failed" ); |
166 | } |
167 | |
168 | namespace { |
169 | class ThinLTODiagnosticInfo : public DiagnosticInfo { |
170 | const Twine &Msg; |
171 | public: |
172 | ThinLTODiagnosticInfo(const Twine &DiagMsg, |
173 | DiagnosticSeverity Severity = DS_Error) |
174 | : DiagnosticInfo(DK_Linker, Severity), Msg(DiagMsg) {} |
175 | void print(DiagnosticPrinter &DP) const override { DP << Msg; } |
176 | }; |
177 | } |
178 | |
179 | /// Verify the module and strip broken debug info. |
180 | static void verifyLoadedModule(Module &TheModule) { |
181 | bool BrokenDebugInfo = false; |
182 | if (verifyModule(M: TheModule, OS: &dbgs(), BrokenDebugInfo: &BrokenDebugInfo)) |
183 | report_fatal_error(reason: "Broken module found, compilation aborted!" ); |
184 | if (BrokenDebugInfo) { |
185 | TheModule.getContext().diagnose(DI: ThinLTODiagnosticInfo( |
186 | "Invalid debug info found, debug info will be stripped" , DS_Warning)); |
187 | StripDebugInfo(M&: TheModule); |
188 | } |
189 | } |
190 | |
191 | static std::unique_ptr<Module> loadModuleFromInput(lto::InputFile *Input, |
192 | LLVMContext &Context, |
193 | bool Lazy, |
194 | bool IsImporting) { |
195 | auto &Mod = Input->getSingleBitcodeModule(); |
196 | SMDiagnostic Err; |
197 | Expected<std::unique_ptr<Module>> ModuleOrErr = |
198 | Lazy ? Mod.getLazyModule(Context, |
199 | /* ShouldLazyLoadMetadata */ true, IsImporting) |
200 | : Mod.parseModule(Context); |
201 | if (!ModuleOrErr) { |
202 | handleAllErrors(E: ModuleOrErr.takeError(), Handlers: [&](ErrorInfoBase &EIB) { |
203 | SMDiagnostic Err = SMDiagnostic(Mod.getModuleIdentifier(), |
204 | SourceMgr::DK_Error, EIB.message()); |
205 | Err.print(ProgName: "ThinLTO" , S&: errs()); |
206 | }); |
207 | report_fatal_error(reason: "Can't load module, abort." ); |
208 | } |
209 | if (!Lazy) |
210 | verifyLoadedModule(TheModule&: *ModuleOrErr.get()); |
211 | return std::move(*ModuleOrErr); |
212 | } |
213 | |
214 | static void |
215 | crossImportIntoModule(Module &TheModule, const ModuleSummaryIndex &Index, |
216 | StringMap<lto::InputFile *> &ModuleMap, |
217 | const FunctionImporter::ImportMapTy &ImportList, |
218 | bool ClearDSOLocalOnDeclarations) { |
219 | auto Loader = [&](StringRef Identifier) { |
220 | auto &Input = ModuleMap[Identifier]; |
221 | return loadModuleFromInput(Input, Context&: TheModule.getContext(), |
222 | /*Lazy=*/true, /*IsImporting*/ true); |
223 | }; |
224 | |
225 | FunctionImporter Importer(Index, Loader, ClearDSOLocalOnDeclarations); |
226 | Expected<bool> Result = Importer.importFunctions(M&: TheModule, ImportList); |
227 | if (!Result) { |
228 | handleAllErrors(E: Result.takeError(), Handlers: [&](ErrorInfoBase &EIB) { |
229 | SMDiagnostic Err = SMDiagnostic(TheModule.getModuleIdentifier(), |
230 | SourceMgr::DK_Error, EIB.message()); |
231 | Err.print(ProgName: "ThinLTO" , S&: errs()); |
232 | }); |
233 | report_fatal_error(reason: "importFunctions failed" ); |
234 | } |
235 | // Verify again after cross-importing. |
236 | verifyLoadedModule(TheModule); |
237 | } |
238 | |
239 | static void optimizeModule(Module &TheModule, TargetMachine &TM, |
240 | unsigned OptLevel, bool Freestanding, |
241 | bool DebugPassManager, ModuleSummaryIndex *Index) { |
242 | std::optional<PGOOptions> PGOOpt; |
243 | LoopAnalysisManager LAM; |
244 | FunctionAnalysisManager FAM; |
245 | CGSCCAnalysisManager CGAM; |
246 | ModuleAnalysisManager MAM; |
247 | |
248 | PassInstrumentationCallbacks PIC; |
249 | StandardInstrumentations SI(TheModule.getContext(), DebugPassManager); |
250 | SI.registerCallbacks(PIC, MAM: &MAM); |
251 | PipelineTuningOptions PTO; |
252 | PTO.LoopVectorization = true; |
253 | PTO.SLPVectorization = true; |
254 | PassBuilder PB(&TM, PTO, PGOOpt, &PIC); |
255 | |
256 | std::unique_ptr<TargetLibraryInfoImpl> TLII( |
257 | new TargetLibraryInfoImpl(Triple(TM.getTargetTriple()))); |
258 | if (Freestanding) |
259 | TLII->disableAllFunctions(); |
260 | FAM.registerPass(PassBuilder: [&] { return TargetLibraryAnalysis(*TLII); }); |
261 | |
262 | // Register all the basic analyses with the managers. |
263 | PB.registerModuleAnalyses(MAM); |
264 | PB.registerCGSCCAnalyses(CGAM); |
265 | PB.registerFunctionAnalyses(FAM); |
266 | PB.registerLoopAnalyses(LAM); |
267 | PB.crossRegisterProxies(LAM, FAM, CGAM, MAM); |
268 | |
269 | ModulePassManager MPM; |
270 | |
271 | OptimizationLevel OL; |
272 | |
273 | switch (OptLevel) { |
274 | default: |
275 | llvm_unreachable("Invalid optimization level" ); |
276 | case 0: |
277 | OL = OptimizationLevel::O0; |
278 | break; |
279 | case 1: |
280 | OL = OptimizationLevel::O1; |
281 | break; |
282 | case 2: |
283 | OL = OptimizationLevel::O2; |
284 | break; |
285 | case 3: |
286 | OL = OptimizationLevel::O3; |
287 | break; |
288 | } |
289 | |
290 | MPM.addPass(Pass: PB.buildThinLTODefaultPipeline(Level: OL, ImportSummary: Index)); |
291 | |
292 | MPM.run(IR&: TheModule, AM&: MAM); |
293 | } |
294 | |
295 | static void |
296 | addUsedSymbolToPreservedGUID(const lto::InputFile &File, |
297 | DenseSet<GlobalValue::GUID> &PreservedGUID) { |
298 | for (const auto &Sym : File.symbols()) { |
299 | if (Sym.isUsed()) |
300 | PreservedGUID.insert(V: GlobalValue::getGUID(GlobalName: Sym.getIRName())); |
301 | } |
302 | } |
303 | |
304 | // Convert the PreservedSymbols map from "Name" based to "GUID" based. |
305 | static void computeGUIDPreservedSymbols(const lto::InputFile &File, |
306 | const StringSet<> &PreservedSymbols, |
307 | const Triple &TheTriple, |
308 | DenseSet<GlobalValue::GUID> &GUIDs) { |
309 | // Iterate the symbols in the input file and if the input has preserved symbol |
310 | // compute the GUID for the symbol. |
311 | for (const auto &Sym : File.symbols()) { |
312 | if (PreservedSymbols.count(Key: Sym.getName()) && !Sym.getIRName().empty()) |
313 | GUIDs.insert(V: GlobalValue::getGUID(GlobalName: GlobalValue::getGlobalIdentifier( |
314 | Name: Sym.getIRName(), Linkage: GlobalValue::ExternalLinkage, FileName: "" ))); |
315 | } |
316 | } |
317 | |
318 | static DenseSet<GlobalValue::GUID> |
319 | computeGUIDPreservedSymbols(const lto::InputFile &File, |
320 | const StringSet<> &PreservedSymbols, |
321 | const Triple &TheTriple) { |
322 | DenseSet<GlobalValue::GUID> GUIDPreservedSymbols(PreservedSymbols.size()); |
323 | computeGUIDPreservedSymbols(File, PreservedSymbols, TheTriple, |
324 | GUIDs&: GUIDPreservedSymbols); |
325 | return GUIDPreservedSymbols; |
326 | } |
327 | |
328 | std::unique_ptr<MemoryBuffer> codegenModule(Module &TheModule, |
329 | TargetMachine &TM) { |
330 | SmallVector<char, 128> OutputBuffer; |
331 | |
332 | // CodeGen |
333 | { |
334 | raw_svector_ostream OS(OutputBuffer); |
335 | legacy::PassManager PM; |
336 | |
337 | // If the bitcode files contain ARC code and were compiled with optimization, |
338 | // the ObjCARCContractPass must be run, so do it unconditionally here. |
339 | PM.add(P: createObjCARCContractPass()); |
340 | |
341 | // Setup the codegen now. |
342 | if (TM.addPassesToEmitFile(PM, OS, nullptr, CodeGenFileType::ObjectFile, |
343 | /* DisableVerify */ true)) |
344 | report_fatal_error(reason: "Failed to setup codegen" ); |
345 | |
346 | // Run codegen now. resulting binary is in OutputBuffer. |
347 | PM.run(M&: TheModule); |
348 | } |
349 | return std::make_unique<SmallVectorMemoryBuffer>( |
350 | args: std::move(OutputBuffer), /*RequiresNullTerminator=*/args: false); |
351 | } |
352 | |
353 | /// Manage caching for a single Module. |
354 | class ModuleCacheEntry { |
355 | SmallString<128> EntryPath; |
356 | |
357 | public: |
358 | // Create a cache entry. This compute a unique hash for the Module considering |
359 | // the current list of export/import, and offer an interface to query to |
360 | // access the content in the cache. |
361 | ModuleCacheEntry( |
362 | StringRef CachePath, const ModuleSummaryIndex &Index, StringRef ModuleID, |
363 | const FunctionImporter::ImportMapTy &ImportList, |
364 | const FunctionImporter::ExportSetTy &ExportList, |
365 | const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR, |
366 | const GVSummaryMapTy &DefinedGVSummaries, unsigned OptLevel, |
367 | bool Freestanding, const TargetMachineBuilder &TMBuilder) { |
368 | if (CachePath.empty()) |
369 | return; |
370 | |
371 | if (!Index.modulePaths().count(Key: ModuleID)) |
372 | // The module does not have an entry, it can't have a hash at all |
373 | return; |
374 | |
375 | if (all_of(Range: Index.getModuleHash(ModPath: ModuleID), |
376 | P: [](uint32_t V) { return V == 0; })) |
377 | // No hash entry, no caching! |
378 | return; |
379 | |
380 | llvm::lto::Config Conf; |
381 | Conf.OptLevel = OptLevel; |
382 | Conf.Options = TMBuilder.Options; |
383 | Conf.CPU = TMBuilder.MCpu; |
384 | Conf.MAttrs.push_back(x: TMBuilder.MAttr); |
385 | Conf.RelocModel = TMBuilder.RelocModel; |
386 | Conf.CGOptLevel = TMBuilder.CGOptLevel; |
387 | Conf.Freestanding = Freestanding; |
388 | SmallString<40> Key; |
389 | computeLTOCacheKey(Key, Conf, Index, ModuleID, ImportList, ExportList, |
390 | ResolvedODR, DefinedGlobals: DefinedGVSummaries); |
391 | |
392 | // This choice of file name allows the cache to be pruned (see pruneCache() |
393 | // in include/llvm/Support/CachePruning.h). |
394 | sys::path::append(path&: EntryPath, a: CachePath, b: "llvmcache-" + Key); |
395 | } |
396 | |
397 | // Access the path to this entry in the cache. |
398 | StringRef getEntryPath() { return EntryPath; } |
399 | |
400 | // Try loading the buffer for this cache entry. |
401 | ErrorOr<std::unique_ptr<MemoryBuffer>> tryLoadingBuffer() { |
402 | if (EntryPath.empty()) |
403 | return std::error_code(); |
404 | SmallString<64> ResultPath; |
405 | Expected<sys::fs::file_t> FDOrErr = sys::fs::openNativeFileForRead( |
406 | Name: Twine(EntryPath), Flags: sys::fs::OF_UpdateAtime, RealPath: &ResultPath); |
407 | if (!FDOrErr) |
408 | return errorToErrorCode(Err: FDOrErr.takeError()); |
409 | ErrorOr<std::unique_ptr<MemoryBuffer>> MBOrErr = MemoryBuffer::getOpenFile( |
410 | FD: *FDOrErr, Filename: EntryPath, /*FileSize=*/-1, /*RequiresNullTerminator=*/false); |
411 | sys::fs::closeFile(F&: *FDOrErr); |
412 | return MBOrErr; |
413 | } |
414 | |
415 | // Cache the Produced object file |
416 | void write(const MemoryBuffer &OutputBuffer) { |
417 | if (EntryPath.empty()) |
418 | return; |
419 | |
420 | if (auto Err = llvm::writeToOutput( |
421 | OutputFileName: EntryPath, Write: [&OutputBuffer](llvm::raw_ostream &OS) -> llvm::Error { |
422 | OS << OutputBuffer.getBuffer(); |
423 | return llvm::Error::success(); |
424 | })) |
425 | report_fatal_error(reason: llvm::formatv(Fmt: "ThinLTO: Can't write file {0}: {1}" , |
426 | Vals&: EntryPath, |
427 | Vals: toString(E: std::move(Err)).c_str())); |
428 | } |
429 | }; |
430 | |
431 | static std::unique_ptr<MemoryBuffer> |
432 | ProcessThinLTOModule(Module &TheModule, ModuleSummaryIndex &Index, |
433 | StringMap<lto::InputFile *> &ModuleMap, TargetMachine &TM, |
434 | const FunctionImporter::ImportMapTy &ImportList, |
435 | const FunctionImporter::ExportSetTy &ExportList, |
436 | const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols, |
437 | const GVSummaryMapTy &DefinedGlobals, |
438 | const ThinLTOCodeGenerator::CachingOptions &CacheOptions, |
439 | bool DisableCodeGen, StringRef SaveTempsDir, |
440 | bool Freestanding, unsigned OptLevel, unsigned count, |
441 | bool DebugPassManager) { |
442 | // "Benchmark"-like optimization: single-source case |
443 | bool SingleModule = (ModuleMap.size() == 1); |
444 | |
445 | // When linking an ELF shared object, dso_local should be dropped. We |
446 | // conservatively do this for -fpic. |
447 | bool ClearDSOLocalOnDeclarations = |
448 | TM.getTargetTriple().isOSBinFormatELF() && |
449 | TM.getRelocationModel() != Reloc::Static && |
450 | TheModule.getPIELevel() == PIELevel::Default; |
451 | |
452 | if (!SingleModule) { |
453 | promoteModule(TheModule, Index, ClearDSOLocalOnDeclarations); |
454 | |
455 | // Apply summary-based prevailing-symbol resolution decisions. |
456 | thinLTOFinalizeInModule(TheModule, DefinedGlobals, /*PropagateAttrs=*/true); |
457 | |
458 | // Save temps: after promotion. |
459 | saveTempBitcode(TheModule, TempDir: SaveTempsDir, count, Suffix: ".1.promoted.bc" ); |
460 | } |
461 | |
462 | // Be friendly and don't nuke totally the module when the client didn't |
463 | // supply anything to preserve. |
464 | if (!ExportList.empty() || !GUIDPreservedSymbols.empty()) { |
465 | // Apply summary-based internalization decisions. |
466 | thinLTOInternalizeModule(TheModule, DefinedGlobals); |
467 | } |
468 | |
469 | // Save internalized bitcode |
470 | saveTempBitcode(TheModule, TempDir: SaveTempsDir, count, Suffix: ".2.internalized.bc" ); |
471 | |
472 | if (!SingleModule) |
473 | crossImportIntoModule(TheModule, Index, ModuleMap, ImportList, |
474 | ClearDSOLocalOnDeclarations); |
475 | |
476 | // Do this after any importing so that imported code is updated. |
477 | // See comment at call to updateVCallVisibilityInIndex() for why |
478 | // WholeProgramVisibilityEnabledInLTO is false. |
479 | updatePublicTypeTestCalls(M&: TheModule, |
480 | /* WholeProgramVisibilityEnabledInLTO */ false); |
481 | |
482 | // Save temps: after cross-module import. |
483 | saveTempBitcode(TheModule, TempDir: SaveTempsDir, count, Suffix: ".3.imported.bc" ); |
484 | |
485 | optimizeModule(TheModule, TM, OptLevel, Freestanding, DebugPassManager, |
486 | Index: &Index); |
487 | |
488 | saveTempBitcode(TheModule, TempDir: SaveTempsDir, count, Suffix: ".4.opt.bc" ); |
489 | |
490 | if (DisableCodeGen) { |
491 | // Configured to stop before CodeGen, serialize the bitcode and return. |
492 | SmallVector<char, 128> OutputBuffer; |
493 | { |
494 | raw_svector_ostream OS(OutputBuffer); |
495 | ProfileSummaryInfo PSI(TheModule); |
496 | auto Index = buildModuleSummaryIndex(M: TheModule, GetBFICallback: nullptr, PSI: &PSI); |
497 | WriteBitcodeToFile(M: TheModule, Out&: OS, ShouldPreserveUseListOrder: true, Index: &Index); |
498 | } |
499 | return std::make_unique<SmallVectorMemoryBuffer>( |
500 | args: std::move(OutputBuffer), /*RequiresNullTerminator=*/args: false); |
501 | } |
502 | |
503 | return codegenModule(TheModule, TM); |
504 | } |
505 | |
506 | /// Resolve prevailing symbols. Record resolutions in the \p ResolvedODR map |
507 | /// for caching, and in the \p Index for application during the ThinLTO |
508 | /// backends. This is needed for correctness for exported symbols (ensure |
509 | /// at least one copy kept) and a compile-time optimization (to drop duplicate |
510 | /// copies when possible). |
511 | static void resolvePrevailingInIndex( |
512 | ModuleSummaryIndex &Index, |
513 | StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> |
514 | &ResolvedODR, |
515 | const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols, |
516 | const DenseMap<GlobalValue::GUID, const GlobalValueSummary *> |
517 | &PrevailingCopy) { |
518 | |
519 | auto isPrevailing = [&](GlobalValue::GUID GUID, const GlobalValueSummary *S) { |
520 | const auto &Prevailing = PrevailingCopy.find(Val: GUID); |
521 | // Not in map means that there was only one copy, which must be prevailing. |
522 | if (Prevailing == PrevailingCopy.end()) |
523 | return true; |
524 | return Prevailing->second == S; |
525 | }; |
526 | |
527 | auto recordNewLinkage = [&](StringRef ModuleIdentifier, |
528 | GlobalValue::GUID GUID, |
529 | GlobalValue::LinkageTypes NewLinkage) { |
530 | ResolvedODR[ModuleIdentifier][GUID] = NewLinkage; |
531 | }; |
532 | |
533 | // TODO Conf.VisibilityScheme can be lto::Config::ELF for ELF. |
534 | lto::Config Conf; |
535 | thinLTOResolvePrevailingInIndex(C: Conf, Index, isPrevailing, recordNewLinkage, |
536 | GUIDPreservedSymbols); |
537 | } |
538 | |
539 | // Initialize the TargetMachine builder for a given Triple |
540 | static void initTMBuilder(TargetMachineBuilder &TMBuilder, |
541 | const Triple &TheTriple) { |
542 | // Set a default CPU for Darwin triples (copied from LTOCodeGenerator). |
543 | // FIXME this looks pretty terrible... |
544 | if (TMBuilder.MCpu.empty() && TheTriple.isOSDarwin()) { |
545 | if (TheTriple.getArch() == llvm::Triple::x86_64) |
546 | TMBuilder.MCpu = "core2" ; |
547 | else if (TheTriple.getArch() == llvm::Triple::x86) |
548 | TMBuilder.MCpu = "yonah" ; |
549 | else if (TheTriple.getArch() == llvm::Triple::aarch64 || |
550 | TheTriple.getArch() == llvm::Triple::aarch64_32) |
551 | TMBuilder.MCpu = "cyclone" ; |
552 | } |
553 | TMBuilder.TheTriple = std::move(TheTriple); |
554 | } |
555 | |
556 | } // end anonymous namespace |
557 | |
558 | void ThinLTOCodeGenerator::addModule(StringRef Identifier, StringRef Data) { |
559 | MemoryBufferRef Buffer(Data, Identifier); |
560 | |
561 | auto InputOrError = lto::InputFile::create(Object: Buffer); |
562 | if (!InputOrError) |
563 | report_fatal_error(reason: Twine("ThinLTO cannot create input file: " ) + |
564 | toString(E: InputOrError.takeError())); |
565 | |
566 | auto TripleStr = (*InputOrError)->getTargetTriple(); |
567 | Triple TheTriple(TripleStr); |
568 | |
569 | if (Modules.empty()) |
570 | initTMBuilder(TMBuilder, TheTriple: Triple(TheTriple)); |
571 | else if (TMBuilder.TheTriple != TheTriple) { |
572 | if (!TMBuilder.TheTriple.isCompatibleWith(Other: TheTriple)) |
573 | report_fatal_error(reason: "ThinLTO modules with incompatible triples not " |
574 | "supported" ); |
575 | initTMBuilder(TMBuilder, TheTriple: Triple(TMBuilder.TheTriple.merge(Other: TheTriple))); |
576 | } |
577 | |
578 | Modules.emplace_back(args: std::move(*InputOrError)); |
579 | } |
580 | |
581 | void ThinLTOCodeGenerator::preserveSymbol(StringRef Name) { |
582 | PreservedSymbols.insert(key: Name); |
583 | } |
584 | |
585 | void ThinLTOCodeGenerator::crossReferenceSymbol(StringRef Name) { |
586 | // FIXME: At the moment, we don't take advantage of this extra information, |
587 | // we're conservatively considering cross-references as preserved. |
588 | // CrossReferencedSymbols.insert(Name); |
589 | PreservedSymbols.insert(key: Name); |
590 | } |
591 | |
592 | // TargetMachine factory |
593 | std::unique_ptr<TargetMachine> TargetMachineBuilder::create() const { |
594 | std::string ErrMsg; |
595 | const Target *TheTarget = |
596 | TargetRegistry::lookupTarget(Triple: TheTriple.str(), Error&: ErrMsg); |
597 | if (!TheTarget) { |
598 | report_fatal_error(reason: Twine("Can't load target for this Triple: " ) + ErrMsg); |
599 | } |
600 | |
601 | // Use MAttr as the default set of features. |
602 | SubtargetFeatures Features(MAttr); |
603 | Features.getDefaultSubtargetFeatures(Triple: TheTriple); |
604 | std::string FeatureStr = Features.getString(); |
605 | |
606 | std::unique_ptr<TargetMachine> TM( |
607 | TheTarget->createTargetMachine(TT: TheTriple.str(), CPU: MCpu, Features: FeatureStr, Options, |
608 | RM: RelocModel, CM: std::nullopt, OL: CGOptLevel)); |
609 | assert(TM && "Cannot create target machine" ); |
610 | |
611 | return TM; |
612 | } |
613 | |
614 | /** |
615 | * Produce the combined summary index from all the bitcode files: |
616 | * "thin-link". |
617 | */ |
618 | std::unique_ptr<ModuleSummaryIndex> ThinLTOCodeGenerator::linkCombinedIndex() { |
619 | std::unique_ptr<ModuleSummaryIndex> CombinedIndex = |
620 | std::make_unique<ModuleSummaryIndex>(/*HaveGVs=*/args: false); |
621 | for (auto &Mod : Modules) { |
622 | auto &M = Mod->getSingleBitcodeModule(); |
623 | if (Error Err = M.readSummary(CombinedIndex&: *CombinedIndex, ModulePath: Mod->getName())) { |
624 | // FIXME diagnose |
625 | logAllUnhandledErrors( |
626 | E: std::move(Err), OS&: errs(), |
627 | ErrorBanner: "error: can't create module summary index for buffer: " ); |
628 | return nullptr; |
629 | } |
630 | } |
631 | return CombinedIndex; |
632 | } |
633 | |
634 | namespace { |
635 | struct IsExported { |
636 | const DenseMap<StringRef, FunctionImporter::ExportSetTy> &ExportLists; |
637 | const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols; |
638 | |
639 | IsExported( |
640 | const DenseMap<StringRef, FunctionImporter::ExportSetTy> &ExportLists, |
641 | const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols) |
642 | : ExportLists(ExportLists), GUIDPreservedSymbols(GUIDPreservedSymbols) {} |
643 | |
644 | bool operator()(StringRef ModuleIdentifier, ValueInfo VI) const { |
645 | const auto &ExportList = ExportLists.find(Val: ModuleIdentifier); |
646 | return (ExportList != ExportLists.end() && ExportList->second.count(V: VI)) || |
647 | GUIDPreservedSymbols.count(V: VI.getGUID()); |
648 | } |
649 | }; |
650 | |
651 | struct IsPrevailing { |
652 | const DenseMap<GlobalValue::GUID, const GlobalValueSummary *> &PrevailingCopy; |
653 | IsPrevailing(const DenseMap<GlobalValue::GUID, const GlobalValueSummary *> |
654 | &PrevailingCopy) |
655 | : PrevailingCopy(PrevailingCopy) {} |
656 | |
657 | bool operator()(GlobalValue::GUID GUID, const GlobalValueSummary *S) const { |
658 | const auto &Prevailing = PrevailingCopy.find(Val: GUID); |
659 | // Not in map means that there was only one copy, which must be prevailing. |
660 | if (Prevailing == PrevailingCopy.end()) |
661 | return true; |
662 | return Prevailing->second == S; |
663 | }; |
664 | }; |
665 | } // namespace |
666 | |
667 | static void computeDeadSymbolsInIndex( |
668 | ModuleSummaryIndex &Index, |
669 | const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols) { |
670 | // We have no symbols resolution available. And can't do any better now in the |
671 | // case where the prevailing symbol is in a native object. It can be refined |
672 | // with linker information in the future. |
673 | auto isPrevailing = [&](GlobalValue::GUID G) { |
674 | return PrevailingType::Unknown; |
675 | }; |
676 | computeDeadSymbolsWithConstProp(Index, GUIDPreservedSymbols, isPrevailing, |
677 | /* ImportEnabled = */ true); |
678 | } |
679 | |
680 | /** |
681 | * Perform promotion and renaming of exported internal functions. |
682 | * Index is updated to reflect linkage changes from weak resolution. |
683 | */ |
684 | void ThinLTOCodeGenerator::promote(Module &TheModule, ModuleSummaryIndex &Index, |
685 | const lto::InputFile &File) { |
686 | auto ModuleCount = Index.modulePaths().size(); |
687 | auto ModuleIdentifier = TheModule.getModuleIdentifier(); |
688 | |
689 | // Collect for each module the list of function it defines (GUID -> Summary). |
690 | DenseMap<StringRef, GVSummaryMapTy> ModuleToDefinedGVSummaries; |
691 | Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries); |
692 | |
693 | // Convert the preserved symbols set from string to GUID |
694 | auto GUIDPreservedSymbols = computeGUIDPreservedSymbols( |
695 | File, PreservedSymbols, TheTriple: Triple(TheModule.getTargetTriple())); |
696 | |
697 | // Add used symbol to the preserved symbols. |
698 | addUsedSymbolToPreservedGUID(File, PreservedGUID&: GUIDPreservedSymbols); |
699 | |
700 | // Compute "dead" symbols, we don't want to import/export these! |
701 | computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols); |
702 | |
703 | // Compute prevailing symbols |
704 | DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy; |
705 | computePrevailingCopies(Index, PrevailingCopy); |
706 | |
707 | // Generate import/export list |
708 | DenseMap<StringRef, FunctionImporter::ImportMapTy> ImportLists(ModuleCount); |
709 | DenseMap<StringRef, FunctionImporter::ExportSetTy> ExportLists(ModuleCount); |
710 | ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries, |
711 | isPrevailing: IsPrevailing(PrevailingCopy), ImportLists, |
712 | ExportLists); |
713 | |
714 | // Resolve prevailing symbols |
715 | StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR; |
716 | resolvePrevailingInIndex(Index, ResolvedODR, GUIDPreservedSymbols, |
717 | PrevailingCopy); |
718 | |
719 | thinLTOFinalizeInModule(TheModule, |
720 | DefinedGlobals: ModuleToDefinedGVSummaries[ModuleIdentifier], |
721 | /*PropagateAttrs=*/false); |
722 | |
723 | // Promote the exported values in the index, so that they are promoted |
724 | // in the module. |
725 | thinLTOInternalizeAndPromoteInIndex( |
726 | Index, isExported: IsExported(ExportLists, GUIDPreservedSymbols), |
727 | isPrevailing: IsPrevailing(PrevailingCopy)); |
728 | |
729 | // FIXME Set ClearDSOLocalOnDeclarations. |
730 | promoteModule(TheModule, Index, /*ClearDSOLocalOnDeclarations=*/false); |
731 | } |
732 | |
733 | /** |
734 | * Perform cross-module importing for the module identified by ModuleIdentifier. |
735 | */ |
736 | void ThinLTOCodeGenerator::crossModuleImport(Module &TheModule, |
737 | ModuleSummaryIndex &Index, |
738 | const lto::InputFile &File) { |
739 | auto ModuleMap = generateModuleMap(Modules); |
740 | auto ModuleCount = Index.modulePaths().size(); |
741 | |
742 | // Collect for each module the list of function it defines (GUID -> Summary). |
743 | DenseMap<StringRef, GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount); |
744 | Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries); |
745 | |
746 | // Convert the preserved symbols set from string to GUID |
747 | auto GUIDPreservedSymbols = computeGUIDPreservedSymbols( |
748 | File, PreservedSymbols, TheTriple: Triple(TheModule.getTargetTriple())); |
749 | |
750 | addUsedSymbolToPreservedGUID(File, PreservedGUID&: GUIDPreservedSymbols); |
751 | |
752 | // Compute "dead" symbols, we don't want to import/export these! |
753 | computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols); |
754 | |
755 | // Compute prevailing symbols |
756 | DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy; |
757 | computePrevailingCopies(Index, PrevailingCopy); |
758 | |
759 | // Generate import/export list |
760 | DenseMap<StringRef, FunctionImporter::ImportMapTy> ImportLists(ModuleCount); |
761 | DenseMap<StringRef, FunctionImporter::ExportSetTy> ExportLists(ModuleCount); |
762 | ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries, |
763 | isPrevailing: IsPrevailing(PrevailingCopy), ImportLists, |
764 | ExportLists); |
765 | auto &ImportList = ImportLists[TheModule.getModuleIdentifier()]; |
766 | |
767 | // FIXME Set ClearDSOLocalOnDeclarations. |
768 | crossImportIntoModule(TheModule, Index, ModuleMap, ImportList, |
769 | /*ClearDSOLocalOnDeclarations=*/false); |
770 | } |
771 | |
772 | /** |
773 | * Compute the list of summaries needed for importing into module. |
774 | */ |
775 | void ThinLTOCodeGenerator::gatherImportedSummariesForModule( |
776 | Module &TheModule, ModuleSummaryIndex &Index, |
777 | std::map<std::string, GVSummaryMapTy> &ModuleToSummariesForIndex, |
778 | const lto::InputFile &File) { |
779 | auto ModuleCount = Index.modulePaths().size(); |
780 | auto ModuleIdentifier = TheModule.getModuleIdentifier(); |
781 | |
782 | // Collect for each module the list of function it defines (GUID -> Summary). |
783 | DenseMap<StringRef, GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount); |
784 | Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries); |
785 | |
786 | // Convert the preserved symbols set from string to GUID |
787 | auto GUIDPreservedSymbols = computeGUIDPreservedSymbols( |
788 | File, PreservedSymbols, TheTriple: Triple(TheModule.getTargetTriple())); |
789 | |
790 | addUsedSymbolToPreservedGUID(File, PreservedGUID&: GUIDPreservedSymbols); |
791 | |
792 | // Compute "dead" symbols, we don't want to import/export these! |
793 | computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols); |
794 | |
795 | // Compute prevailing symbols |
796 | DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy; |
797 | computePrevailingCopies(Index, PrevailingCopy); |
798 | |
799 | // Generate import/export list |
800 | DenseMap<StringRef, FunctionImporter::ImportMapTy> ImportLists(ModuleCount); |
801 | DenseMap<StringRef, FunctionImporter::ExportSetTy> ExportLists(ModuleCount); |
802 | ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries, |
803 | isPrevailing: IsPrevailing(PrevailingCopy), ImportLists, |
804 | ExportLists); |
805 | |
806 | llvm::gatherImportedSummariesForModule( |
807 | ModulePath: ModuleIdentifier, ModuleToDefinedGVSummaries, |
808 | ImportList: ImportLists[ModuleIdentifier], ModuleToSummariesForIndex); |
809 | } |
810 | |
811 | /** |
812 | * Emit the list of files needed for importing into module. |
813 | */ |
814 | void ThinLTOCodeGenerator::emitImports(Module &TheModule, StringRef OutputName, |
815 | ModuleSummaryIndex &Index, |
816 | const lto::InputFile &File) { |
817 | auto ModuleCount = Index.modulePaths().size(); |
818 | auto ModuleIdentifier = TheModule.getModuleIdentifier(); |
819 | |
820 | // Collect for each module the list of function it defines (GUID -> Summary). |
821 | DenseMap<StringRef, GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount); |
822 | Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries); |
823 | |
824 | // Convert the preserved symbols set from string to GUID |
825 | auto GUIDPreservedSymbols = computeGUIDPreservedSymbols( |
826 | File, PreservedSymbols, TheTriple: Triple(TheModule.getTargetTriple())); |
827 | |
828 | addUsedSymbolToPreservedGUID(File, PreservedGUID&: GUIDPreservedSymbols); |
829 | |
830 | // Compute "dead" symbols, we don't want to import/export these! |
831 | computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols); |
832 | |
833 | // Compute prevailing symbols |
834 | DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy; |
835 | computePrevailingCopies(Index, PrevailingCopy); |
836 | |
837 | // Generate import/export list |
838 | DenseMap<StringRef, FunctionImporter::ImportMapTy> ImportLists(ModuleCount); |
839 | DenseMap<StringRef, FunctionImporter::ExportSetTy> ExportLists(ModuleCount); |
840 | ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries, |
841 | isPrevailing: IsPrevailing(PrevailingCopy), ImportLists, |
842 | ExportLists); |
843 | |
844 | std::map<std::string, GVSummaryMapTy> ModuleToSummariesForIndex; |
845 | llvm::gatherImportedSummariesForModule( |
846 | ModulePath: ModuleIdentifier, ModuleToDefinedGVSummaries, |
847 | ImportList: ImportLists[ModuleIdentifier], ModuleToSummariesForIndex); |
848 | |
849 | std::error_code EC; |
850 | if ((EC = EmitImportsFiles(ModulePath: ModuleIdentifier, OutputFilename: OutputName, |
851 | ModuleToSummariesForIndex))) |
852 | report_fatal_error(reason: Twine("Failed to open " ) + OutputName + |
853 | " to save imports lists\n" ); |
854 | } |
855 | |
856 | /** |
857 | * Perform internalization. Runs promote and internalization together. |
858 | * Index is updated to reflect linkage changes. |
859 | */ |
860 | void ThinLTOCodeGenerator::internalize(Module &TheModule, |
861 | ModuleSummaryIndex &Index, |
862 | const lto::InputFile &File) { |
863 | initTMBuilder(TMBuilder, TheTriple: Triple(TheModule.getTargetTriple())); |
864 | auto ModuleCount = Index.modulePaths().size(); |
865 | auto ModuleIdentifier = TheModule.getModuleIdentifier(); |
866 | |
867 | // Convert the preserved symbols set from string to GUID |
868 | auto GUIDPreservedSymbols = |
869 | computeGUIDPreservedSymbols(File, PreservedSymbols, TheTriple: TMBuilder.TheTriple); |
870 | |
871 | addUsedSymbolToPreservedGUID(File, PreservedGUID&: GUIDPreservedSymbols); |
872 | |
873 | // Collect for each module the list of function it defines (GUID -> Summary). |
874 | DenseMap<StringRef, GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount); |
875 | Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries); |
876 | |
877 | // Compute "dead" symbols, we don't want to import/export these! |
878 | computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols); |
879 | |
880 | // Compute prevailing symbols |
881 | DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy; |
882 | computePrevailingCopies(Index, PrevailingCopy); |
883 | |
884 | // Generate import/export list |
885 | DenseMap<StringRef, FunctionImporter::ImportMapTy> ImportLists(ModuleCount); |
886 | DenseMap<StringRef, FunctionImporter::ExportSetTy> ExportLists(ModuleCount); |
887 | ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries, |
888 | isPrevailing: IsPrevailing(PrevailingCopy), ImportLists, |
889 | ExportLists); |
890 | auto &ExportList = ExportLists[ModuleIdentifier]; |
891 | |
892 | // Be friendly and don't nuke totally the module when the client didn't |
893 | // supply anything to preserve. |
894 | if (ExportList.empty() && GUIDPreservedSymbols.empty()) |
895 | return; |
896 | |
897 | // Resolve prevailing symbols |
898 | StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR; |
899 | resolvePrevailingInIndex(Index, ResolvedODR, GUIDPreservedSymbols, |
900 | PrevailingCopy); |
901 | |
902 | // Promote the exported values in the index, so that they are promoted |
903 | // in the module. |
904 | thinLTOInternalizeAndPromoteInIndex( |
905 | Index, isExported: IsExported(ExportLists, GUIDPreservedSymbols), |
906 | isPrevailing: IsPrevailing(PrevailingCopy)); |
907 | |
908 | // FIXME Set ClearDSOLocalOnDeclarations. |
909 | promoteModule(TheModule, Index, /*ClearDSOLocalOnDeclarations=*/false); |
910 | |
911 | // Internalization |
912 | thinLTOFinalizeInModule(TheModule, |
913 | DefinedGlobals: ModuleToDefinedGVSummaries[ModuleIdentifier], |
914 | /*PropagateAttrs=*/false); |
915 | |
916 | thinLTOInternalizeModule(TheModule, |
917 | DefinedGlobals: ModuleToDefinedGVSummaries[ModuleIdentifier]); |
918 | } |
919 | |
920 | /** |
921 | * Perform post-importing ThinLTO optimizations. |
922 | */ |
923 | void ThinLTOCodeGenerator::optimize(Module &TheModule) { |
924 | initTMBuilder(TMBuilder, TheTriple: Triple(TheModule.getTargetTriple())); |
925 | |
926 | // Optimize now |
927 | optimizeModule(TheModule, TM&: *TMBuilder.create(), OptLevel, Freestanding, |
928 | DebugPassManager, Index: nullptr); |
929 | } |
930 | |
931 | /// Write out the generated object file, either from CacheEntryPath or from |
932 | /// OutputBuffer, preferring hard-link when possible. |
933 | /// Returns the path to the generated file in SavedObjectsDirectoryPath. |
934 | std::string |
935 | ThinLTOCodeGenerator::writeGeneratedObject(int count, StringRef CacheEntryPath, |
936 | const MemoryBuffer &OutputBuffer) { |
937 | auto ArchName = TMBuilder.TheTriple.getArchName(); |
938 | SmallString<128> OutputPath(SavedObjectsDirectoryPath); |
939 | llvm::sys::path::append(path&: OutputPath, |
940 | a: Twine(count) + "." + ArchName + ".thinlto.o" ); |
941 | OutputPath.c_str(); // Ensure the string is null terminated. |
942 | if (sys::fs::exists(Path: OutputPath)) |
943 | sys::fs::remove(path: OutputPath); |
944 | |
945 | // We don't return a memory buffer to the linker, just a list of files. |
946 | if (!CacheEntryPath.empty()) { |
947 | // Cache is enabled, hard-link the entry (or copy if hard-link fails). |
948 | auto Err = sys::fs::create_hard_link(to: CacheEntryPath, from: OutputPath); |
949 | if (!Err) |
950 | return std::string(OutputPath); |
951 | // Hard linking failed, try to copy. |
952 | Err = sys::fs::copy_file(From: CacheEntryPath, To: OutputPath); |
953 | if (!Err) |
954 | return std::string(OutputPath); |
955 | // Copy failed (could be because the CacheEntry was removed from the cache |
956 | // in the meantime by another process), fall back and try to write down the |
957 | // buffer to the output. |
958 | errs() << "remark: can't link or copy from cached entry '" << CacheEntryPath |
959 | << "' to '" << OutputPath << "'\n" ; |
960 | } |
961 | // No cache entry, just write out the buffer. |
962 | std::error_code Err; |
963 | raw_fd_ostream OS(OutputPath, Err, sys::fs::OF_None); |
964 | if (Err) |
965 | report_fatal_error(reason: Twine("Can't open output '" ) + OutputPath + "'\n" ); |
966 | OS << OutputBuffer.getBuffer(); |
967 | return std::string(OutputPath); |
968 | } |
969 | |
970 | // Main entry point for the ThinLTO processing |
971 | void ThinLTOCodeGenerator::run() { |
972 | timeTraceProfilerBegin(Name: "ThinLink" , Detail: StringRef("" )); |
973 | auto TimeTraceScopeExit = llvm::make_scope_exit(F: []() { |
974 | if (llvm::timeTraceProfilerEnabled()) |
975 | llvm::timeTraceProfilerEnd(); |
976 | }); |
977 | // Prepare the resulting object vector |
978 | assert(ProducedBinaries.empty() && "The generator should not be reused" ); |
979 | if (SavedObjectsDirectoryPath.empty()) |
980 | ProducedBinaries.resize(new_size: Modules.size()); |
981 | else { |
982 | sys::fs::create_directories(path: SavedObjectsDirectoryPath); |
983 | bool IsDir; |
984 | sys::fs::is_directory(path: SavedObjectsDirectoryPath, result&: IsDir); |
985 | if (!IsDir) |
986 | report_fatal_error(reason: Twine("Unexistent dir: '" ) + SavedObjectsDirectoryPath + "'" ); |
987 | ProducedBinaryFiles.resize(new_size: Modules.size()); |
988 | } |
989 | |
990 | if (CodeGenOnly) { |
991 | // Perform only parallel codegen and return. |
992 | ThreadPool Pool; |
993 | int count = 0; |
994 | for (auto &Mod : Modules) { |
995 | Pool.async(F: [&](int count) { |
996 | LLVMContext Context; |
997 | Context.setDiscardValueNames(LTODiscardValueNames); |
998 | |
999 | // Parse module now |
1000 | auto TheModule = loadModuleFromInput(Input: Mod.get(), Context, Lazy: false, |
1001 | /*IsImporting*/ false); |
1002 | |
1003 | // CodeGen |
1004 | auto OutputBuffer = codegenModule(TheModule&: *TheModule, TM&: *TMBuilder.create()); |
1005 | if (SavedObjectsDirectoryPath.empty()) |
1006 | ProducedBinaries[count] = std::move(OutputBuffer); |
1007 | else |
1008 | ProducedBinaryFiles[count] = |
1009 | writeGeneratedObject(count, CacheEntryPath: "" , OutputBuffer: *OutputBuffer); |
1010 | }, ArgList: count++); |
1011 | } |
1012 | |
1013 | return; |
1014 | } |
1015 | |
1016 | // Sequential linking phase |
1017 | auto Index = linkCombinedIndex(); |
1018 | |
1019 | // Save temps: index. |
1020 | if (!SaveTempsDir.empty()) { |
1021 | auto SaveTempPath = SaveTempsDir + "index.bc" ; |
1022 | std::error_code EC; |
1023 | raw_fd_ostream OS(SaveTempPath, EC, sys::fs::OF_None); |
1024 | if (EC) |
1025 | report_fatal_error(reason: Twine("Failed to open " ) + SaveTempPath + |
1026 | " to save optimized bitcode\n" ); |
1027 | writeIndexToFile(Index: *Index, Out&: OS); |
1028 | } |
1029 | |
1030 | |
1031 | // Prepare the module map. |
1032 | auto ModuleMap = generateModuleMap(Modules); |
1033 | auto ModuleCount = Modules.size(); |
1034 | |
1035 | // Collect for each module the list of function it defines (GUID -> Summary). |
1036 | DenseMap<StringRef, GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount); |
1037 | Index->collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries); |
1038 | |
1039 | // Convert the preserved symbols set from string to GUID, this is needed for |
1040 | // computing the caching hash and the internalization. |
1041 | DenseSet<GlobalValue::GUID> GUIDPreservedSymbols; |
1042 | for (const auto &M : Modules) |
1043 | computeGUIDPreservedSymbols(File: *M, PreservedSymbols, TheTriple: TMBuilder.TheTriple, |
1044 | GUIDs&: GUIDPreservedSymbols); |
1045 | |
1046 | // Add used symbol from inputs to the preserved symbols. |
1047 | for (const auto &M : Modules) |
1048 | addUsedSymbolToPreservedGUID(File: *M, PreservedGUID&: GUIDPreservedSymbols); |
1049 | |
1050 | // Compute "dead" symbols, we don't want to import/export these! |
1051 | computeDeadSymbolsInIndex(Index&: *Index, GUIDPreservedSymbols); |
1052 | |
1053 | // Synthesize entry counts for functions in the combined index. |
1054 | computeSyntheticCounts(Index&: *Index); |
1055 | |
1056 | // Currently there is no support for enabling whole program visibility via a |
1057 | // linker option in the old LTO API, but this call allows it to be specified |
1058 | // via the internal option. Must be done before WPD below. |
1059 | if (hasWholeProgramVisibility(/* WholeProgramVisibilityEnabledInLTO */ false)) |
1060 | Index->setWithWholeProgramVisibility(); |
1061 | |
1062 | // FIXME: This needs linker information via a TBD new interface |
1063 | updateVCallVisibilityInIndex(Index&: *Index, |
1064 | /*WholeProgramVisibilityEnabledInLTO=*/false, |
1065 | // FIXME: These need linker information via a |
1066 | // TBD new interface. |
1067 | /*DynamicExportSymbols=*/{}, |
1068 | /*VisibleToRegularObjSymbols=*/{}); |
1069 | |
1070 | // Perform index-based WPD. This will return immediately if there are |
1071 | // no index entries in the typeIdMetadata map (e.g. if we are instead |
1072 | // performing IR-based WPD in hybrid regular/thin LTO mode). |
1073 | std::map<ValueInfo, std::vector<VTableSlotSummary>> LocalWPDTargetsMap; |
1074 | std::set<GlobalValue::GUID> ExportedGUIDs; |
1075 | runWholeProgramDevirtOnIndex(Summary&: *Index, ExportedGUIDs, LocalWPDTargetsMap); |
1076 | for (auto GUID : ExportedGUIDs) |
1077 | GUIDPreservedSymbols.insert(V: GUID); |
1078 | |
1079 | // Compute prevailing symbols |
1080 | DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy; |
1081 | computePrevailingCopies(Index: *Index, PrevailingCopy); |
1082 | |
1083 | // Collect the import/export lists for all modules from the call-graph in the |
1084 | // combined index. |
1085 | DenseMap<StringRef, FunctionImporter::ImportMapTy> ImportLists(ModuleCount); |
1086 | DenseMap<StringRef, FunctionImporter::ExportSetTy> ExportLists(ModuleCount); |
1087 | ComputeCrossModuleImport(Index: *Index, ModuleToDefinedGVSummaries, |
1088 | isPrevailing: IsPrevailing(PrevailingCopy), ImportLists, |
1089 | ExportLists); |
1090 | |
1091 | // We use a std::map here to be able to have a defined ordering when |
1092 | // producing a hash for the cache entry. |
1093 | // FIXME: we should be able to compute the caching hash for the entry based |
1094 | // on the index, and nuke this map. |
1095 | StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR; |
1096 | |
1097 | // Resolve prevailing symbols, this has to be computed early because it |
1098 | // impacts the caching. |
1099 | resolvePrevailingInIndex(Index&: *Index, ResolvedODR, GUIDPreservedSymbols, |
1100 | PrevailingCopy); |
1101 | |
1102 | // Use global summary-based analysis to identify symbols that can be |
1103 | // internalized (because they aren't exported or preserved as per callback). |
1104 | // Changes are made in the index, consumed in the ThinLTO backends. |
1105 | updateIndexWPDForExports(Summary&: *Index, |
1106 | isExported: IsExported(ExportLists, GUIDPreservedSymbols), |
1107 | LocalWPDTargetsMap); |
1108 | thinLTOInternalizeAndPromoteInIndex( |
1109 | Index&: *Index, isExported: IsExported(ExportLists, GUIDPreservedSymbols), |
1110 | isPrevailing: IsPrevailing(PrevailingCopy)); |
1111 | |
1112 | thinLTOPropagateFunctionAttrs(Index&: *Index, isPrevailing: IsPrevailing(PrevailingCopy)); |
1113 | |
1114 | // Make sure that every module has an entry in the ExportLists, ImportList, |
1115 | // GVSummary and ResolvedODR maps to enable threaded access to these maps |
1116 | // below. |
1117 | for (auto &Module : Modules) { |
1118 | auto ModuleIdentifier = Module->getName(); |
1119 | ExportLists[ModuleIdentifier]; |
1120 | ImportLists[ModuleIdentifier]; |
1121 | ResolvedODR[ModuleIdentifier]; |
1122 | ModuleToDefinedGVSummaries[ModuleIdentifier]; |
1123 | } |
1124 | |
1125 | std::vector<BitcodeModule *> ModulesVec; |
1126 | ModulesVec.reserve(n: Modules.size()); |
1127 | for (auto &Mod : Modules) |
1128 | ModulesVec.push_back(x: &Mod->getSingleBitcodeModule()); |
1129 | std::vector<int> ModulesOrdering = lto::generateModulesOrdering(R: ModulesVec); |
1130 | |
1131 | if (llvm::timeTraceProfilerEnabled()) |
1132 | llvm::timeTraceProfilerEnd(); |
1133 | |
1134 | TimeTraceScopeExit.release(); |
1135 | |
1136 | // Parallel optimizer + codegen |
1137 | { |
1138 | ThreadPool Pool(heavyweight_hardware_concurrency(ThreadCount)); |
1139 | for (auto IndexCount : ModulesOrdering) { |
1140 | auto &Mod = Modules[IndexCount]; |
1141 | Pool.async(F: [&](int count) { |
1142 | auto ModuleIdentifier = Mod->getName(); |
1143 | auto &ExportList = ExportLists[ModuleIdentifier]; |
1144 | |
1145 | auto &DefinedGVSummaries = ModuleToDefinedGVSummaries[ModuleIdentifier]; |
1146 | |
1147 | // The module may be cached, this helps handling it. |
1148 | ModuleCacheEntry CacheEntry(CacheOptions.Path, *Index, ModuleIdentifier, |
1149 | ImportLists[ModuleIdentifier], ExportList, |
1150 | ResolvedODR[ModuleIdentifier], |
1151 | DefinedGVSummaries, OptLevel, Freestanding, |
1152 | TMBuilder); |
1153 | auto CacheEntryPath = CacheEntry.getEntryPath(); |
1154 | |
1155 | { |
1156 | auto ErrOrBuffer = CacheEntry.tryLoadingBuffer(); |
1157 | LLVM_DEBUG(dbgs() << "Cache " << (ErrOrBuffer ? "hit" : "miss" ) |
1158 | << " '" << CacheEntryPath << "' for buffer " |
1159 | << count << " " << ModuleIdentifier << "\n" ); |
1160 | |
1161 | if (ErrOrBuffer) { |
1162 | // Cache Hit! |
1163 | if (SavedObjectsDirectoryPath.empty()) |
1164 | ProducedBinaries[count] = std::move(ErrOrBuffer.get()); |
1165 | else |
1166 | ProducedBinaryFiles[count] = writeGeneratedObject( |
1167 | count, CacheEntryPath, OutputBuffer: *ErrOrBuffer.get()); |
1168 | return; |
1169 | } |
1170 | } |
1171 | |
1172 | LLVMContext Context; |
1173 | Context.setDiscardValueNames(LTODiscardValueNames); |
1174 | Context.enableDebugTypeODRUniquing(); |
1175 | auto DiagFileOrErr = lto::setupLLVMOptimizationRemarks( |
1176 | Context, RemarksFilename, RemarksPasses, RemarksFormat, |
1177 | RemarksWithHotness, RemarksHotnessThreshold, Count: count); |
1178 | if (!DiagFileOrErr) { |
1179 | errs() << "Error: " << toString(E: DiagFileOrErr.takeError()) << "\n" ; |
1180 | report_fatal_error(reason: "ThinLTO: Can't get an output file for the " |
1181 | "remarks" ); |
1182 | } |
1183 | |
1184 | // Parse module now |
1185 | auto TheModule = loadModuleFromInput(Input: Mod.get(), Context, Lazy: false, |
1186 | /*IsImporting*/ false); |
1187 | |
1188 | // Save temps: original file. |
1189 | saveTempBitcode(TheModule: *TheModule, TempDir: SaveTempsDir, count, Suffix: ".0.original.bc" ); |
1190 | |
1191 | auto &ImportList = ImportLists[ModuleIdentifier]; |
1192 | // Run the main process now, and generates a binary |
1193 | auto OutputBuffer = ProcessThinLTOModule( |
1194 | TheModule&: *TheModule, Index&: *Index, ModuleMap, TM&: *TMBuilder.create(), ImportList, |
1195 | ExportList, GUIDPreservedSymbols, |
1196 | DefinedGlobals: ModuleToDefinedGVSummaries[ModuleIdentifier], CacheOptions, |
1197 | DisableCodeGen, SaveTempsDir, Freestanding, OptLevel, count, |
1198 | DebugPassManager); |
1199 | |
1200 | // Commit to the cache (if enabled) |
1201 | CacheEntry.write(OutputBuffer: *OutputBuffer); |
1202 | |
1203 | if (SavedObjectsDirectoryPath.empty()) { |
1204 | // We need to generated a memory buffer for the linker. |
1205 | if (!CacheEntryPath.empty()) { |
1206 | // When cache is enabled, reload from the cache if possible. |
1207 | // Releasing the buffer from the heap and reloading it from the |
1208 | // cache file with mmap helps us to lower memory pressure. |
1209 | // The freed memory can be used for the next input file. |
1210 | // The final binary link will read from the VFS cache (hopefully!) |
1211 | // or from disk (if the memory pressure was too high). |
1212 | auto ReloadedBufferOrErr = CacheEntry.tryLoadingBuffer(); |
1213 | if (auto EC = ReloadedBufferOrErr.getError()) { |
1214 | // On error, keep the preexisting buffer and print a diagnostic. |
1215 | errs() << "remark: can't reload cached file '" << CacheEntryPath |
1216 | << "': " << EC.message() << "\n" ; |
1217 | } else { |
1218 | OutputBuffer = std::move(*ReloadedBufferOrErr); |
1219 | } |
1220 | } |
1221 | ProducedBinaries[count] = std::move(OutputBuffer); |
1222 | return; |
1223 | } |
1224 | ProducedBinaryFiles[count] = writeGeneratedObject( |
1225 | count, CacheEntryPath, OutputBuffer: *OutputBuffer); |
1226 | }, ArgList&: IndexCount); |
1227 | } |
1228 | } |
1229 | |
1230 | pruneCache(Path: CacheOptions.Path, Policy: CacheOptions.Policy, Files: ProducedBinaries); |
1231 | |
1232 | // If statistics were requested, print them out now. |
1233 | if (llvm::AreStatisticsEnabled()) |
1234 | llvm::PrintStatistics(); |
1235 | reportAndResetTimings(); |
1236 | } |
1237 | |