1//===-ThinLTOCodeGenerator.cpp - LLVM Link Time Optimizer -----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the Thin Link Time Optimization library. This library is
10// intended to be used by linker to optimize code at link time.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/LTO/legacy/ThinLTOCodeGenerator.h"
15#include "llvm/Support/CommandLine.h"
16
17#include "llvm/ADT/ScopeExit.h"
18#include "llvm/ADT/Statistic.h"
19#include "llvm/ADT/StringExtras.h"
20#include "llvm/Analysis/AliasAnalysis.h"
21#include "llvm/Analysis/ModuleSummaryAnalysis.h"
22#include "llvm/Analysis/ProfileSummaryInfo.h"
23#include "llvm/Analysis/TargetLibraryInfo.h"
24#include "llvm/Bitcode/BitcodeReader.h"
25#include "llvm/Bitcode/BitcodeWriter.h"
26#include "llvm/Bitcode/BitcodeWriterPass.h"
27#include "llvm/Config/llvm-config.h"
28#include "llvm/IR/DebugInfo.h"
29#include "llvm/IR/DiagnosticPrinter.h"
30#include "llvm/IR/LegacyPassManager.h"
31#include "llvm/IR/LLVMContext.h"
32#include "llvm/IR/LLVMRemarkStreamer.h"
33#include "llvm/IR/Mangler.h"
34#include "llvm/IR/PassTimingInfo.h"
35#include "llvm/IR/Verifier.h"
36#include "llvm/IRReader/IRReader.h"
37#include "llvm/LTO/LTO.h"
38#include "llvm/LTO/SummaryBasedOptimizations.h"
39#include "llvm/MC/TargetRegistry.h"
40#include "llvm/Object/IRObjectFile.h"
41#include "llvm/Passes/PassBuilder.h"
42#include "llvm/Passes/StandardInstrumentations.h"
43#include "llvm/Remarks/HotnessThresholdParser.h"
44#include "llvm/Support/CachePruning.h"
45#include "llvm/Support/Debug.h"
46#include "llvm/Support/Error.h"
47#include "llvm/Support/FileSystem.h"
48#include "llvm/Support/Path.h"
49#include "llvm/Support/SHA1.h"
50#include "llvm/Support/SmallVectorMemoryBuffer.h"
51#include "llvm/Support/ThreadPool.h"
52#include "llvm/Support/Threading.h"
53#include "llvm/Support/ToolOutputFile.h"
54#include "llvm/Support/raw_ostream.h"
55#include "llvm/Target/TargetMachine.h"
56#include "llvm/TargetParser/SubtargetFeature.h"
57#include "llvm/Transforms/IPO/FunctionAttrs.h"
58#include "llvm/Transforms/IPO/FunctionImport.h"
59#include "llvm/Transforms/IPO/Internalize.h"
60#include "llvm/Transforms/IPO/WholeProgramDevirt.h"
61#include "llvm/Transforms/ObjCARC.h"
62#include "llvm/Transforms/Utils/FunctionImportUtils.h"
63
64#include <numeric>
65
66#if !defined(_MSC_VER) && !defined(__MINGW32__)
67#include <unistd.h>
68#else
69#include <io.h>
70#endif
71
72using namespace llvm;
73
74#define DEBUG_TYPE "thinlto"
75
76namespace llvm {
77// Flags -discard-value-names, defined in LTOCodeGenerator.cpp
78extern cl::opt<bool> LTODiscardValueNames;
79extern cl::opt<std::string> RemarksFilename;
80extern cl::opt<std::string> RemarksPasses;
81extern cl::opt<bool> RemarksWithHotness;
82extern cl::opt<std::optional<uint64_t>, false, remarks::HotnessThresholdParser>
83 RemarksHotnessThreshold;
84extern cl::opt<std::string> RemarksFormat;
85}
86
87namespace {
88
89// Default to using all available threads in the system, but using only one
90// thred per core, as indicated by the usage of
91// heavyweight_hardware_concurrency() below.
92static cl::opt<int> ThreadCount("threads", cl::init(Val: 0));
93
94// Simple helper to save temporary files for debug.
95static void saveTempBitcode(const Module &TheModule, StringRef TempDir,
96 unsigned count, StringRef Suffix) {
97 if (TempDir.empty())
98 return;
99 // User asked to save temps, let dump the bitcode file after import.
100 std::string SaveTempPath = (TempDir + llvm::Twine(count) + Suffix).str();
101 std::error_code EC;
102 raw_fd_ostream OS(SaveTempPath, EC, sys::fs::OF_None);
103 if (EC)
104 report_fatal_error(reason: Twine("Failed to open ") + SaveTempPath +
105 " to save optimized bitcode\n");
106 WriteBitcodeToFile(M: TheModule, Out&: OS, /* ShouldPreserveUseListOrder */ true);
107}
108
109static const GlobalValueSummary *
110getFirstDefinitionForLinker(const GlobalValueSummaryList &GVSummaryList) {
111 // If there is any strong definition anywhere, get it.
112 auto StrongDefForLinker = llvm::find_if(
113 Range: GVSummaryList, P: [](const std::unique_ptr<GlobalValueSummary> &Summary) {
114 auto Linkage = Summary->linkage();
115 return !GlobalValue::isAvailableExternallyLinkage(Linkage) &&
116 !GlobalValue::isWeakForLinker(Linkage);
117 });
118 if (StrongDefForLinker != GVSummaryList.end())
119 return StrongDefForLinker->get();
120 // Get the first *linker visible* definition for this global in the summary
121 // list.
122 auto FirstDefForLinker = llvm::find_if(
123 Range: GVSummaryList, P: [](const std::unique_ptr<GlobalValueSummary> &Summary) {
124 auto Linkage = Summary->linkage();
125 return !GlobalValue::isAvailableExternallyLinkage(Linkage);
126 });
127 // Extern templates can be emitted as available_externally.
128 if (FirstDefForLinker == GVSummaryList.end())
129 return nullptr;
130 return FirstDefForLinker->get();
131}
132
133// Populate map of GUID to the prevailing copy for any multiply defined
134// symbols. Currently assume first copy is prevailing, or any strong
135// definition. Can be refined with Linker information in the future.
136static void computePrevailingCopies(
137 const ModuleSummaryIndex &Index,
138 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> &PrevailingCopy) {
139 auto HasMultipleCopies = [&](const GlobalValueSummaryList &GVSummaryList) {
140 return GVSummaryList.size() > 1;
141 };
142
143 for (auto &I : Index) {
144 if (HasMultipleCopies(I.second.SummaryList))
145 PrevailingCopy[I.first] =
146 getFirstDefinitionForLinker(GVSummaryList: I.second.SummaryList);
147 }
148}
149
150static StringMap<lto::InputFile *>
151generateModuleMap(std::vector<std::unique_ptr<lto::InputFile>> &Modules) {
152 StringMap<lto::InputFile *> ModuleMap;
153 for (auto &M : Modules) {
154 LLVM_DEBUG(dbgs() << "Adding module " << M->getName() << " to ModuleMap\n");
155 assert(!ModuleMap.contains(M->getName()) &&
156 "Expect unique Buffer Identifier");
157 ModuleMap[M->getName()] = M.get();
158 }
159 return ModuleMap;
160}
161
162static void promoteModule(Module &TheModule, const ModuleSummaryIndex &Index,
163 bool ClearDSOLocalOnDeclarations) {
164 if (renameModuleForThinLTO(M&: TheModule, Index, ClearDSOLocalOnDeclarations))
165 report_fatal_error(reason: "renameModuleForThinLTO failed");
166}
167
168namespace {
169class ThinLTODiagnosticInfo : public DiagnosticInfo {
170 const Twine &Msg;
171public:
172 ThinLTODiagnosticInfo(const Twine &DiagMsg,
173 DiagnosticSeverity Severity = DS_Error)
174 : DiagnosticInfo(DK_Linker, Severity), Msg(DiagMsg) {}
175 void print(DiagnosticPrinter &DP) const override { DP << Msg; }
176};
177}
178
179/// Verify the module and strip broken debug info.
180static void verifyLoadedModule(Module &TheModule) {
181 bool BrokenDebugInfo = false;
182 if (verifyModule(M: TheModule, OS: &dbgs(), BrokenDebugInfo: &BrokenDebugInfo))
183 report_fatal_error(reason: "Broken module found, compilation aborted!");
184 if (BrokenDebugInfo) {
185 TheModule.getContext().diagnose(DI: ThinLTODiagnosticInfo(
186 "Invalid debug info found, debug info will be stripped", DS_Warning));
187 StripDebugInfo(M&: TheModule);
188 }
189}
190
191static std::unique_ptr<Module> loadModuleFromInput(lto::InputFile *Input,
192 LLVMContext &Context,
193 bool Lazy,
194 bool IsImporting) {
195 auto &Mod = Input->getSingleBitcodeModule();
196 SMDiagnostic Err;
197 Expected<std::unique_ptr<Module>> ModuleOrErr =
198 Lazy ? Mod.getLazyModule(Context,
199 /* ShouldLazyLoadMetadata */ true, IsImporting)
200 : Mod.parseModule(Context);
201 if (!ModuleOrErr) {
202 handleAllErrors(E: ModuleOrErr.takeError(), Handlers: [&](ErrorInfoBase &EIB) {
203 SMDiagnostic Err = SMDiagnostic(Mod.getModuleIdentifier(),
204 SourceMgr::DK_Error, EIB.message());
205 Err.print(ProgName: "ThinLTO", S&: errs());
206 });
207 report_fatal_error(reason: "Can't load module, abort.");
208 }
209 if (!Lazy)
210 verifyLoadedModule(TheModule&: *ModuleOrErr.get());
211 return std::move(*ModuleOrErr);
212}
213
214static void
215crossImportIntoModule(Module &TheModule, const ModuleSummaryIndex &Index,
216 StringMap<lto::InputFile *> &ModuleMap,
217 const FunctionImporter::ImportMapTy &ImportList,
218 bool ClearDSOLocalOnDeclarations) {
219 auto Loader = [&](StringRef Identifier) {
220 auto &Input = ModuleMap[Identifier];
221 return loadModuleFromInput(Input, Context&: TheModule.getContext(),
222 /*Lazy=*/true, /*IsImporting*/ true);
223 };
224
225 FunctionImporter Importer(Index, Loader, ClearDSOLocalOnDeclarations);
226 Expected<bool> Result = Importer.importFunctions(M&: TheModule, ImportList);
227 if (!Result) {
228 handleAllErrors(E: Result.takeError(), Handlers: [&](ErrorInfoBase &EIB) {
229 SMDiagnostic Err = SMDiagnostic(TheModule.getModuleIdentifier(),
230 SourceMgr::DK_Error, EIB.message());
231 Err.print(ProgName: "ThinLTO", S&: errs());
232 });
233 report_fatal_error(reason: "importFunctions failed");
234 }
235 // Verify again after cross-importing.
236 verifyLoadedModule(TheModule);
237}
238
239static void optimizeModule(Module &TheModule, TargetMachine &TM,
240 unsigned OptLevel, bool Freestanding,
241 bool DebugPassManager, ModuleSummaryIndex *Index) {
242 std::optional<PGOOptions> PGOOpt;
243 LoopAnalysisManager LAM;
244 FunctionAnalysisManager FAM;
245 CGSCCAnalysisManager CGAM;
246 ModuleAnalysisManager MAM;
247
248 PassInstrumentationCallbacks PIC;
249 StandardInstrumentations SI(TheModule.getContext(), DebugPassManager);
250 SI.registerCallbacks(PIC, MAM: &MAM);
251 PipelineTuningOptions PTO;
252 PTO.LoopVectorization = true;
253 PTO.SLPVectorization = true;
254 PassBuilder PB(&TM, PTO, PGOOpt, &PIC);
255
256 std::unique_ptr<TargetLibraryInfoImpl> TLII(
257 new TargetLibraryInfoImpl(Triple(TM.getTargetTriple())));
258 if (Freestanding)
259 TLII->disableAllFunctions();
260 FAM.registerPass(PassBuilder: [&] { return TargetLibraryAnalysis(*TLII); });
261
262 // Register all the basic analyses with the managers.
263 PB.registerModuleAnalyses(MAM);
264 PB.registerCGSCCAnalyses(CGAM);
265 PB.registerFunctionAnalyses(FAM);
266 PB.registerLoopAnalyses(LAM);
267 PB.crossRegisterProxies(LAM, FAM, CGAM, MAM);
268
269 ModulePassManager MPM;
270
271 OptimizationLevel OL;
272
273 switch (OptLevel) {
274 default:
275 llvm_unreachable("Invalid optimization level");
276 case 0:
277 OL = OptimizationLevel::O0;
278 break;
279 case 1:
280 OL = OptimizationLevel::O1;
281 break;
282 case 2:
283 OL = OptimizationLevel::O2;
284 break;
285 case 3:
286 OL = OptimizationLevel::O3;
287 break;
288 }
289
290 MPM.addPass(Pass: PB.buildThinLTODefaultPipeline(Level: OL, ImportSummary: Index));
291
292 MPM.run(IR&: TheModule, AM&: MAM);
293}
294
295static void
296addUsedSymbolToPreservedGUID(const lto::InputFile &File,
297 DenseSet<GlobalValue::GUID> &PreservedGUID) {
298 for (const auto &Sym : File.symbols()) {
299 if (Sym.isUsed())
300 PreservedGUID.insert(V: GlobalValue::getGUID(GlobalName: Sym.getIRName()));
301 }
302}
303
304// Convert the PreservedSymbols map from "Name" based to "GUID" based.
305static void computeGUIDPreservedSymbols(const lto::InputFile &File,
306 const StringSet<> &PreservedSymbols,
307 const Triple &TheTriple,
308 DenseSet<GlobalValue::GUID> &GUIDs) {
309 // Iterate the symbols in the input file and if the input has preserved symbol
310 // compute the GUID for the symbol.
311 for (const auto &Sym : File.symbols()) {
312 if (PreservedSymbols.count(Key: Sym.getName()) && !Sym.getIRName().empty())
313 GUIDs.insert(V: GlobalValue::getGUID(GlobalName: GlobalValue::getGlobalIdentifier(
314 Name: Sym.getIRName(), Linkage: GlobalValue::ExternalLinkage, FileName: "")));
315 }
316}
317
318static DenseSet<GlobalValue::GUID>
319computeGUIDPreservedSymbols(const lto::InputFile &File,
320 const StringSet<> &PreservedSymbols,
321 const Triple &TheTriple) {
322 DenseSet<GlobalValue::GUID> GUIDPreservedSymbols(PreservedSymbols.size());
323 computeGUIDPreservedSymbols(File, PreservedSymbols, TheTriple,
324 GUIDs&: GUIDPreservedSymbols);
325 return GUIDPreservedSymbols;
326}
327
328std::unique_ptr<MemoryBuffer> codegenModule(Module &TheModule,
329 TargetMachine &TM) {
330 SmallVector<char, 128> OutputBuffer;
331
332 // CodeGen
333 {
334 raw_svector_ostream OS(OutputBuffer);
335 legacy::PassManager PM;
336
337 // If the bitcode files contain ARC code and were compiled with optimization,
338 // the ObjCARCContractPass must be run, so do it unconditionally here.
339 PM.add(P: createObjCARCContractPass());
340
341 // Setup the codegen now.
342 if (TM.addPassesToEmitFile(PM, OS, nullptr, CodeGenFileType::ObjectFile,
343 /* DisableVerify */ true))
344 report_fatal_error(reason: "Failed to setup codegen");
345
346 // Run codegen now. resulting binary is in OutputBuffer.
347 PM.run(M&: TheModule);
348 }
349 return std::make_unique<SmallVectorMemoryBuffer>(
350 args: std::move(OutputBuffer), /*RequiresNullTerminator=*/args: false);
351}
352
353/// Manage caching for a single Module.
354class ModuleCacheEntry {
355 SmallString<128> EntryPath;
356
357public:
358 // Create a cache entry. This compute a unique hash for the Module considering
359 // the current list of export/import, and offer an interface to query to
360 // access the content in the cache.
361 ModuleCacheEntry(
362 StringRef CachePath, const ModuleSummaryIndex &Index, StringRef ModuleID,
363 const FunctionImporter::ImportMapTy &ImportList,
364 const FunctionImporter::ExportSetTy &ExportList,
365 const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,
366 const GVSummaryMapTy &DefinedGVSummaries, unsigned OptLevel,
367 bool Freestanding, const TargetMachineBuilder &TMBuilder) {
368 if (CachePath.empty())
369 return;
370
371 if (!Index.modulePaths().count(Key: ModuleID))
372 // The module does not have an entry, it can't have a hash at all
373 return;
374
375 if (all_of(Range: Index.getModuleHash(ModPath: ModuleID),
376 P: [](uint32_t V) { return V == 0; }))
377 // No hash entry, no caching!
378 return;
379
380 llvm::lto::Config Conf;
381 Conf.OptLevel = OptLevel;
382 Conf.Options = TMBuilder.Options;
383 Conf.CPU = TMBuilder.MCpu;
384 Conf.MAttrs.push_back(x: TMBuilder.MAttr);
385 Conf.RelocModel = TMBuilder.RelocModel;
386 Conf.CGOptLevel = TMBuilder.CGOptLevel;
387 Conf.Freestanding = Freestanding;
388 SmallString<40> Key;
389 computeLTOCacheKey(Key, Conf, Index, ModuleID, ImportList, ExportList,
390 ResolvedODR, DefinedGlobals: DefinedGVSummaries);
391
392 // This choice of file name allows the cache to be pruned (see pruneCache()
393 // in include/llvm/Support/CachePruning.h).
394 sys::path::append(path&: EntryPath, a: CachePath, b: "llvmcache-" + Key);
395 }
396
397 // Access the path to this entry in the cache.
398 StringRef getEntryPath() { return EntryPath; }
399
400 // Try loading the buffer for this cache entry.
401 ErrorOr<std::unique_ptr<MemoryBuffer>> tryLoadingBuffer() {
402 if (EntryPath.empty())
403 return std::error_code();
404 SmallString<64> ResultPath;
405 Expected<sys::fs::file_t> FDOrErr = sys::fs::openNativeFileForRead(
406 Name: Twine(EntryPath), Flags: sys::fs::OF_UpdateAtime, RealPath: &ResultPath);
407 if (!FDOrErr)
408 return errorToErrorCode(Err: FDOrErr.takeError());
409 ErrorOr<std::unique_ptr<MemoryBuffer>> MBOrErr = MemoryBuffer::getOpenFile(
410 FD: *FDOrErr, Filename: EntryPath, /*FileSize=*/-1, /*RequiresNullTerminator=*/false);
411 sys::fs::closeFile(F&: *FDOrErr);
412 return MBOrErr;
413 }
414
415 // Cache the Produced object file
416 void write(const MemoryBuffer &OutputBuffer) {
417 if (EntryPath.empty())
418 return;
419
420 if (auto Err = llvm::writeToOutput(
421 OutputFileName: EntryPath, Write: [&OutputBuffer](llvm::raw_ostream &OS) -> llvm::Error {
422 OS << OutputBuffer.getBuffer();
423 return llvm::Error::success();
424 }))
425 report_fatal_error(reason: llvm::formatv(Fmt: "ThinLTO: Can't write file {0}: {1}",
426 Vals&: EntryPath,
427 Vals: toString(E: std::move(Err)).c_str()));
428 }
429};
430
431static std::unique_ptr<MemoryBuffer>
432ProcessThinLTOModule(Module &TheModule, ModuleSummaryIndex &Index,
433 StringMap<lto::InputFile *> &ModuleMap, TargetMachine &TM,
434 const FunctionImporter::ImportMapTy &ImportList,
435 const FunctionImporter::ExportSetTy &ExportList,
436 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols,
437 const GVSummaryMapTy &DefinedGlobals,
438 const ThinLTOCodeGenerator::CachingOptions &CacheOptions,
439 bool DisableCodeGen, StringRef SaveTempsDir,
440 bool Freestanding, unsigned OptLevel, unsigned count,
441 bool DebugPassManager) {
442 // "Benchmark"-like optimization: single-source case
443 bool SingleModule = (ModuleMap.size() == 1);
444
445 // When linking an ELF shared object, dso_local should be dropped. We
446 // conservatively do this for -fpic.
447 bool ClearDSOLocalOnDeclarations =
448 TM.getTargetTriple().isOSBinFormatELF() &&
449 TM.getRelocationModel() != Reloc::Static &&
450 TheModule.getPIELevel() == PIELevel::Default;
451
452 if (!SingleModule) {
453 promoteModule(TheModule, Index, ClearDSOLocalOnDeclarations);
454
455 // Apply summary-based prevailing-symbol resolution decisions.
456 thinLTOFinalizeInModule(TheModule, DefinedGlobals, /*PropagateAttrs=*/true);
457
458 // Save temps: after promotion.
459 saveTempBitcode(TheModule, TempDir: SaveTempsDir, count, Suffix: ".1.promoted.bc");
460 }
461
462 // Be friendly and don't nuke totally the module when the client didn't
463 // supply anything to preserve.
464 if (!ExportList.empty() || !GUIDPreservedSymbols.empty()) {
465 // Apply summary-based internalization decisions.
466 thinLTOInternalizeModule(TheModule, DefinedGlobals);
467 }
468
469 // Save internalized bitcode
470 saveTempBitcode(TheModule, TempDir: SaveTempsDir, count, Suffix: ".2.internalized.bc");
471
472 if (!SingleModule)
473 crossImportIntoModule(TheModule, Index, ModuleMap, ImportList,
474 ClearDSOLocalOnDeclarations);
475
476 // Do this after any importing so that imported code is updated.
477 // See comment at call to updateVCallVisibilityInIndex() for why
478 // WholeProgramVisibilityEnabledInLTO is false.
479 updatePublicTypeTestCalls(M&: TheModule,
480 /* WholeProgramVisibilityEnabledInLTO */ false);
481
482 // Save temps: after cross-module import.
483 saveTempBitcode(TheModule, TempDir: SaveTempsDir, count, Suffix: ".3.imported.bc");
484
485 optimizeModule(TheModule, TM, OptLevel, Freestanding, DebugPassManager,
486 Index: &Index);
487
488 saveTempBitcode(TheModule, TempDir: SaveTempsDir, count, Suffix: ".4.opt.bc");
489
490 if (DisableCodeGen) {
491 // Configured to stop before CodeGen, serialize the bitcode and return.
492 SmallVector<char, 128> OutputBuffer;
493 {
494 raw_svector_ostream OS(OutputBuffer);
495 ProfileSummaryInfo PSI(TheModule);
496 auto Index = buildModuleSummaryIndex(M: TheModule, GetBFICallback: nullptr, PSI: &PSI);
497 WriteBitcodeToFile(M: TheModule, Out&: OS, ShouldPreserveUseListOrder: true, Index: &Index);
498 }
499 return std::make_unique<SmallVectorMemoryBuffer>(
500 args: std::move(OutputBuffer), /*RequiresNullTerminator=*/args: false);
501 }
502
503 return codegenModule(TheModule, TM);
504}
505
506/// Resolve prevailing symbols. Record resolutions in the \p ResolvedODR map
507/// for caching, and in the \p Index for application during the ThinLTO
508/// backends. This is needed for correctness for exported symbols (ensure
509/// at least one copy kept) and a compile-time optimization (to drop duplicate
510/// copies when possible).
511static void resolvePrevailingInIndex(
512 ModuleSummaryIndex &Index,
513 StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>>
514 &ResolvedODR,
515 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols,
516 const DenseMap<GlobalValue::GUID, const GlobalValueSummary *>
517 &PrevailingCopy) {
518
519 auto isPrevailing = [&](GlobalValue::GUID GUID, const GlobalValueSummary *S) {
520 const auto &Prevailing = PrevailingCopy.find(Val: GUID);
521 // Not in map means that there was only one copy, which must be prevailing.
522 if (Prevailing == PrevailingCopy.end())
523 return true;
524 return Prevailing->second == S;
525 };
526
527 auto recordNewLinkage = [&](StringRef ModuleIdentifier,
528 GlobalValue::GUID GUID,
529 GlobalValue::LinkageTypes NewLinkage) {
530 ResolvedODR[ModuleIdentifier][GUID] = NewLinkage;
531 };
532
533 // TODO Conf.VisibilityScheme can be lto::Config::ELF for ELF.
534 lto::Config Conf;
535 thinLTOResolvePrevailingInIndex(C: Conf, Index, isPrevailing, recordNewLinkage,
536 GUIDPreservedSymbols);
537}
538
539// Initialize the TargetMachine builder for a given Triple
540static void initTMBuilder(TargetMachineBuilder &TMBuilder,
541 const Triple &TheTriple) {
542 // Set a default CPU for Darwin triples (copied from LTOCodeGenerator).
543 // FIXME this looks pretty terrible...
544 if (TMBuilder.MCpu.empty() && TheTriple.isOSDarwin()) {
545 if (TheTriple.getArch() == llvm::Triple::x86_64)
546 TMBuilder.MCpu = "core2";
547 else if (TheTriple.getArch() == llvm::Triple::x86)
548 TMBuilder.MCpu = "yonah";
549 else if (TheTriple.getArch() == llvm::Triple::aarch64 ||
550 TheTriple.getArch() == llvm::Triple::aarch64_32)
551 TMBuilder.MCpu = "cyclone";
552 }
553 TMBuilder.TheTriple = std::move(TheTriple);
554}
555
556} // end anonymous namespace
557
558void ThinLTOCodeGenerator::addModule(StringRef Identifier, StringRef Data) {
559 MemoryBufferRef Buffer(Data, Identifier);
560
561 auto InputOrError = lto::InputFile::create(Object: Buffer);
562 if (!InputOrError)
563 report_fatal_error(reason: Twine("ThinLTO cannot create input file: ") +
564 toString(E: InputOrError.takeError()));
565
566 auto TripleStr = (*InputOrError)->getTargetTriple();
567 Triple TheTriple(TripleStr);
568
569 if (Modules.empty())
570 initTMBuilder(TMBuilder, TheTriple: Triple(TheTriple));
571 else if (TMBuilder.TheTriple != TheTriple) {
572 if (!TMBuilder.TheTriple.isCompatibleWith(Other: TheTriple))
573 report_fatal_error(reason: "ThinLTO modules with incompatible triples not "
574 "supported");
575 initTMBuilder(TMBuilder, TheTriple: Triple(TMBuilder.TheTriple.merge(Other: TheTriple)));
576 }
577
578 Modules.emplace_back(args: std::move(*InputOrError));
579}
580
581void ThinLTOCodeGenerator::preserveSymbol(StringRef Name) {
582 PreservedSymbols.insert(key: Name);
583}
584
585void ThinLTOCodeGenerator::crossReferenceSymbol(StringRef Name) {
586 // FIXME: At the moment, we don't take advantage of this extra information,
587 // we're conservatively considering cross-references as preserved.
588 // CrossReferencedSymbols.insert(Name);
589 PreservedSymbols.insert(key: Name);
590}
591
592// TargetMachine factory
593std::unique_ptr<TargetMachine> TargetMachineBuilder::create() const {
594 std::string ErrMsg;
595 const Target *TheTarget =
596 TargetRegistry::lookupTarget(Triple: TheTriple.str(), Error&: ErrMsg);
597 if (!TheTarget) {
598 report_fatal_error(reason: Twine("Can't load target for this Triple: ") + ErrMsg);
599 }
600
601 // Use MAttr as the default set of features.
602 SubtargetFeatures Features(MAttr);
603 Features.getDefaultSubtargetFeatures(Triple: TheTriple);
604 std::string FeatureStr = Features.getString();
605
606 std::unique_ptr<TargetMachine> TM(
607 TheTarget->createTargetMachine(TT: TheTriple.str(), CPU: MCpu, Features: FeatureStr, Options,
608 RM: RelocModel, CM: std::nullopt, OL: CGOptLevel));
609 assert(TM && "Cannot create target machine");
610
611 return TM;
612}
613
614/**
615 * Produce the combined summary index from all the bitcode files:
616 * "thin-link".
617 */
618std::unique_ptr<ModuleSummaryIndex> ThinLTOCodeGenerator::linkCombinedIndex() {
619 std::unique_ptr<ModuleSummaryIndex> CombinedIndex =
620 std::make_unique<ModuleSummaryIndex>(/*HaveGVs=*/args: false);
621 for (auto &Mod : Modules) {
622 auto &M = Mod->getSingleBitcodeModule();
623 if (Error Err = M.readSummary(CombinedIndex&: *CombinedIndex, ModulePath: Mod->getName())) {
624 // FIXME diagnose
625 logAllUnhandledErrors(
626 E: std::move(Err), OS&: errs(),
627 ErrorBanner: "error: can't create module summary index for buffer: ");
628 return nullptr;
629 }
630 }
631 return CombinedIndex;
632}
633
634namespace {
635struct IsExported {
636 const DenseMap<StringRef, FunctionImporter::ExportSetTy> &ExportLists;
637 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols;
638
639 IsExported(
640 const DenseMap<StringRef, FunctionImporter::ExportSetTy> &ExportLists,
641 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols)
642 : ExportLists(ExportLists), GUIDPreservedSymbols(GUIDPreservedSymbols) {}
643
644 bool operator()(StringRef ModuleIdentifier, ValueInfo VI) const {
645 const auto &ExportList = ExportLists.find(Val: ModuleIdentifier);
646 return (ExportList != ExportLists.end() && ExportList->second.count(V: VI)) ||
647 GUIDPreservedSymbols.count(V: VI.getGUID());
648 }
649};
650
651struct IsPrevailing {
652 const DenseMap<GlobalValue::GUID, const GlobalValueSummary *> &PrevailingCopy;
653 IsPrevailing(const DenseMap<GlobalValue::GUID, const GlobalValueSummary *>
654 &PrevailingCopy)
655 : PrevailingCopy(PrevailingCopy) {}
656
657 bool operator()(GlobalValue::GUID GUID, const GlobalValueSummary *S) const {
658 const auto &Prevailing = PrevailingCopy.find(Val: GUID);
659 // Not in map means that there was only one copy, which must be prevailing.
660 if (Prevailing == PrevailingCopy.end())
661 return true;
662 return Prevailing->second == S;
663 };
664};
665} // namespace
666
667static void computeDeadSymbolsInIndex(
668 ModuleSummaryIndex &Index,
669 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols) {
670 // We have no symbols resolution available. And can't do any better now in the
671 // case where the prevailing symbol is in a native object. It can be refined
672 // with linker information in the future.
673 auto isPrevailing = [&](GlobalValue::GUID G) {
674 return PrevailingType::Unknown;
675 };
676 computeDeadSymbolsWithConstProp(Index, GUIDPreservedSymbols, isPrevailing,
677 /* ImportEnabled = */ true);
678}
679
680/**
681 * Perform promotion and renaming of exported internal functions.
682 * Index is updated to reflect linkage changes from weak resolution.
683 */
684void ThinLTOCodeGenerator::promote(Module &TheModule, ModuleSummaryIndex &Index,
685 const lto::InputFile &File) {
686 auto ModuleCount = Index.modulePaths().size();
687 auto ModuleIdentifier = TheModule.getModuleIdentifier();
688
689 // Collect for each module the list of function it defines (GUID -> Summary).
690 DenseMap<StringRef, GVSummaryMapTy> ModuleToDefinedGVSummaries;
691 Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
692
693 // Convert the preserved symbols set from string to GUID
694 auto GUIDPreservedSymbols = computeGUIDPreservedSymbols(
695 File, PreservedSymbols, TheTriple: Triple(TheModule.getTargetTriple()));
696
697 // Add used symbol to the preserved symbols.
698 addUsedSymbolToPreservedGUID(File, PreservedGUID&: GUIDPreservedSymbols);
699
700 // Compute "dead" symbols, we don't want to import/export these!
701 computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols);
702
703 // Compute prevailing symbols
704 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
705 computePrevailingCopies(Index, PrevailingCopy);
706
707 // Generate import/export list
708 DenseMap<StringRef, FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
709 DenseMap<StringRef, FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
710 ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries,
711 isPrevailing: IsPrevailing(PrevailingCopy), ImportLists,
712 ExportLists);
713
714 // Resolve prevailing symbols
715 StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
716 resolvePrevailingInIndex(Index, ResolvedODR, GUIDPreservedSymbols,
717 PrevailingCopy);
718
719 thinLTOFinalizeInModule(TheModule,
720 DefinedGlobals: ModuleToDefinedGVSummaries[ModuleIdentifier],
721 /*PropagateAttrs=*/false);
722
723 // Promote the exported values in the index, so that they are promoted
724 // in the module.
725 thinLTOInternalizeAndPromoteInIndex(
726 Index, isExported: IsExported(ExportLists, GUIDPreservedSymbols),
727 isPrevailing: IsPrevailing(PrevailingCopy));
728
729 // FIXME Set ClearDSOLocalOnDeclarations.
730 promoteModule(TheModule, Index, /*ClearDSOLocalOnDeclarations=*/false);
731}
732
733/**
734 * Perform cross-module importing for the module identified by ModuleIdentifier.
735 */
736void ThinLTOCodeGenerator::crossModuleImport(Module &TheModule,
737 ModuleSummaryIndex &Index,
738 const lto::InputFile &File) {
739 auto ModuleMap = generateModuleMap(Modules);
740 auto ModuleCount = Index.modulePaths().size();
741
742 // Collect for each module the list of function it defines (GUID -> Summary).
743 DenseMap<StringRef, GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
744 Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
745
746 // Convert the preserved symbols set from string to GUID
747 auto GUIDPreservedSymbols = computeGUIDPreservedSymbols(
748 File, PreservedSymbols, TheTriple: Triple(TheModule.getTargetTriple()));
749
750 addUsedSymbolToPreservedGUID(File, PreservedGUID&: GUIDPreservedSymbols);
751
752 // Compute "dead" symbols, we don't want to import/export these!
753 computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols);
754
755 // Compute prevailing symbols
756 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
757 computePrevailingCopies(Index, PrevailingCopy);
758
759 // Generate import/export list
760 DenseMap<StringRef, FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
761 DenseMap<StringRef, FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
762 ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries,
763 isPrevailing: IsPrevailing(PrevailingCopy), ImportLists,
764 ExportLists);
765 auto &ImportList = ImportLists[TheModule.getModuleIdentifier()];
766
767 // FIXME Set ClearDSOLocalOnDeclarations.
768 crossImportIntoModule(TheModule, Index, ModuleMap, ImportList,
769 /*ClearDSOLocalOnDeclarations=*/false);
770}
771
772/**
773 * Compute the list of summaries needed for importing into module.
774 */
775void ThinLTOCodeGenerator::gatherImportedSummariesForModule(
776 Module &TheModule, ModuleSummaryIndex &Index,
777 std::map<std::string, GVSummaryMapTy> &ModuleToSummariesForIndex,
778 const lto::InputFile &File) {
779 auto ModuleCount = Index.modulePaths().size();
780 auto ModuleIdentifier = TheModule.getModuleIdentifier();
781
782 // Collect for each module the list of function it defines (GUID -> Summary).
783 DenseMap<StringRef, GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
784 Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
785
786 // Convert the preserved symbols set from string to GUID
787 auto GUIDPreservedSymbols = computeGUIDPreservedSymbols(
788 File, PreservedSymbols, TheTriple: Triple(TheModule.getTargetTriple()));
789
790 addUsedSymbolToPreservedGUID(File, PreservedGUID&: GUIDPreservedSymbols);
791
792 // Compute "dead" symbols, we don't want to import/export these!
793 computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols);
794
795 // Compute prevailing symbols
796 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
797 computePrevailingCopies(Index, PrevailingCopy);
798
799 // Generate import/export list
800 DenseMap<StringRef, FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
801 DenseMap<StringRef, FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
802 ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries,
803 isPrevailing: IsPrevailing(PrevailingCopy), ImportLists,
804 ExportLists);
805
806 llvm::gatherImportedSummariesForModule(
807 ModulePath: ModuleIdentifier, ModuleToDefinedGVSummaries,
808 ImportList: ImportLists[ModuleIdentifier], ModuleToSummariesForIndex);
809}
810
811/**
812 * Emit the list of files needed for importing into module.
813 */
814void ThinLTOCodeGenerator::emitImports(Module &TheModule, StringRef OutputName,
815 ModuleSummaryIndex &Index,
816 const lto::InputFile &File) {
817 auto ModuleCount = Index.modulePaths().size();
818 auto ModuleIdentifier = TheModule.getModuleIdentifier();
819
820 // Collect for each module the list of function it defines (GUID -> Summary).
821 DenseMap<StringRef, GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
822 Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
823
824 // Convert the preserved symbols set from string to GUID
825 auto GUIDPreservedSymbols = computeGUIDPreservedSymbols(
826 File, PreservedSymbols, TheTriple: Triple(TheModule.getTargetTriple()));
827
828 addUsedSymbolToPreservedGUID(File, PreservedGUID&: GUIDPreservedSymbols);
829
830 // Compute "dead" symbols, we don't want to import/export these!
831 computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols);
832
833 // Compute prevailing symbols
834 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
835 computePrevailingCopies(Index, PrevailingCopy);
836
837 // Generate import/export list
838 DenseMap<StringRef, FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
839 DenseMap<StringRef, FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
840 ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries,
841 isPrevailing: IsPrevailing(PrevailingCopy), ImportLists,
842 ExportLists);
843
844 std::map<std::string, GVSummaryMapTy> ModuleToSummariesForIndex;
845 llvm::gatherImportedSummariesForModule(
846 ModulePath: ModuleIdentifier, ModuleToDefinedGVSummaries,
847 ImportList: ImportLists[ModuleIdentifier], ModuleToSummariesForIndex);
848
849 std::error_code EC;
850 if ((EC = EmitImportsFiles(ModulePath: ModuleIdentifier, OutputFilename: OutputName,
851 ModuleToSummariesForIndex)))
852 report_fatal_error(reason: Twine("Failed to open ") + OutputName +
853 " to save imports lists\n");
854}
855
856/**
857 * Perform internalization. Runs promote and internalization together.
858 * Index is updated to reflect linkage changes.
859 */
860void ThinLTOCodeGenerator::internalize(Module &TheModule,
861 ModuleSummaryIndex &Index,
862 const lto::InputFile &File) {
863 initTMBuilder(TMBuilder, TheTriple: Triple(TheModule.getTargetTriple()));
864 auto ModuleCount = Index.modulePaths().size();
865 auto ModuleIdentifier = TheModule.getModuleIdentifier();
866
867 // Convert the preserved symbols set from string to GUID
868 auto GUIDPreservedSymbols =
869 computeGUIDPreservedSymbols(File, PreservedSymbols, TheTriple: TMBuilder.TheTriple);
870
871 addUsedSymbolToPreservedGUID(File, PreservedGUID&: GUIDPreservedSymbols);
872
873 // Collect for each module the list of function it defines (GUID -> Summary).
874 DenseMap<StringRef, GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
875 Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
876
877 // Compute "dead" symbols, we don't want to import/export these!
878 computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols);
879
880 // Compute prevailing symbols
881 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
882 computePrevailingCopies(Index, PrevailingCopy);
883
884 // Generate import/export list
885 DenseMap<StringRef, FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
886 DenseMap<StringRef, FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
887 ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries,
888 isPrevailing: IsPrevailing(PrevailingCopy), ImportLists,
889 ExportLists);
890 auto &ExportList = ExportLists[ModuleIdentifier];
891
892 // Be friendly and don't nuke totally the module when the client didn't
893 // supply anything to preserve.
894 if (ExportList.empty() && GUIDPreservedSymbols.empty())
895 return;
896
897 // Resolve prevailing symbols
898 StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
899 resolvePrevailingInIndex(Index, ResolvedODR, GUIDPreservedSymbols,
900 PrevailingCopy);
901
902 // Promote the exported values in the index, so that they are promoted
903 // in the module.
904 thinLTOInternalizeAndPromoteInIndex(
905 Index, isExported: IsExported(ExportLists, GUIDPreservedSymbols),
906 isPrevailing: IsPrevailing(PrevailingCopy));
907
908 // FIXME Set ClearDSOLocalOnDeclarations.
909 promoteModule(TheModule, Index, /*ClearDSOLocalOnDeclarations=*/false);
910
911 // Internalization
912 thinLTOFinalizeInModule(TheModule,
913 DefinedGlobals: ModuleToDefinedGVSummaries[ModuleIdentifier],
914 /*PropagateAttrs=*/false);
915
916 thinLTOInternalizeModule(TheModule,
917 DefinedGlobals: ModuleToDefinedGVSummaries[ModuleIdentifier]);
918}
919
920/**
921 * Perform post-importing ThinLTO optimizations.
922 */
923void ThinLTOCodeGenerator::optimize(Module &TheModule) {
924 initTMBuilder(TMBuilder, TheTriple: Triple(TheModule.getTargetTriple()));
925
926 // Optimize now
927 optimizeModule(TheModule, TM&: *TMBuilder.create(), OptLevel, Freestanding,
928 DebugPassManager, Index: nullptr);
929}
930
931/// Write out the generated object file, either from CacheEntryPath or from
932/// OutputBuffer, preferring hard-link when possible.
933/// Returns the path to the generated file in SavedObjectsDirectoryPath.
934std::string
935ThinLTOCodeGenerator::writeGeneratedObject(int count, StringRef CacheEntryPath,
936 const MemoryBuffer &OutputBuffer) {
937 auto ArchName = TMBuilder.TheTriple.getArchName();
938 SmallString<128> OutputPath(SavedObjectsDirectoryPath);
939 llvm::sys::path::append(path&: OutputPath,
940 a: Twine(count) + "." + ArchName + ".thinlto.o");
941 OutputPath.c_str(); // Ensure the string is null terminated.
942 if (sys::fs::exists(Path: OutputPath))
943 sys::fs::remove(path: OutputPath);
944
945 // We don't return a memory buffer to the linker, just a list of files.
946 if (!CacheEntryPath.empty()) {
947 // Cache is enabled, hard-link the entry (or copy if hard-link fails).
948 auto Err = sys::fs::create_hard_link(to: CacheEntryPath, from: OutputPath);
949 if (!Err)
950 return std::string(OutputPath);
951 // Hard linking failed, try to copy.
952 Err = sys::fs::copy_file(From: CacheEntryPath, To: OutputPath);
953 if (!Err)
954 return std::string(OutputPath);
955 // Copy failed (could be because the CacheEntry was removed from the cache
956 // in the meantime by another process), fall back and try to write down the
957 // buffer to the output.
958 errs() << "remark: can't link or copy from cached entry '" << CacheEntryPath
959 << "' to '" << OutputPath << "'\n";
960 }
961 // No cache entry, just write out the buffer.
962 std::error_code Err;
963 raw_fd_ostream OS(OutputPath, Err, sys::fs::OF_None);
964 if (Err)
965 report_fatal_error(reason: Twine("Can't open output '") + OutputPath + "'\n");
966 OS << OutputBuffer.getBuffer();
967 return std::string(OutputPath);
968}
969
970// Main entry point for the ThinLTO processing
971void ThinLTOCodeGenerator::run() {
972 timeTraceProfilerBegin(Name: "ThinLink", Detail: StringRef(""));
973 auto TimeTraceScopeExit = llvm::make_scope_exit(F: []() {
974 if (llvm::timeTraceProfilerEnabled())
975 llvm::timeTraceProfilerEnd();
976 });
977 // Prepare the resulting object vector
978 assert(ProducedBinaries.empty() && "The generator should not be reused");
979 if (SavedObjectsDirectoryPath.empty())
980 ProducedBinaries.resize(new_size: Modules.size());
981 else {
982 sys::fs::create_directories(path: SavedObjectsDirectoryPath);
983 bool IsDir;
984 sys::fs::is_directory(path: SavedObjectsDirectoryPath, result&: IsDir);
985 if (!IsDir)
986 report_fatal_error(reason: Twine("Unexistent dir: '") + SavedObjectsDirectoryPath + "'");
987 ProducedBinaryFiles.resize(new_size: Modules.size());
988 }
989
990 if (CodeGenOnly) {
991 // Perform only parallel codegen and return.
992 ThreadPool Pool;
993 int count = 0;
994 for (auto &Mod : Modules) {
995 Pool.async(F: [&](int count) {
996 LLVMContext Context;
997 Context.setDiscardValueNames(LTODiscardValueNames);
998
999 // Parse module now
1000 auto TheModule = loadModuleFromInput(Input: Mod.get(), Context, Lazy: false,
1001 /*IsImporting*/ false);
1002
1003 // CodeGen
1004 auto OutputBuffer = codegenModule(TheModule&: *TheModule, TM&: *TMBuilder.create());
1005 if (SavedObjectsDirectoryPath.empty())
1006 ProducedBinaries[count] = std::move(OutputBuffer);
1007 else
1008 ProducedBinaryFiles[count] =
1009 writeGeneratedObject(count, CacheEntryPath: "", OutputBuffer: *OutputBuffer);
1010 }, ArgList: count++);
1011 }
1012
1013 return;
1014 }
1015
1016 // Sequential linking phase
1017 auto Index = linkCombinedIndex();
1018
1019 // Save temps: index.
1020 if (!SaveTempsDir.empty()) {
1021 auto SaveTempPath = SaveTempsDir + "index.bc";
1022 std::error_code EC;
1023 raw_fd_ostream OS(SaveTempPath, EC, sys::fs::OF_None);
1024 if (EC)
1025 report_fatal_error(reason: Twine("Failed to open ") + SaveTempPath +
1026 " to save optimized bitcode\n");
1027 writeIndexToFile(Index: *Index, Out&: OS);
1028 }
1029
1030
1031 // Prepare the module map.
1032 auto ModuleMap = generateModuleMap(Modules);
1033 auto ModuleCount = Modules.size();
1034
1035 // Collect for each module the list of function it defines (GUID -> Summary).
1036 DenseMap<StringRef, GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
1037 Index->collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
1038
1039 // Convert the preserved symbols set from string to GUID, this is needed for
1040 // computing the caching hash and the internalization.
1041 DenseSet<GlobalValue::GUID> GUIDPreservedSymbols;
1042 for (const auto &M : Modules)
1043 computeGUIDPreservedSymbols(File: *M, PreservedSymbols, TheTriple: TMBuilder.TheTriple,
1044 GUIDs&: GUIDPreservedSymbols);
1045
1046 // Add used symbol from inputs to the preserved symbols.
1047 for (const auto &M : Modules)
1048 addUsedSymbolToPreservedGUID(File: *M, PreservedGUID&: GUIDPreservedSymbols);
1049
1050 // Compute "dead" symbols, we don't want to import/export these!
1051 computeDeadSymbolsInIndex(Index&: *Index, GUIDPreservedSymbols);
1052
1053 // Synthesize entry counts for functions in the combined index.
1054 computeSyntheticCounts(Index&: *Index);
1055
1056 // Currently there is no support for enabling whole program visibility via a
1057 // linker option in the old LTO API, but this call allows it to be specified
1058 // via the internal option. Must be done before WPD below.
1059 if (hasWholeProgramVisibility(/* WholeProgramVisibilityEnabledInLTO */ false))
1060 Index->setWithWholeProgramVisibility();
1061
1062 // FIXME: This needs linker information via a TBD new interface
1063 updateVCallVisibilityInIndex(Index&: *Index,
1064 /*WholeProgramVisibilityEnabledInLTO=*/false,
1065 // FIXME: These need linker information via a
1066 // TBD new interface.
1067 /*DynamicExportSymbols=*/{},
1068 /*VisibleToRegularObjSymbols=*/{});
1069
1070 // Perform index-based WPD. This will return immediately if there are
1071 // no index entries in the typeIdMetadata map (e.g. if we are instead
1072 // performing IR-based WPD in hybrid regular/thin LTO mode).
1073 std::map<ValueInfo, std::vector<VTableSlotSummary>> LocalWPDTargetsMap;
1074 std::set<GlobalValue::GUID> ExportedGUIDs;
1075 runWholeProgramDevirtOnIndex(Summary&: *Index, ExportedGUIDs, LocalWPDTargetsMap);
1076 for (auto GUID : ExportedGUIDs)
1077 GUIDPreservedSymbols.insert(V: GUID);
1078
1079 // Compute prevailing symbols
1080 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
1081 computePrevailingCopies(Index: *Index, PrevailingCopy);
1082
1083 // Collect the import/export lists for all modules from the call-graph in the
1084 // combined index.
1085 DenseMap<StringRef, FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
1086 DenseMap<StringRef, FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
1087 ComputeCrossModuleImport(Index: *Index, ModuleToDefinedGVSummaries,
1088 isPrevailing: IsPrevailing(PrevailingCopy), ImportLists,
1089 ExportLists);
1090
1091 // We use a std::map here to be able to have a defined ordering when
1092 // producing a hash for the cache entry.
1093 // FIXME: we should be able to compute the caching hash for the entry based
1094 // on the index, and nuke this map.
1095 StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
1096
1097 // Resolve prevailing symbols, this has to be computed early because it
1098 // impacts the caching.
1099 resolvePrevailingInIndex(Index&: *Index, ResolvedODR, GUIDPreservedSymbols,
1100 PrevailingCopy);
1101
1102 // Use global summary-based analysis to identify symbols that can be
1103 // internalized (because they aren't exported or preserved as per callback).
1104 // Changes are made in the index, consumed in the ThinLTO backends.
1105 updateIndexWPDForExports(Summary&: *Index,
1106 isExported: IsExported(ExportLists, GUIDPreservedSymbols),
1107 LocalWPDTargetsMap);
1108 thinLTOInternalizeAndPromoteInIndex(
1109 Index&: *Index, isExported: IsExported(ExportLists, GUIDPreservedSymbols),
1110 isPrevailing: IsPrevailing(PrevailingCopy));
1111
1112 thinLTOPropagateFunctionAttrs(Index&: *Index, isPrevailing: IsPrevailing(PrevailingCopy));
1113
1114 // Make sure that every module has an entry in the ExportLists, ImportList,
1115 // GVSummary and ResolvedODR maps to enable threaded access to these maps
1116 // below.
1117 for (auto &Module : Modules) {
1118 auto ModuleIdentifier = Module->getName();
1119 ExportLists[ModuleIdentifier];
1120 ImportLists[ModuleIdentifier];
1121 ResolvedODR[ModuleIdentifier];
1122 ModuleToDefinedGVSummaries[ModuleIdentifier];
1123 }
1124
1125 std::vector<BitcodeModule *> ModulesVec;
1126 ModulesVec.reserve(n: Modules.size());
1127 for (auto &Mod : Modules)
1128 ModulesVec.push_back(x: &Mod->getSingleBitcodeModule());
1129 std::vector<int> ModulesOrdering = lto::generateModulesOrdering(R: ModulesVec);
1130
1131 if (llvm::timeTraceProfilerEnabled())
1132 llvm::timeTraceProfilerEnd();
1133
1134 TimeTraceScopeExit.release();
1135
1136 // Parallel optimizer + codegen
1137 {
1138 ThreadPool Pool(heavyweight_hardware_concurrency(ThreadCount));
1139 for (auto IndexCount : ModulesOrdering) {
1140 auto &Mod = Modules[IndexCount];
1141 Pool.async(F: [&](int count) {
1142 auto ModuleIdentifier = Mod->getName();
1143 auto &ExportList = ExportLists[ModuleIdentifier];
1144
1145 auto &DefinedGVSummaries = ModuleToDefinedGVSummaries[ModuleIdentifier];
1146
1147 // The module may be cached, this helps handling it.
1148 ModuleCacheEntry CacheEntry(CacheOptions.Path, *Index, ModuleIdentifier,
1149 ImportLists[ModuleIdentifier], ExportList,
1150 ResolvedODR[ModuleIdentifier],
1151 DefinedGVSummaries, OptLevel, Freestanding,
1152 TMBuilder);
1153 auto CacheEntryPath = CacheEntry.getEntryPath();
1154
1155 {
1156 auto ErrOrBuffer = CacheEntry.tryLoadingBuffer();
1157 LLVM_DEBUG(dbgs() << "Cache " << (ErrOrBuffer ? "hit" : "miss")
1158 << " '" << CacheEntryPath << "' for buffer "
1159 << count << " " << ModuleIdentifier << "\n");
1160
1161 if (ErrOrBuffer) {
1162 // Cache Hit!
1163 if (SavedObjectsDirectoryPath.empty())
1164 ProducedBinaries[count] = std::move(ErrOrBuffer.get());
1165 else
1166 ProducedBinaryFiles[count] = writeGeneratedObject(
1167 count, CacheEntryPath, OutputBuffer: *ErrOrBuffer.get());
1168 return;
1169 }
1170 }
1171
1172 LLVMContext Context;
1173 Context.setDiscardValueNames(LTODiscardValueNames);
1174 Context.enableDebugTypeODRUniquing();
1175 auto DiagFileOrErr = lto::setupLLVMOptimizationRemarks(
1176 Context, RemarksFilename, RemarksPasses, RemarksFormat,
1177 RemarksWithHotness, RemarksHotnessThreshold, Count: count);
1178 if (!DiagFileOrErr) {
1179 errs() << "Error: " << toString(E: DiagFileOrErr.takeError()) << "\n";
1180 report_fatal_error(reason: "ThinLTO: Can't get an output file for the "
1181 "remarks");
1182 }
1183
1184 // Parse module now
1185 auto TheModule = loadModuleFromInput(Input: Mod.get(), Context, Lazy: false,
1186 /*IsImporting*/ false);
1187
1188 // Save temps: original file.
1189 saveTempBitcode(TheModule: *TheModule, TempDir: SaveTempsDir, count, Suffix: ".0.original.bc");
1190
1191 auto &ImportList = ImportLists[ModuleIdentifier];
1192 // Run the main process now, and generates a binary
1193 auto OutputBuffer = ProcessThinLTOModule(
1194 TheModule&: *TheModule, Index&: *Index, ModuleMap, TM&: *TMBuilder.create(), ImportList,
1195 ExportList, GUIDPreservedSymbols,
1196 DefinedGlobals: ModuleToDefinedGVSummaries[ModuleIdentifier], CacheOptions,
1197 DisableCodeGen, SaveTempsDir, Freestanding, OptLevel, count,
1198 DebugPassManager);
1199
1200 // Commit to the cache (if enabled)
1201 CacheEntry.write(OutputBuffer: *OutputBuffer);
1202
1203 if (SavedObjectsDirectoryPath.empty()) {
1204 // We need to generated a memory buffer for the linker.
1205 if (!CacheEntryPath.empty()) {
1206 // When cache is enabled, reload from the cache if possible.
1207 // Releasing the buffer from the heap and reloading it from the
1208 // cache file with mmap helps us to lower memory pressure.
1209 // The freed memory can be used for the next input file.
1210 // The final binary link will read from the VFS cache (hopefully!)
1211 // or from disk (if the memory pressure was too high).
1212 auto ReloadedBufferOrErr = CacheEntry.tryLoadingBuffer();
1213 if (auto EC = ReloadedBufferOrErr.getError()) {
1214 // On error, keep the preexisting buffer and print a diagnostic.
1215 errs() << "remark: can't reload cached file '" << CacheEntryPath
1216 << "': " << EC.message() << "\n";
1217 } else {
1218 OutputBuffer = std::move(*ReloadedBufferOrErr);
1219 }
1220 }
1221 ProducedBinaries[count] = std::move(OutputBuffer);
1222 return;
1223 }
1224 ProducedBinaryFiles[count] = writeGeneratedObject(
1225 count, CacheEntryPath, OutputBuffer: *OutputBuffer);
1226 }, ArgList&: IndexCount);
1227 }
1228 }
1229
1230 pruneCache(Path: CacheOptions.Path, Policy: CacheOptions.Policy, Files: ProducedBinaries);
1231
1232 // If statistics were requested, print them out now.
1233 if (llvm::AreStatisticsEnabled())
1234 llvm::PrintStatistics();
1235 reportAndResetTimings();
1236}
1237

source code of llvm/lib/LTO/ThinLTOCodeGenerator.cpp